投稿
  1. 极客之音首页
  2. 其他分类
  3. 云原生

深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘

青莲明月 • • 云原生 • 阅读 262

4.2、基于 Elasticsearch + kibana+logstash 实现 IP 地址分布地图可视化

4.2.1、ES解决geoip的location不为geo_point格式

问题：使用了geoip插件，入库的索引中location仍不是geo_point格式，以致想做地图经纬度的展示时做不了。原因: 默认的log_template模板中没有任何匹配。解决办法：新建索引模板，设置匹配成geo_point，且优先度调高。另外为防以后新项目入log重新碰到这个问题，将默认的log_template模板也加上geo_point的相关设置 打开kibana的Dev Tools，本次操作索引模板在web界面下操作比较方便。log_template会默认就匹配上我们的索引：在kibana的Dev Tools里的Console中输入 GET /_template/logstash 点击三角形图标执行，右边会出现结果：

深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘
1660119055262.png

默认应该是没有这些的 index_patterns中可以看到匹配索引名称中带log的：

"index_patterns" : [
      "*upixel-h5*"

所以以防万一，在这个默认优先级0的模板中也加上：

PUT /_template/logstash
{
  "order" : 0,
  "index_patterns" : [
      "*upixel-h5*"
  ],
  "settings" : {
    "index" : {
      "number_of_replicas" : "0"
    }
  },
  "mappings" : {
    "properties": {
      "client_ip": {
        "type": "ip"
      },
      "geoip": {
        "dynamic": true,
        "type": "object",
        "properties": {
          "location": {
            "type": "geo_point"
          }
        }
      }
    }
  },
  "aliases" : { }
}

这里主要是看：

"mappings" : {
    "properties": {
      "client_ip": {
        "type": "ip"
      },
      "geoip": {
        "dynamic": true,
        "type": "object",
        "properties": {
          "location": {
            "type": "geo_point"
          }
        }
      }
    }
  },

其中生效的是geoip的那一段，mappings的properties中添加了geoip这个字段，并且使用dynamic，允许Logstash的geoip插件将解析后的详细字段也保存到ES索引中。geoip插件解析出来会带有一个location字段，这个字段就是经纬度的坐标点，所以重点是这里要设置geoip.location字段的类型是geo_point。Geo-point表示为一个object，具有lat和lon两个key。改完可以GET再去看一下生效没有。

我这里按照自己的情况新建了一个优先度为10且一定匹配得上的模板，例：

PUT /_template/logstash-upixel-h5
{
  "index_patterns" : [
      "logstash-upixel-h5*"
  ],
  "order" : 10,
  "mappings": {
    "properties": {
      "client_ip": {
        "type": "ip"
      },
      "geoip": {
        "dynamic": true,
        "type": "object",
        "properties": {
          "location": {
            "type": "geo_point"
          }
        }
      }
    }
  },
  "aliases" : {
      "logstash-upixel" : { }
  }
}

4.2.2、nginx修改日志格式为json

user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

#    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
 #                     '$status $body_bytes_sent "$http_referer" '
  #                    '"$http_user_agent" "$http_x_forwarded_for"';
     log_format main   '{"@timestamp":"$time_iso8601",'
                    '"@source":"$server_addr",'
                    '"hostname":"$hostname",'
                    '"remote_user":"$remote_user",'
                    '"ip":"$http_x_forwarded_for",'
                    '"client":"$remote_addr",'
                    '"request_method":"$request_method",'
                    '"scheme":"$scheme",'
                    '"domain":"$server_name",'
                    '"referer":"$http_referer",'
                    '"request":"$request_uri",'
                    '"requesturl":"$request",'
                    '"args":"$args",'
                    '"size":$body_bytes_sent,'
                    '"status": $status,'
                    '"responsetime":$request_time,'
                    '"upstreamtime":"$upstream_response_time",'
                    '"upstreamaddr":"$upstream_addr",'
                    '"http_user_agent":"$http_user_agent",'
                    '"http_cookie":"$http_cookie",'
                    '"https":"$https"'
                    '}';


   # log_format main_json '{"domain":"$server_name",'
   # '"remote_addr":"$remote_addr",'
   # '"time_local":"$time_iso8601",'
   # '"request":"$request",'
    #'"request_body":"$request_body",'
    #'"status":$status,'
    #'"body_bytes_sent":"$body_bytes_sent",'
    #'"http_referer":"$http_referer",'
   # '"upstream_response_time":"$upstream_response_time",'
  #  '"request_time":"$request_time",'
    #'"http_user_agent":"$http_user_agent",'
    #'"upstream_addr":"$upstream_addr",'
   # '"upstream_status":"$upstream_status"}';
  #  '"@source":"$server_addr",';


#    access_log  /var/log/nginx/access.log  main_json;
    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

4.2.3、Logstash配置Geoip，通过IP地址解析地理位置

logstash 7.x 已经自带geoip插件，不需要额外安装

input {
    beats {
        codec => plain{charset => "UTF-8"} #设置编解码器为utf8
        port => "5044"    
    }
   
}

filter {
  if [filetype] == "upixel-h5" {
   grok {
          match => {
           "message" => '^{"@timestamp":"(?<log.timestamp>.+)","@source":"(?<log.@source>.+)","hostname":"(?<log.hostname>.+)","remote_user":"(?<log.remote_user>.+)","ip":"(?<log.ip>.+)","client":"(?<log.client>.+)","request_method":"(?<log.request_method>.+)","scheme":"(?<log.scheme>.+)","domain":"(?<log.domain>.+)","referer":"(?<log.referer>.+)","request":"(?<log.request>.+)","requesturl":"(?<log.requesturl>.+)","args":"(?<log.args>.+)","size":(?<log.size>.+),"status":(?<log.status>.+),"responsetime":(?<log.responsetime>.+),"upstreamaddr":(?<log.upstreamaddr>.+),"http_user_agent":(?<log.http_user_agent>.+)'
  }
 }
        geoip {
            source => "log.ip"  ##日志格式里的ip来源，这里是解析后的log.ip这个字段（client":"$remote_addr"）
            target => "geoip"
            database =>"/data/logstash-7.3.0/GeoLite2/GeoLite2-City.mmdb"   ##### 下载GeoIP库
            add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
            add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
          }
         mutate {
            convert => [ "[geoip][coordinates]", "float"]
  }
}
 else {   
      multiline {
           pattern => "^["       # 正则匹配以[开头的
           negate => true         # true:表示不匹配正则表达式时，false:匹配正则表达式时（negate相反的）
           what => "previous"     # 设置未匹配的内容是向前合并还是先后合并,previous向前合并，next向后合并
       }

       grok {
          match => {
           "message" => '[(?<log.level_name>[a-zA-Z]+)] [(?<log.create_time>.*?)] [(?<log.logger_name>.*?)] [(?<log.process_info>.*?)] [(?<log.thread_info>.*?)] [(?<log.pathname>.*?)] [(?<log.path_module_info>.*?)] [(?<log.extend_info.server_name>.*?)] [(?<log.extend_info.server_module_name>.*?)] [(?<log.extend_info.server_module_function_name>.*?)] [(?<log.extend_info.trace_id>.*?)] [(?<log.extend_info.trace_id_index>.*?)] [(?<log.extend_info.request_type>.*?)] [(?<log.extend_info.request_method>.*?)] [(?<log.extend_info.request_token>.*?)] [(?<log.extend_info.request_url>.*?)] [(?<log.extend_info.request_body>.*?)] [(?<log.extend_info.request_param>.*?)] [(?<log.extend_info.response_code>.*?)] [(?<log.extend_info.response_data>.*?)] [(?<log.extend_info.version>.*?)] [(?<log.extend_info.time_consuming>.*?)] [(?<log.extend_info.error_message>.*)'}

  }

}
}

4.2.4、配置Kibana大图

查看解析后的日志格式

{
  "_index": "logstash-upixel-h52022.08.10",
  "_type": "_doc",
  "_id": "aTHghoIBee1KWV38CcXD",
  "_version": 1,
  "_score": null,
  "_source": {
    "ecs": {
      "version": "1.0.1"
    },
    "log.upstreamaddr": ""-"",
    "log.args": "-",
    "log.responsetime": "0.000,"upstreamtime":"-"",
    "log": {
      "offset": 4775122,
      "file": {
        "path": "/data/ks/upixel-upixel-h5-pvc-968db287-bbed-484d-b19c-4db6b97ca1b1/access.log"
      }
    },
    "log.remote_user": "-",
    "log.http_user_agent": ""Mozilla/5.0 (iPhone; CPU iPhone OS 14_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1","http_cookie":"-","https":""}",
    "log.client": "10.233.70.17",
    "log.timestamp": "2022-08-10T08:28:10+00:00",
    "log.domain": "0.0.0.0",
    "log.scheme": "http",
    "host": {
      "architecture": "x86_64",
      "containerized": false,
      "hostname": "slave1",
      "name": "slave1",
      "id": "97f4033a009562e7d55c14c5622eec4e",
      "os": {
        "family": "debian",
        "name": "Ubuntu",
        "platform": "ubuntu",
        "kernel": "4.4.0-186-generic",
        "version": "16.04.7 LTS (Xenial Xerus)",
        "codename": "xenial"
      }
    },
    "log.@source": "10.233.70.97",
    "@version": "1",
    "agent": {
      "type": "filebeat",
      "id": "7baff6ed-6724-4d95-9625-86a259887e99",
      "hostname": "slave1",
      "version": "7.3.0",
      "ephemeral_id": "f6c59957-6a77-49a5-aee8-dbcb84265ed3"
    },
    "log.ip": "124.64.17.19",
    "log.referer": "http://register.unionstrongtech.com/",
    "@timestamp": "2022-08-10T08:28:37.215Z",
    "log.size": "1440",
    "message": "{"@timestamp":"2022-08-10T08:28:10+00:00","@source":"10.233.70.97","hostname":"nginx-h5-6465749bdb-66bcm","remote_user":"-","ip":"124.64.17.19","client":"10.233.70.17","request_method":"GET","scheme":"http","domain":"0.0.0.0","referer":"http://register.unionstrongtech.com/","request":"/runtime.06daa30a2963fa413676.js","requesturl":"GET /runtime.06daa30a2963fa413676.js HTTP/1.1","args":"-","size":1440,"status": 200,"responsetime":0.000,"upstreamtime":"-","upstreamaddr":"-","http_user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 14_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1","http_cookie":"-","https":""}",
    "log.status": " 200",
    "log.hostname": "nginx-h5-6465749bdb-66bcm",
    "log.requesturl": "GET /runtime.06daa30a2963fa413676.js HTTP/1.1",
    "log.request_method": "GET",
    "log.request": "/runtime.06daa30a2963fa413676.js",
    "geoip": {
      "country_code3": "CN",
      "coordinates": [
        116.3861,
        39.9143
      ],
      "latitude": 39.9143,
      "ip": "124.64.17.19",
      "country_code2": "CN",
      "country_name": "China",
      "region_code": "BJ",
      "timezone": "Asia/Shanghai",
      "longitude": 116.3861,
      "region_name": "Beijing",
      "location": {
        "lat": 39.9143,
        "lon": 116.3861
      },
      "continent_code": "AS",
      "city_name": "Beijing"
    },
    "tags": [
      "beats_input_codec_plain_applied"
    ],
    "input": {
      "type": "log"
    },
    "filetype": "upixel-h5"
  },
  "fields": {
    "log.timestamp": [
      "2022-08-10T08:28:10.000Z"
    ],
    "@timestamp": [
      "2022-08-10T08:28:37.215Z"
    ]
  },
  "sort": [
    1660120117215
  ]
}

注意：默认使用logstash索引模板（匹配logstash-*），如果索引名称为logstash-*，则直接接入Elasticsearch即可，否则需要添加对应索引模板 geoip”: { “country_code3”: “CN”, “coordinates”: [
116.3861, 39.9143 ], “latitude”: 39.9143, “ip”: “124.64.17.19”, “country_code2”: “CN”, “country_name”: “China”, “region_code”: “BJ”, “timezone”: “Asia/Shanghai”, “longitude”: 116.3861, “region_name”: “Beijing”, #地区 “location”: { “lat”: 39.9143, “lon”: 116.3861 #经纬度 编辑坐标地图

深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘
深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘

整体大盘

深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘
深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘
深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘

深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘


原文始发于微信公众号（背带裤的云原生）：深度剖析 ELK：实战经验分享与实用策略之实现 IP 地址分布地图可视化大盘

版权声明：本文内容由互联网用户自发贡献，该文观点仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 举报，一经查实，本站将立刻删除。

文章由极客之音整理，本文链接：https://www.bmabk.com/index.php/post/218853.html

赞 (0)
青莲明月的头像青莲明月
0 0

相关推荐

发表回复

登录后才能评论
极客之音——专业性很强的中文编程技术网站，欢迎收藏到浏览器，订阅我们！