高阶使用如何收集多份日志上报logstash应用不同的Filter
4.1.1、配置filebeat
#=========================== Filebeat inputs =============================
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/nginx/access.log
fields:
filetype: canal
fields_under_root: true
#-------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-log-pvc-5f44e8ad-0b10-4b50-80eb-93b5ef161f29/uwsgi.log*
fields:
filetype: upixel-uwsgi
fields_under_root: true
#-------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-celery-logs-pvc-3667b363-ffaa-4c91-af82-72749d1c77de/api.log*
fields:
filetype: upixel-celery-api
fields_under_root: true
#-------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-celery-logs-pvc-3667b363-ffaa-4c91-af82-72749d1c77de/make_report.log*
fields:
filetype: upixel-make_report
fields_under_root: true
#-------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-celery-logs-pvc-3667b363-ffaa-4c91-af82-72749d1c77de/canal_service_celery_sync_es_work.log*
fields:
filetype: upixel-canal-service-celery
fields_under_root: true
#-------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-celery-logs-pvc-3667b363-ffaa-4c91-af82-72749d1c77de/canal_admin_celery_sync_es_work.log*
fields:
filetype: upixel-canal-admin-celery
fields_under_root: true
#-------------------------------------------------------------------
#########################uguard-demo###########################
- type: log
enabled: true
paths:
- /data/ks/uguard-demo-uguard-demo-logs-pvc-9646598b-4c0d-4cdb-9ef0-091e8c1fd41f/uwsgi.log*
fields:
filetype: uguard-demo-uwsgi
fields_under_root: true
#----------------------------mark-------------------------------------
- type: log
enabled: true
paths:
- /data/ks/mark-backend-dev-mark-dev-log-pvc-30e0a9f3-74d2-4e79-8e6c-915c2f3cc98d/*.log
fields:
filetype: mark-api
fields_under_root: true
#----------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/uscreen-pre-check-logs-data-pvc-bb2ba51f-12bb-473a-a4a0-a377962d637f/api.log*
fields:
filetype: pre-check-api
fields_under_root: true
#----------------------------------------------------------------------
#- type: log
# enabled: true
# paths:
# - /data/ks/icad-backend-icad-logs-pvc-9ff4df80-77fa-4340-ab9d-ea0b5b6e6bae/api.log
# fields:
# filetype: icad-api
# fields_under_root: true
#----------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/icad-backend-icad-logs-pvc-9ff4df80-77fa-4340-ab9d-ea0b5b6e6bae/transport.log
fields:
filetype: icad-transport
fields_under_root: true
#----------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-celery-logs-pvc-3667b363-ffaa-4c91-af82-72749d1c77de/celery_worker.log*
fields:
filetype: upixel-celery-worker
fields_under_root: true
#----------------------------------------------------------------------
- type: log
enabled: true
paths:
- /data/ks/upixel-upixel-h5-pvc-968db287-bbed-484d-b19c-4db6b97ca1b1/access.log*
fields:
filetype: upixel-h5
fields_under_root: true
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash: #取消注释，把日志放到logstash中
hosts: ["172.16.103.5:5044"] #取消注释，修改logstash服务器IP和端口
logging.level: info #调整日志级别
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~4.1.2、配置Logstash
input {
beats {
codec => plain{charset => "UTF-8"} #设置编解码器为utf8
port => "5044"
client_inactivity_timeout => 36000
}
}
filter {
if [filetype] == "canal" {
grok {
match => {
"message" => '(?<log.time>d{4}-d{2}-d{2}sd{2}:d{2}:d{2},d{3}) [(?<log.theard_name>.*?)] (?<log.level_name>.*?) (?<log.message>.*)'
}
}
}
else {
multiline {
pattern => "^[" # 正则匹配以[开头的
negate => true # true:表示不匹配正则表达式时，false:匹配正则表达式时（negate相反的）
what => "previous" # 设置未匹配的内容是向前合并还是先后合并,previous向前合并，next向后合并
}
grok {
match => {
"message" => '[(?<log.level_name>[a-zA-Z]+)] [(?<log.create_time>.*?)] [(?<log.logger_name>.*?)] [(?<log.process_info>.*?)] [(?<log.thread_info>.*?)] [(?<log.pathname>.*?)] [(?<log.path_module_info>.*?)] [(?<log.extend_info.server_name>.*?)] [(?<log.extend_info.server_module_name>.*?)] [(?<log.extend_info.server_module_function_name>.*?)] [(?<log.extend_info.trace_id>.*?)] [(?<log.extend_info.trace_id_index>.*?)] [(?<log.extend_info.request_type>.*?)] [(?<log.extend_info.request_method>.*?)] [(?<log.extend_info.request_token>.*?)] [(?<log.extend_info.request_url>.*?)] [(?<log.extend_info.request_body>.*?)] [(?<log.extend_info.request_param>.*?)] [(?<log.extend_info.response_code>.*?)] [(?<log.extend_info.response_data>.*?)] [(?<log.extend_info.version>.*?)] [(?<log.extend_info.time_consuming>.*?)] [(?<log.extend_info.error_message>.*)'}
}
}
}
output {
stdout {
codec => "rubydebug"
}
if [filetype] == "api134" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.34-api-test%{+yyyy.MM.dd}"
}
}
else if [filetype] == "api135" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.35-api-dev%{+yyyy.MM.dd}"
}
}
else if [filetype] == "canal_admin" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.34-canal_admin-test%{+yyyy.MM.dd}"
}
}
else if [filetype] == "canal_service" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.34-canal_service-test%{+yyyy.MM.dd}"
}
}
else if [filetype] == "canal_transport" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.34-canal_transport-test%{+yyyy.MM.dd}"
}
}
else if [filetype] == "admin_celery_es" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.35-admin_celery_es-dev%{+yyyy.MM.dd}"
}
}
else if [filetype] == "service_celery_es" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.35-service_celery_es-dev%{+yyyy.MM.dd}"
}
}
else if [filetype] == "make_report-test" {
elasticsearch {
hosts => "172.16.1.24:9200"
index => "172.16.1.34-make_report-test%{+yyyy.MM.dd}"
}
}
}根据自己的日志格式来匹配规则
原文始发于微信公众号（背带裤的云原生）：深度剖析 ELK：实战经验分享与实用策略之高阶使用logstash
版权声明：本文内容由互联网用户自发贡献，该文观点仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 举报，一经查实，本站将立刻删除。
文章由极客之音整理，本文链接：https://www.bmabk.com/index.php/post/218865.html
