ELK收集Nginx日志 | 易学教程

#由于10.192.27.111 上已经安装了ES 接下来安装Kibana
[root@web01 soft]# rpm -ivh kibana-6.6.0-x86_64.rpm
[root@web01 ~]# rpm -qc kibana  #查看Kibana配置文件
/etc/kibana/kibana.yml  
[root@web01 ~]# grep "^[a-z]" /etc/kibana/kibana.yml #修改后的配置文件
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]
kibana.index: ".kibana"
[root@web01 ~]# 
[root@web01 ~]# systemctl start kibana
[root@web01 ~]# systemctl status kibana
[root@web01 ~]# netstat -lntup|grep 5601
tcp        0      0 0.0.0.0:5601            0.0.0.0:*               LISTEN      69594/node          
[root@web01 ~]#

#测试Nginx负载均衡日志
[root@web01 ~]# ab -n 100 -c 100 http://10.192.27.111:6443/  
[root@web01 ~]# tailf /var/log/nginx/k8s-access.log 
10.192.27.111 10.192.27.114:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.100:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.114:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.100:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.114:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.100:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.114:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.100:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.114:6443 - [07/Dec/2019:11:45:45 +0800] 200 86
10.192.27.111 10.192.27.100:6443 - [07/Dec/2019:11:45:45 +0800] 200 86

#安装filebeat
[root@web01 soft]# rpm -ivh filebeat-6.6.0-x86_64.rpm
警告：filebeat-6.6.0-x86_64.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中...                          ################################# [100%]
正在升级/安装...
   1:filebeat-6.6.0-1                 ################################# [100%]
[root@web01 soft]# rpm -qc filebeat
/etc/filebeat/filebeat.yml
/etc/filebeat/modules.d/apache2.yml.disabled
/etc/filebeat/modules.d/auditd.yml.disabled
/etc/filebeat/modules.d/elasticsearch.yml.disabled
/etc/filebeat/modules.d/haproxy.yml.disabled
/etc/filebeat/modules.d/icinga.yml.disabled
/etc/filebeat/modules.d/iis.yml.disabled
/etc/filebeat/modules.d/kafka.yml.disabled
/etc/filebeat/modules.d/kibana.yml.disabled
/etc/filebeat/modules.d/logstash.yml.disabled
/etc/filebeat/modules.d/mongodb.yml.disabled
/etc/filebeat/modules.d/mysql.yml.disabled
/etc/filebeat/modules.d/nginx.yml.disabled
/etc/filebeat/modules.d/osquery.yml.disabled
/etc/filebeat/modules.d/postgresql.yml.disabled
/etc/filebeat/modules.d/redis.yml.disabled
/etc/filebeat/modules.d/suricata.yml.disabled
/etc/filebeat/modules.d/system.yml.disabled
/etc/filebeat/modules.d/traefik.yml.disabled
[root@web01 soft]# cd
[root@web01 ~]# egrep -v "#|^$" /etc/filebeat/filebeat.yml #修改后的配置文件
filebeat.inputs:
- type: log  #log模式
  enabled: True
  paths:
    - /var/log/nginx/k8s-access.log  #日志目录
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3  #三个副本
setup.kibana:
output.elasticsearch:
  hosts: ["localhost:9200"]  #es地址
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
[root@web01 ~]#

简单的访问一下：http://10.192.27.111:5601/

上面是收集简单的Nginx日志，接下来我们收集json格式的日志

例如：Nginx日志如下

user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format json  '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time"'
' }';

    access_log  /var/log/nginx/access.log  json;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

filebeat删减版日志

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true #关键字顶级
  json.overwrite_keys: true  #

setup.kibana:
  host: "10.192.27.111:5601"

output.elasticsearch:
  hosts: ["10.192.27.111:9200"]
  index: "nginx-%{[beat.version]}-%{+yyyy.MM}"  #定义索引名称
setup.template.name: "nginx"     #定义模板名称
setup.template.pattern: "nginx-*"  #模板正则匹配
setup.template.enabled: false      #不使用系统模板
setup.template.overwrite: true     #覆盖

重启服务然后再次访问步骤更上面一致

详细情况可以参考官网：https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html

来源：https://www.cnblogs.com/linux985/p/12010657.html

标签

nginx

filebeat

log

kibana