Logstash收集Nginx日志

基础环境配置

#安装
rpm -ivh logstash-5.6.5.rpm 

#配置软链
ln -s /etc/logstash /usr/share/logstash/config

#配置
vim /etc/logstash/logstash.yml 
path.config: /usr/share/logstash/config/conf.d     #修改path.config到这个目录


#启动
systemctl start logstash
systemctl status logstash
systemctl enable logstash

#软链启动命令
ln -s /usr/share/logstash/bin/logstash  /sbin  

部署Nginx服务

yum install -y gcc gcc-c++ gcc-g77 flex bison autoconf automake bzip2-devel zlib-devel ncurses-devel libjpeg-devel libpng-devel libtiff-devel freetype-devel pam-devel openssl-devel libxml2-devel gettext-devel pcre-devel

#下载nginx
cd /usr/local/src/ && wget http://nginx.org/download/nginx-1.10.3.tar.gz

tar xvf nginx-1.10.3.tar.gz
cd nginx-1.10.3 && ./configure --prefix=/usr/local/nginx-1.10.3
make && make install
ln -sv /usr/local/nginx-1.10.3 /usr/local/nginx

• 编辑配置文件并准备web界面

cd /usr/local/nginx && vim conf/nginx.conf   #配置改为如下
location /web {
  root html;
  index index.html index.htm;
 }
 
mkdir /usr/local/nginx/html/web
echo " Nginx WebPage! " >/usr/local/nginx/html/web/index.html

/usr/local/nginx/sbin/nginx -t #测试配置文件语法
/usr/local/nginx/sbin/nginx #启动服务
/usr/local/nginx/sbin/nginx -s reload #重读配置文件

#检查端口
[root@localhost nginx]# ss -lntup|grep 80
tcp    LISTEN     0      128       *:80                    *:*                   users:(("nginx",pid=27111,fd=6),("nginx",pid=27108,fd=6))

• 访问nginx测试界面

[root@localhost nginx]# curl http://127.0.0.1/web/
Nginx WebPage! 

将nginx日志转换为json格式

vim conf/nginx.conf   #修改为如下json日志格式

log_format access_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"status":"$status"}';
access_log /var/log/nginx/access.log access_json;

#创建nginx日志目录
mkdir /var/log/nginx

#测试nginx配置文件
/usr/local/nginx/sbin/nginx -t

#重启nginx服务
/usr/local/nginx/sbin/nginx -s reload  

#测试访问:目的生成访问日志
[root@localhost nginx]# curl http://127.0.0.1/web/
Nginx WebPage! 

确认日志格式为json

访问https://www.bejson.com/验证日志是否为标准json
[root@localhost nginx]# tail /var/log/nginx/access.log

{"@timestamp":"2017-12-19T18:32:46+08:00","host":"127.0.0.1","clientip":"127.0.0.1","size":16,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"127.0.0.1","url":"/web/index.html","domain":"127.0.0.1","xff":"-","referer":"-","status":"200"}

配置logstash收集nginx访问日志到前台页面(测试输出)

vim /etc/logstash/conf.d/nginx.conf

input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "end"
    type => "nginx-accesslog"
    codec => json
  }
}

output {
    stdout {
        codec => rubydebug
    }
}

• 检测logstash配置文件语法并重启服务

logstash -f /usr/share/logstash/config/conf.d/nginx.conf -t

• 前台启动logstash

systemctl stop logstash
logstash -f /usr/share/logstash/config/conf.d/nginx.conf
刷新nginx测试页面,等待屏幕输出

如果输出的是json格式说明OK

nginx日志配置输出到elasticsearch

input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "end"
    type => "nginx-accesslog"
    codec => json
  }
}


output {
  if [type] == "nginx-accesslog" {
    elasticsearch {
      hosts => ["10.0.10.21:9200"]
      index => "logstash-nginx-accesslog-1024-%{+YYYY.MM.dd}"
    }}
}

访问elasticsearch-head插件

访问nginx页面:http://10.0.10.24/web/ ，多刷新几次，目的：再次生成访问日志。
访问head插件页面：http://10.0.10.21:9100 查看刚才的nginx索引 

添加到kibana

把索引配置到kibana

• 本文作者： GaryWu
• 本文链接： https://garywu520.github.io/2017/12/19/Logstash收集Nginx日志/
• 版权声明： 本博客所有文章除特别声明外，均采用 MIT 许可协议。转载请注明出处！