ELK – Elasticsearch, Logstash & Kibana 日志搜集 — 实践 二 搜集 Nginx 日志(Redis缓存)

ELK – Elasticsearch, Logstash & Kibana 日志搜集 — 实践 一

http://www.01wneo.xin/?p=1030

192.168.1.5-Logstash-Client-2G 安装 nginx

[root@1-5 ~]# wget -c https://mirrors.ustc.edu.cn/epel/7/x86_64/Packages/e/epel-release-7-11.noarch.rpm # 下载 nginx 镜像源
[root@1-5 ~]# rpm -ivh epel-release-7-11.noarch.rpm # 安装 nginx 镜像源
[root@1-5 ~]# yum install nginx -y # 安装 nginx
[root@1-5 ~]# /usr/sbin/nginx # 启动 nginx
[root@1-5 ~]# tail -fn 100 /var/log/nginx/access.log # nginx 访问日志目录
# 搜集 nginx 日志
[root@1-5 logstash]# cd /usr/local/logstash/config/etc/
[root@1-5 etc]# vim nginx.conf
input {
    file {
        type => "nginx-access"
        path => "/var/log/nginx/access.log"
    }
}

output {
    elasticsearch {
        hosts => "192.168.1.3"
    }
}
[root@1-5 etc]# ../../bin/logstash -f nginx.conf

192.168.1.4-Kibana-WEB-1G 对 1.5 进行压测

[root@1-4 bin]# yum -y install httpd-tools # 安装 ab 压测工具
[root@1-4 bin]# while sleep 10;do ab -c 1 -n 1 http://192.168.1.5/;done # 10 秒压测一次

192.168.1.4-Kibana-WEB-1G 配置 Redis 缓存

[root@1-4 ~]# yum install gcc* -y
[root@1-4 ~]# wget http://download.redis.io/releases/redis-4.0.1.tar.gz
[root@1-4 ~]# tar xzf redis-4.0.1.tar.gz
[root@1-4 ~]# cd redis-4.0.1
[root@1-4 redis-4.0.1]# make
[root@1-4 redis-4.0.1]# nohup src/redis-server & # 后台启动 Redis,回车保持后台运行
[2] 6205
[root@1-4 redis-4.0.1]# nohup: 忽略输入并把输出追加到"nohup.out"

[2]+  退出 1                nohup src/redis-server
[root@1-4 redis-4.0.1]# netstat -ntlp # 查看 Redis 进程是否保持后台启动
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:6379            0.0.0.0:*               LISTEN      6201/src/redis-serv 
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1506/sshd           
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      1985/master         
tcp        0      0 0.0.0.0:5601            0.0.0.0:*               LISTEN      2640/./../node/bin/ 
tcp6       0      0 :::6379                 :::*                    LISTEN      6201/src/redis-serv 
tcp6       0      0 :::22                   :::*                    LISTEN      1506/sshd           
tcp6       0      0 ::1:25                  :::*                    LISTEN      1985/master         
[root@1-4 redis-4.0.1]# src/redis-cli 
127.0.0.1:6379> keys * # 查看 key
(empty list or set)
127.0.0.1:6379> FLUSHALL # 清空 key
6201:M 12 Mar 18:35:16.969 * DB saved on disk
OK
127.0.0.1:6379>  

192.168.1.5-Logstash-Client-2G  将日志写入 Redis 缓存

[root@1-5 ~]# vim /usr/local/logstash/config/etc/nginx.conf 
input {
    file {
        type => "nginx-access"
        path => "/var/log/nginx/access.log"
    }
}

output {
    redis {
        host => "192.168.1.4"
        port => 6379
        data_type => "list"
        key => "logstash"
    }
}
[root@1-5 ~]# cd /usr/local/logstash/config/etc/
[root@1-5 etc]# mv nginx.conf nginx_input_redis.conf
[root@1-5 etc]# cp nginx_input_redis.conf nginx_redis_output.conf
[root@1-5 etc]# vim nginx_redis_output.conf 
input {
    redis {
        host => "192.168.1.4"
        port => "6379"
        data_type => "list"
        key => "logstash"
        type => "redis-input"
        #batch_count => 1
    }
}

output {
    elasticsearch {
        hosts => "192.168.1.3"
    }
}
[root@1-5 etc]# ll nginx*
-rw-r--r-- 1 root root 237 3月 12 18:53 nginx_input_redis.conf
-rw-r--r-- 1 root root 260 3月 12 19:00 nginx_redis_output.conf
[root@1-5 etc]# nohup ../../bin/logstash -f nginx_input_redis.conf & # 每次启动前,删除之前的 nohup.out
[1] 5891
[root@1-5 etc]# nohup: 忽略输入并把输出追加到"nohup.out"

[root@1-5 etc]# nohup ../../bin/logstash -f nginx_redis_output.conf & # 每次启动前,删除之前的 nohup.out
[2] 5922
[root@1-5 etc]# nohup: 忽略输入并把输出追加到"nohup.out"

[root@1-5 etc]# tail -fn 20 nohup.out # 报错 Redis 连接问题
[2018-03-12T19:18:37,639][WARN ][logstash.inputs.redis ] Redis connection problem {:exception=>#<Redis::CommandError: DENIED Redis is running in protected mode because protected mode is enabled, no bind address was specified, no authentication password is requested to clients. In this mode connections are only accepted from the loopback interface. If you want to connect from external computers to Redis you may adopt one of the following solutions: 1) Just disable protected mode sending the command 'CONFIG SET protected-mode no' from the loopback interface by connecting to Redis from the same host the server is running, however MAKE SURE Redis is not publicly accessible from internet if you do so. Use CONFIG REWRITE to make this change permanent. 2) Alternatively you can just disable the protected mode by editing the Redis configuration file, and setting the protected mode option to 'no', and then restarting the server. 3) If you started the server manually just for testing, restart it with the '--protected-mode no' option. 4) Setup a bind address or an authentication password. NOTE: You only need to do one of the above things in order for the server to start accepting connections from the outside.>}
[root@1-5 etc]# yum install telnet -y # 使用 telnet 连接查看
[root@1-5 etc]# telnet 192.168.1.4 6379 # 出现相同的错误
Trying 192.168.1.4...
Connected to 192.168.1.4.
Escape character is '^]'.
-DENIED Redis is running in protected mode because protected mode is enabled, no bind address was specified, no authentication password is requested to clients. In this mode connections are only accepted from the loopback interface. If you want to connect from external computers to Redis you may adopt one of the following solutions: 1) Just disable protected mode sending the command 'CONFIG SET protected-mode no' from the loopback interface by connecting to Redis from the same host the server is running, however MAKE SURE Redis is not publicly accessible from internet if you do so. Use CONFIG REWRITE to make this change permanent. 2) Alternatively you can just disable the protected mode by editing the Redis configuration file, and setting the protected mode option to 'no', and then restarting the server. 3) If you started the server manually just for testing, restart it with the '--protected-mode no' option. 4) Setup a bind address or an authentication password. NOTE: You only need to do one of the above things in order for the server to start accepting connections from the outside.
Connection closed by foreign host.

修改 192.168.1.4-Kibana-WEB-1G Redis配置

[root@1-4 redis-4.0.1]# vim redis.conf
...
# JUST COMMENT THE FOLLOWING LINE.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
bind 0.0.0.0
...
# are explicitly listed using the "bind" directive.
protected-mode no
...
[root@1-4 redis-4.0.1]# nohup src/redis-server &
[2] 6236
[root@1-4 redis-4.0.1]# nohup: 忽略输入并把输出追加到"nohup.out"

[2]+  退出 1                nohup src/redis-server
[root@1-4 redis-4.0.1]# src/redis-cli
127.0.0.1:6379> config set protected-mode "no"
OK

192.168.1.5-Logstash-Client-2G  telnet 连接正常

[root@1-5 etc]# telnet 192.168.1.4 6379
Trying 192.168.1.4...
Connected to 192.168.1.4.
Escape character is '^]'. # Ctrl D 退出

192.168.1.4-Kibana-WEB-1G 对 1.5 进行压测

[root@1-4 ~]# while sleep 1;do ab -c 1 -n 1 http://192.168.1.5/;done # 1 秒压测一次
# 重新打开一个终端
[root@1-4 ~]# cd /root/redis-4.0.1
[root@1-4 redis-4.0.1]# src/redis-cli
127.0.0.1:6379> keys *
1) "logstash" # 出现包含 logstash 的 key 说明日志从 Redis 中读取的
127.0.0.1:6379> keys *
(empty list or set)

192.168.1.5-Logstash-Client-2G  将日志写入 Redis 缓存

[root@1-5 ~]# cd /usr/local/logstash/config/etc
[root@1-5 etc]# pkill java
[root@1-5 etc]# rm -rf nohup.out
[root@1-5 etc]# nohup ../../bin/logstash -f nginx_input_redis.conf &
[1] 2570
[root@1-5 etc]# nohup: 忽略输入并把输出追加到"nohup.out"

[root@1-5 etc]# nohup ../../bin/logstash -f nginx_redis_output.conf &
[2] 2601
[root@1-5 etc]# nohup: 忽略输入并把输出追加到"nohup.out" 

192.168.1.5-Logstash-Client-2G nginx 日志收集文件整合

[root@1-5 ~]# /usr/local/logstash/config/etc
[root@1-5 etc]# cat nginx_redis_output.conf >>nginx_input_redis.conf
[root@1-5 etc]# mv nginx_input_redis.conf nginx_access.conf
[root@1-5 etc]# vim nginx_access.conf
input {
    file {
        type => "nginx-access"
        path => "/var/log/nginx/access.log"
    }
}

output {
    redis {
        host => "192.168.1.4"
        port => 6379
        data_type => "list"
        key => "logstash"
    }
}

input {
    redis {
        host => "192.168.1.4"
        port => "6379"
        data_type => "list"
        key => "logstash"
        type => "redis-input"
        #batch_count => 1
    }
}

output {
    elasticsearch {
        hosts => "192.168.1.3"
    }
}

 

发表评论

zh_CNChinese
zh_CNChinese