ELK – Elasticsearch, Logstash & Kibana 日志搜集 — 实践 一
环境准备：如果是单独的一台服务器内存要 4 G以上
192.168.1.3-ElasticSearch-2G
192.168.1.4-Kibana-WEB-1G
192.168.1.5-Logstash-Client-2G
# 临时关闭三台服务器的 selinux、防火墙
# 192.168.1.3-ElasticSearch-2G
[root@1-3 ~]# ll elasticsearch*
-rw-r--r-- 1 root root 33696963 3月 4 11:53 elasticsearch-5.3.0.tar.gz
[root@1-3 ~]# ll jdk*
-rw-r--r-- 1 root root 185526495 3月 4 11:55 jdk1.8.0_131.tar.gz
[root@1-3 ~]# yum -y install ntpdate
[root@1-3 ~]# ntpdate pool.ntp.org
# 192.168.1.4-Kibana-WEB-1G
[root@1-4 ~]# ll kibana*
-rw-r--r-- 1 root root 38912478 3月 4 11:53 kibana-5.3.0-linux-x86_64.tar.gz
[root@1-4 ~]# yum -y install ntpdate
[root@1-4 ~]# ntpdate pool.ntp.org
# 192.168.1.5-Logstash-Client-2G
[root@1-5 ~]# ll logstash*
-rw-r--r-- 1 root root 94087323 3月 4 11:54 logstash-5.3.0.tar.gz
[root@1-5 ~]# ll jdk*
-rw-r--r-- 1 root root 185526495 3月 4 11:55 jdk1.8.0_131.tar.gz
[root@1-5 ~]# yum -y install ntpdate
[root@1-5 ~]# ntpdate pool.ntp.org
####################################################################
192.168.1.3-ElasticSearch-2G
# 配置 java 环境变量
[root@1-3 ~]# yum -y install wget
[root@1-3 ~]# yum -y install lrzsz
[root@1-3 ~]# rz -y # 上传jdk
[root@1-3 ~]# tar -zxf jdk1.8.0_131.tar.gz
[root@1-3 ~]# mkdir -p /usr/java/
[root@1-3 ~]# mv jdk1.8.0_131/ /usr/java/
[root@1-3 ~]# cd /usr/java/
[root@1-3 java]# vim /etc/profile # 在系统配置文件中加入 java 的环境变量配置
... # 这里是省略前面代码的意思不要写
export JAVA_HOME=/usr/java/jdk1.8.0_131
export CLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH:$HOMER/bin
[root@1-3 java]# source /etc/profile # 使环境变量生效
[root@1-3 java]# java -version
java version "1.8.0_131"
Java(TM) SE Runtime Environment (build 1.8.0_131-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode)
Elasticsearch 安装
[root@1-3 java]# cd
[root@1-3 ~]# tar zxf elasticsearch-5.3.0.tar.gz
[root@1-3 ~]# mv elasticsearch-5.3.0 /usr/local/
[root@1-3 ~]# cd /usr/local/
[root@1-3 local]# mv elasticsearch-5.3.0 elasticsearch
[root@1-3 local]# cd elasticsearch/
[root@1-3 elasticsearch]# cd config/
[root@1-3 config]# vim jvm.options
...
# Xmx represents the maximum size of total heap space
-Xms1g # 修改为1g，Xms Xmx 最大最小内存，最大最小堆栈，初始化的时候需要占用的内存，直接从物理内存中划出1g，来给jvm使用
-Xmx1g
...
[root@1-3 config]# vim elasticsearch.yml # 主配置文件
...
# Set the bind address to a specific IP (IPv4 or IPv6):
#
network.host: 0.0.0.0
...
[root@1-3 elasticsearch]# useradd elk
[root@1-3 elasticsearch]# chown -R elk /usr/local/elasticsearch/
[root@1-3 elasticsearch]# chmod 1777 /tmp/
[root@1-3 config]# su elk
[elk@1-3 config]$ /usr/local/elasticsearch/bin/elasticsearch -d # 启动 elasticsearch
[elk@1-3 config]$ tail -fn 10 ../logs/elasticsearch.log # 查看 elasticsearch 主日志；启动 elasticsearch elasticsearch 启动报错，进行修改
...
[2018-03-09T15:00:58,085][ERROR][o.e.b.Bootstrap ] [unnwKH1] node validation exception
bootstrap checks failed
max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]
max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
...
[elk@1-3 config]$ vim elasticsearch.yml
...
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
...
[elk@1-3 config]$ exit
exit
[root@1-3 config]# vim /etc/security/limits.conf
...
#* soft core 0
* soft nofile 65536
* hard nofile 65536
#* hard rss 10000
...
[root@1-3 config]# cd /etc/security/limits.d/
[root@1-3 limits.d]# vim 20-nproc.conf
# Default limit for number of user's processes to prevent
# accidental fork bombs.
See rhbz #432903 for reasoning.
▽
* soft nproc 4096
root soft nproc unlimited
soft nproc 2048
[root@1-3 limits.d]# vim /etc/sysctl.conf
...
# For more information, see sysctl.conf(5) and sysctl.d(5).
vm.max_map_count = 655360
[root@1-3 limits.d]# sysctl -p
vm.max_map_count = 655360
[root@1-3 limits.d]# exit
登出
Last login: Fri Mar 9 13:51:33 2018 from 192.168.1.1
[root@1-3 ~]# pkill java # 如果java启动，需要杀掉
[root@1-3 ~]# su elk
[elk@1-3 root]$ /usr/local/elasticsearch/bin/elasticsearch -d # 启动 elasticsearch
[elk@1-3 root]$ ps -ef | grep java # 查看 java 是否启动
elk 8663 1 0 10:03 pts/0 00:00:21 /usr/java/jdk1.8.0_131/bin/java -Xms1g -Xmx1g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -XX:+AlwaysPreTouch -server -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -Djdk.io.permissionsUseCanonicalPath=true -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j.skipJansi=true -XX:+HeapDumpOnOutOfMemoryError -Des.path.home=/usr/local/elasticsearch -cp /usr/local/elasticsearch/lib/elasticsearch-5.3.0.jar:/usr/local/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch -d
elk 9380 9247 0 12:22 pts/2 00:00:00 grep --color=auto java
[elk@1-3 root]$ tail -fn 20 /usr/local/elasticsearch/logs/elasticsearch.log # 查看 elasticsearch 主日志；没有报错
[elk@1-3 root]$ netstat -ntlp | grep -E "9200|9300" # 9200 9300 监听中代表elasticsearch 启动正常
# 9200 数据搜索，数据存储，数据分析
# 9300 集群通讯端口，和其他 elasticsearch 节点进行通讯的
(Not all processes could be identified, non-owned process info
will not be shown, you would have to be root to see it all.)
tcp6 0 0 :::9200 :::* LISTEN 8663/java
tcp6 0 0 :::9300 :::* LISTEN 8663/java Elasticsearch HEAD 插件安装
参考链接：
http://blog.csdn.net/laotoumo/article/details/53890279
https://www.cnblogs.com/tielemao/p/8479065.html
[elk@1-3 root]$ exit
exit
[root@1-3 ~]# yum -y install git
[root@1-3 ~]# wget https://nodejs.org/dist/v6.9.2/node-v6.9.2-linux-x64.tar.xz
[root@1-3 ~]# xz -d node-v6.9.2-linux-x64.tar.xz
[root@1-3 ~]# tar -xvf node-v6.9.2-linux-x64.tar
[root@1-3 ~]# mkdir -p /alidata/app/node
[root@1-3 ~]# mv node-v6.9.2-linux-x64/* /alidata/app/node
[root@1-3 ~]# vim /etc/profile
...
xport PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH:$HOMER/bin
export NODE_HOME=/alidata/app/node
export PATH=/$PATH:$NODE_HOME/bin
[root@1-3 ~]# source /etc/profile
[root@1-3 ~]# node -v
v6.9.2
[root@1-3 ~]# npm -v
3.10.9
[root@1-3 ~]# git clone https://github.com/mobz/elasticsearch-head.git
正克隆到 'elasticsearch-head'...
remote: Counting objects: 4224, done.
remote: Total 4224 (delta 0), reused 0 (delta 0), pack-reused 4224
接收对象中: 100% (4224/4224), 2.16 MiB | 8.00 KiB/s, done.
处理 delta 中: 100% (2329/2329), done.
[root@1-3 ~]# cd elasticsearch-head/
[root@1-3 elasticsearch-head]# npm install
npm WARN deprecated coffee-script@1.10.0: CoffeeScript on NPM has moved to "coffeescript" (no hyphen)
npm WARN deprecated http2@3.3.7: Use the built-in module in node 9.0.0 or newer, instead
npm WARN prefer global coffee-script@1.10.0 should be installed with -g
> phantomjs-prebuilt@2.1.16 install /root/elasticsearch-head/node_modules/phantomjs-prebuilt
> node install.js
PhantomJS not found on PATH
Downloading https://github.com/Medium/phantomjs/releases/download/v2.1.1/phantomjs-2.1.1-linux-x86_64.tar.bz2
Saving to /tmp/phantomjs/phantomjs-2.1.1-linux-x86_64.tar.bz2
Receiving...
Error making request.
Error: connect ECONNREFUSED 54.231.32.107:443
at Object.exports._errnoException (util.js:1022:11)
at exports._exceptionWithHostPort (util.js:1045:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1087:14)
Please report this full log at https://github.com/Medium/phantomjs
elasticsearch-head@0.0.0 /root/elasticsearch-head
# phantomjs 安装出错 这个可以忽略
[root@1-3 elasticsearch-head]# cd /usr/local/elasticsearch/config/
[root@1-3 config]# vim elasticsearch.yml
...
#action.destructive_requires_name: true
http.cors.enabled: true
http.cors.allow-origin: "*"
[root@1-3 config]# cd /root/elasticsearch-head/
[root@1-3 elasticsearch-head]# vim Gruntfile.js
...
options: {
hostname: "0.0.0.0",
port: 9100,
...
[root@1-3 elasticsearch-head]# vim _site/app.js
...
this.base_uri = this.config.base_uri || this.prefs.get("app-base_uri") || "http://192.168
.1.3:9200";
...
[root@1-3 elasticsearch-head]# nohup ./node_modules/grunt/bin/grunt server &
[1] 12502
[root@1-3 elasticsearch-head]# nohup: 忽略输入并把输出追加到"nohup.out" # 回车退出
[root@1-3 elasticsearch-head]# tail -fn 100 nohup.out
>> Local Npm module "grunt-contrib-jasmine" not found. Is it installed? # 缺少一个插件
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
[root@1-3 elasticsearch-head]# pkill java
[root@1-3 elasticsearch-head]# npm install grunt-contrib-jasmine --registry=https://registry.npm.taobao.org # 安装该插件
[root@1-3 elasticsearch-head]# ps -ef | grep 9100
root 12580 2146 0 11:26 pts/2 00:00:00 grep --color=auto 9100
[root@1-3 elasticsearch-head]# netstat -ntlp | grep 9100
tcp 0 0 0.0.0.0:9100 0.0.0.0:* LISTEN 12502/grunt
[root@1-3 elasticsearch-head]# kill -9 12502
[root@1-3 elasticsearch-head]# nohup ./node_modules/grunt/bin/grunt server & # 回车退出
[1] 12659
[root@1-3 elasticsearch-head]# nohup: 忽略输入并把输出追加到"nohup.out"
[root@1-3 elasticsearch-head]# tail -fn 100 nohup.out
>> Local Npm module "grunt-contrib-jasmine" not found. Is it installed?
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
>> Local Npm module "grunt-contrib-jasmine" not found. Is it installed?
Running "connect:server" (connect) task
Waiting forever...
Fatal error: Port 9100 is already in use by another process.
>> Local Npm module "grunt-contrib-jasmine" not found. Is it installed?
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100 # 启动正常# 浏览器访问效果
Kibana安装配置
[root@1-4 ~]# tar zxf kibana-5.3.0-linux-x86_64.tar.gz
[root@1-4 ~]# mv kibana-5.3.0-linux-x86_64 /usr/local/
[root@1-4 ~]# cd /usr/local/
[root@1-4 local]# mv kibana-5.3.0-linux-x86_64 kibana
[root@1-4 local]# cd kibana/
[root@1-4 kibana]# cd config/
[root@1-4 config]# vim kibana.yml
# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601
# Specifies the address to which the Kibana server will bind. IP addresses ...
server.host: "0.0.0.0"
...
elasticsearch.url: "http://192.168.1.3:9200"
...
[root@1-4 config]# cd ../bin/
[root@1-4 bin]# ll kibana
-rwxrwxr-x 1 1000 1000 612 3月 23 2017 kibana
[root@1-4 bin]# nohup sh kibana &
[1] 4102
[root@1-4 bin]# nohup: 忽略输入并把输出追加到"nohup.out"
# 回车
[root@1-4 bin]# tail -fn 100 nohup.out # 查看日志
[root@1-4 bin]# netstat -ntlp | grep 5601 # 查看 kibana 启动进程
tcp 0 0 0.0.0.0:5601 0.0.0.0:* LISTEN 4102/./../node/bin/
[root@1-4 bin]# kill -9 4102 # 杀掉 kibana 进程，当前不需要操作该命令
# 在 Logstash log 中输入 helloworld 此处便可以进行操作
Logstash 安装配置
[root@1-5 ~]# tar zxf logstash-5.3.0.tar.gz
[root@1-5 ~]# mv logstash-5.3.0 /usr/local/
[root@1-5 ~]# cd /usr/local/
[root@1-5 local]# mv logstash-5.3.0 logstash
[root@1-5 local]# cd logstash/
[root@1-5 logstash]# cd config/
[root@1-5 config]# pwd
/usr/local/logstash/config
[root@1-5 config]# mkdir etc/
[root@1-5 config]# cd etc/
[root@1-5 etc]# vim logstash.conf
input { # 负责读取数据，读取日志的模块
stdin { } # 大括号里面的可以输入路径，stain 标准输入(键盘写入)
}
output { # 负责输出数据
stdout { # 标准输出
codec => rubydebug {} # 输出的格式；codec 编码格式 ruby(一门编程语言) 有时间记录，有索引记录，有信息记录
}
elasticsearch { # 输出的日志，追加到 hosts 主机
hosts => "192.168.1.3:9200" # 这里只能写 elasticsearch 的主机ip地址
}
}
[root@1-5 config]# vim jvm.options
## JVM configuration
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms1g
-Xmx1g
[root@1-5 etc]# /usr/local/logstash/bin/logstash -f logstash.conf # 要在前台启动，因为需要和键盘交互
# 输入 helloworld
Chinese