elk搭建
访问量:1740
创建时间:2020-08-06
简介本文章主要目的：通过elk分析nginx日志，给grafana提供大屏数据。
| 机器 | 角色 |
|---|---|
| 192.168.1.76 | 安装es,kibana,logstash |
| nginx服务器 | 安装 nginx ,filebeat |
下载
下载地址：https://www.elastic.co/cn/downloads/ 并且准备jdk
[root@admin ~]# ll
-rw-r--r--. 1 root root 318401743 8月 6 15:39 elasticsearch-7.8.1-x86_64.rpm
-rw-r--r--. 1 root root 28557354 8月 6 15:39 filebeat-7.8.1-x86_64.rpm
-rw-r--r--. 1 root root 194151339 8月 6 15:39 jdk-8u231-linux-x64.tar.gz
-rw-r--r--. 1 root root 61604633 8月 6 15:39 kafka_2.12-2.5.0.tgz
-rw-r--r--. 1 root root 347916016 8月 6 15:40 kibana-7.8.1-x86_64.rpm
-rw-r--r--. 1 root root 159752536 8月 6 15:40 logstash-7.8.1.rpm
#安装jdk
[root@admin ~]# tar xf jdk-8u231-linux-x64.tar.gz
[root@admin ~]# mv jdk1.8.0_231/ /usr/lib/
[root@admin ~]# vim /etc/profile
#新增加
export JAVA_HOME=/usr/lib/jdk1.8.0_231
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin:${PATH}
[root@admin ~]# source /etc/profile
部署单节点elasticsearch-7.8
[root@admin ~]# rpm -ivh elasticsearch-7.8.1-x86_64.rpm
警告：elasticsearch-7.8.1-x86_64.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中... ################################# [100%]
Creating elasticsearch group... OK
Creating elasticsearch user... OK
正在升级/安装...
1:elasticsearch-0:7.8.1-1 ################################# [100%]
### NOT starting on installation, please execute the following statements to configure elasticsearch service to start automatically using systemd
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service
### You can start elasticsearch service by executing
sudo systemctl start elasticsearch.service
future versions of Elasticsearch will require Java 11; your Java version from [/usr/lib/jdk1.8.0_231/jre] does not meet this requirement
Created elasticsearch keystore in /etc/elasticsearch/elasticsearch.keystore
[root@admin ~]# systemctl daemon-reload
[root@admin ~]# systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@admin ~]# cd /data/
[root@admin data]# mkdir elasticsearch/log -pv
mkdir: 已创建目录 "elasticsearch"
mkdir: 已创建目录 "elasticsearch/log"
[root@admin data]# mkdir elasticsearch/data -pv
mkdir: 已创建目录 "elasticsearch/data"
[root@admin data]# chown -R elasticsearch.elasticsearch elasticsearch
[root@admin data]# vim /etc/elasticsearch/elasticsearch.yml
node.name: node-1
path.data: /data/elasticsearch/data
path.logs: /data/elasticsearch/log
network.host: 0.0.0.0
discovery.seed_hosts: ["192.168.1.76"]
cluster.initial_master_nodes: ["node-1"]
http.cors.enabled: true
http.cors.allow-origin: "*"
#系统简单优化
[root@admin log]# vim /etc/sysctl.conf
vm.max_map_count = 655360
[root@admin log]# sysctl -p
vm.max_map_count = 655360
[root@admin log]# cat >> /etc/security/limits.conf <<EOF
> elasticsearch soft nofile 65536
> elasticsearch hard nofile 65536
> elasticsearch soft nproc 4096
> elasticsearch hard nproc 4096
> EOF
[root@admin log]# systemctl start elasticsearch.service
[root@admin log]# curl http://192.168.1.76:9200
{
"name" : "admin",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "_na_",
"version" : {
"number" : "7.8.1",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "b5ca9c58fb664ca8bf9e4057fc229b3396bf3a89",
"build_date" : "2020-07-21T16:40:44.668009Z",
"build_snapshot" : false,
"lucene_version" : "8.5.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}
#ES的环境变量配置文件/etc/sysconfig/elasticsearch
kibana安装
[root@admin ~]# rpm -ivh kibana-7.8.1-x86_64.rpm
[root@admin ~]# vim /etc/kibana/kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://192.168.1.76:9200"]
elasticsearch.requestTimeout: 90000
i18n.locale: "zh-CN"
[root@admin ~]# systemctl daemon-reload
[root@admin ~]# systemctl restart kibana
[root@admin ~]# systemctl status kibana
logstash 安装
[root@admin ~]# rpm -ivh logstash-7.8.1.rpm
警告：logstash-7.8.1.rpm: 头V4 RSA/SHA512 Signature, 密钥 ID d88e42b4: NOKEY
准备中... ################################# [100%]
正在升级/安装...
1:logstash-1:7.8.1-1 ################################# [100%]
Using provided startup.options file: /etc/logstash/startup.options
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/pleaserun-0.0.31/lib/pleaserun/platform/base.rb:112: warning: constant ::Fixnum is deprecated
Successfully created system startup script for Logstash
#修改环境变量
[root@admin ~]# vim /etc/profile
export PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin:${PATH}:/usr/share/logstash/bin
[root@admin ~]# source /etc/profile
[root@admin conf.d]# mkdir /data/logstash/data
[root@admin conf.d]# mkdir /data/logstash/log
[root@admin ~]# chown -R logstash.logstash /data/logstash/
[root@admin ~]# vim /etc/logstash/logstash.yml
path.data: /data/logstash/data
path.logs: /data/logstash/log
path.config: /etc/logstash/conf.d/*.conf
config.reload.interval: 10s
[root@admin ~]# vim /etc/logstash/conf.d/nginx.conf
input {
beats {
port => "5044"
}
}
filter {
if [fileset][name] == "access" {
json {
source => "message"
remove_field => "message"
remove_field => "@timestamp"
}
date {
match => ["time_local", "ISO8601"]
target => "@timestamp"
}
grok {
match => { "request" => "%{WORD:method} (?<url>.* )" }
}
mutate {
remove_field => ["host","event","input","request","offset","prospector","source","type","tags","beat"]
rename => {"http_user_agent" => "agent"}
rename => {"upstream_response_time" => "response_time"}
rename => {"http_x_forwarded_for" => "x_forwarded_for"}
#split => {"x_forwarded_for" => ", "}
#split => {"response_time" => ", "}
}
geoip {
source => "real_ip"
}
}
if [fileset][name] == "error" {
mutate {
remove_field => ["@timestamp"]
}
grok {
match => {"message" => "(?<datetime>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<real_ip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:domain}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"}
}
date {
match => ["datetime", "yyyy/MM/dd HH:mm:ss"]
target => "@timestamp"
}
mutate {
remove_field => ["message","request","http_referer","host","event","input","offset","prospector","source","type","tags","beat"]
}
}
}
#output {
# stdout {
# codec => "rubydebug"
# }
#}
output {
elasticsearch {
hosts => ["192.168.1.76"]
index => "logstash-nginx-%{+YYYY.MM.dd}"
}
}
[root@admin ~]# systemctl start logstash
filebeat 安装
在nginx机器上安装filebeat
[root@Nginx ~]# rpm -ivh filebeat-7.8.1-x86_64.rpm
warning: filebeat-7.8.1-x86_64.rpm: Header V4 RSA/SHA512 Signature, key ID d88e42b4: NOKEY
Preparing... ################################# [100%]
Updating / installing...
1:filebeat-7.8.1-1 ################################# [100%]
[root@Nginx ~]# cd /etc/filebeat/
[root@Nginx filebeat]# filebeat modules enable nginx
Enabled nginx
[root@Nginx filebeat]# vim /etc/filebeat/filebeat.yml
#output.elasticsearch: 注释掉
# Array of hosts to connect to.
#hosts: ["localhost:9200"] 注释掉
#开启以下内容
output.logstash:
# The Logstash hosts
hosts: ["192.168.1.76:5044"]
#配置filebeat监控的日志文件(可以使用正则表达式匹配多个文件)
[root@Nginx filebeat]# vim modules.d/nginx.yml
- module: nginx
access:
enabled: true
var.paths: ["/var/log/nginx/access.log"]
error:
enabled: true
var.paths: ["/var/log/nginx/error.log"]
[root@Nginx filebeat]# systemctl start filebeat
配置nginx日志
配置nginx log
[root@Nginx filebeat]# vim /etc/nginx/nginx.conf
log_format main_json '{"domain":"$server_name",'
'"http_x_forwarded_for":"$http_x_forwarded_for",'
'"time_local":"$time_iso8601",'
'"request":"$request",'
'"request_body":"$request_body",'
'"status":$status,'
'"body_bytes_sent":"$body_bytes_sent",'
'"http_referer":"$http_referer",'
'"upstream_response_time":"$upstream_response_time",'
'"request_time":"$request_time",'
'"http_user_agent":"$http_user_agent",'
'"upstream_addr":"$upstream_addr",'
'"upstream_status":"$upstream_status"}';
access_log /var/log/nginx/access.log main_json;
[root@Nginx filebeat]# nginx -s reload
登陆评论:
使用GITHUB登陆