ELK stack 5.x [Docker]ΒΆ
sudo sysctl -w vm.max_map_count=262144
docker-compose up -d
docker-compose.yml
:
version: '2'
services:
filebeat:
image: prima/filebeat:5.1.1
container_name: filebeat
volumes:
- ./logs:/logs
- ./filebeat/data:/data
- ./filebeat/filebeat.yml:/filebeat.yml
logstash:
image: logstash:5.1.1
container_name: logstash
volumes:
- ./logstash.conf:/logstash.conf
- ./geo/GeoLite2-City.mmdb:/etc/logstash/geo_db
command: -f /logstash.conf
elasticsearch:
image: elasticsearch:5.1.1
container_name: elasticsearch
environment:
- cluster.name=docker-cluster
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
mem_limit: 1g
volumes:
- ./esdata:/usr/share/elasticsearch/data
kibana:
image: kibana:5.1.1
container_name: kibana
environment:
- ELASTICSEARCH_URL=http://elasticsearch:9200
ports:
- "5601:5601"
filebeat/filebeat.yml
:
filebeat.prospectors:
- input_type: log
# Paths that should be crawled and fetched. Glob based paths.
paths:
- /logs/*.log
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts
hosts: ["logstash:5044"]
logstash.conf
:
input {
beats {
port => "5044"
}
}
filter {
grok {
match => { "message" => "%{WORD}\[%{NUMBER}\]: \[%{HTTPDATE:time}\] %{IPORHOST:ip} \"(?:%{WORD:verbs} %{NOTSPACE:request}(?: HTTP/%{NUMBER:http})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:agent} \"%{NUMBER:duration}\"" }
}
date {
match => [ "time", "dd/MMM/yyyy:HH:mm:ss Z" ]
}
if "_grokparsefailure" in [tags] {
drop { }
}
geoip {
source => "ip"
database => "/etc/logstash/geo_db"
fields => [
"city_name",
"continent_code",
"country_code2",
#"country_code3",
"country_name",
#"dma_code",
#"ip",
#"latitude",
"location",
#"longitude",
#"postal_code",
"region_name",
"timezone"
]
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
}