Tento repositář obsahuje zejména konfigurační soubory použité pro přednášku na konferenci Elastic.
Soubor install.sh
obsahuje příkazy použité pro instalaci a spuštění jednotlivých nástrojů.
Tento repositář obsahuje zejména konfigurační soubory použité pro přednášku na konferenci Elastic.
Soubor install.sh
obsahuje příkazy použité pro instalaci a spuštění jednotlivých nástrojů.
require 'opbeat' | |
config = Opbeat::Configuration.new do |config| | |
config.organization_id = ENV['OPBEAT_ORGANIZATION_ID'] | |
config.secret_token = ENV['OPBEAT_SECRET_TOKEN'] | |
config.app_id = '29f263051e' | |
config.enabled_environments += %w[ default development ] | |
config.transaction_post_interval = 10 | |
config.logger = Logger.new(STDERR) | |
end | |
Opbeat.start! config | |
require "sinatra" | |
class Application < Sinatra::Base | |
use Opbeat::Middleware | |
enable :logging | |
get '/' do | |
content_type :text | |
rand(1..6).to_s | |
end | |
get '/*' do |path| | |
content_type :text | |
params[:path].to_s | |
end | |
post '/*' do |path| | |
content_type :text | |
params[:path].to_s | |
end | |
end | |
Application.run! if $0 == __FILE__ |
# Export AWS SES credentials as environment variables | |
cluster.routing.allocation.disk.threshold_enabled: false | |
xpack.notification.email.account: | |
ses_account: | |
smtp: | |
auth: true | |
starttls.enable: true | |
starttls.required: true | |
host: email-smtp.us-east-1.amazonaws.com | |
port: 587 | |
user: ${AWS_SES_USER} | |
password: ${AWS_SES_PASSWORD} |
filebeat.modules: | |
- module: nginx | |
access: | |
var.paths: ["/tmp/nginx/log/access.log*"] | |
error: | |
var.paths: ["/tmp/nginx/log/error.log*"] | |
output.elasticsearch: | |
hosts: ["localhost"] | |
username: "elastic" | |
password: "<REPLACE>" |
# Download packages for Mac OS X | |
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0-rc1.tar.gz | |
wget https://artifacts.elastic.co/downloads/kibana/kibana-6.0.0-rc1-darwin-x86_64.tar.gz | |
wget https://artifacts.elastic.co/downloads/logstash/logstash-6.0.0-rc1.tar.gz | |
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.0.0-rc1-darwin-x86_64.tar.gz | |
for f in *.tar.gz; do tar xf $f; done | |
find . -name '.DS_Store' -delete # :) | |
# Setup Elasticsearch with X-Pack | |
./elasticsearch-6.0.0-rc1/bin/elasticsearch-plugin install x-pack | |
./elasticsearch-6.0.0-rc1/bin/elasticsearch | |
# Generate passwords | |
./elasticsearch-6.0.0-rc1/bin/x-pack/setup-passwords auto --batch | tee passwords.txt | |
$EDITOR config/elasticsearch.yml | |
# Setup Kibana with X-Pack | |
./kibana-6.0.0-rc1-darwin-x86_64/bin/kibana-plugin install x-pack | |
$EDITOR config/kibana.yml | |
./kibana-6.0.0-rc1-darwin-x86_64/kibana | |
# Install plugins for Filebeat Nginx module | |
./elasticsearch-6.0.0-rc1/bin/elasticsearch-plugin install ingest-user-agent | |
./elasticsearch-6.0.0-rc1/bin/elasticsearch-plugin install ingest-geoip | |
# Run filebeat against `/tmp/nginx/log/` | |
./filebeat-6.0.0-rc1-darwin-x86_64/filebeat -v -e -c filebeat.yml setup | |
./filebeat-6.0.0-rc1-darwin-x86_64/filebeat -v -e -c filebeat.yml run | |
open 'http://localhost:5601/app/kibana#/dashboard/Filebeat-Nginx-Dashboard' | |
# Run Metricbeat | |
./metricbeat-6.0.0-rc1-darwin-x86_64/metricbeat -v -e -c /Users/karmi/Contracts/Elasticsearch/Talks/Datascript-2017-10/gist/metricbeat.yml run | |
open 'http://localhost:5601/app/kibana#/dashboard/Metricbeat-system-overview' |
elasticsearch.username: "kibana" | |
elasticsearch.password: "<REPLACE>" |
metricbeat.modules: | |
- module: system | |
period: 10s | |
metricsets: | |
- cpu | |
- load | |
- memory | |
- network | |
- process | |
- process_summary | |
output.elasticsearch: | |
hosts: ["localhost"] | |
username: "elastic" | |
password: "<REPLACE>" |
events { | |
worker_connections 1024; | |
} | |
http { | |
upstream application { | |
server localhost:4567; | |
} | |
server { | |
listen 8080; | |
access_log /tmp/nginx/log/access.log combined; | |
error_log /tmp/nginx/log/error.log error; | |
location / { | |
proxy_set_header Host $host; | |
proxy_set_header X-Real-IP $remote_addr; | |
proxy_set_header X-Forwarded-Proto https; | |
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; | |
proxy_pass http://application; | |
} | |
} | |
} |
{ | |
"trigger": { | |
"schedule": { | |
"interval": "60s" | |
} | |
}, | |
"input": { | |
"search": { | |
"request": { | |
"indices": ["filebeat-*"], | |
"body": { | |
"size": 0, | |
"query": { | |
"bool": { | |
"must": [ | |
{ | |
"range": { | |
"nginx.access.response_code": { | |
"gte": 500 | |
} | |
} | |
}, | |
{ | |
"range": { | |
"@timestamp": { | |
"from": "{{ctx.trigger.scheduled_time}}||-60s", | |
"to": "{{ctx.trigger.triggered_time}}" | |
} | |
} | |
} | |
] | |
} | |
}, | |
"aggregations" : { | |
"top_urls" : { | |
"terms" : { | |
"field" : "nginx.access.url" | |
} | |
} | |
} | |
} | |
} | |
} | |
}, | |
"condition": { | |
"compare": { | |
"ctx.payload.hits.total": { | |
"gt": 0 | |
} | |
} | |
}, | |
"actions": { | |
"debug": { | |
"logging": { | |
"level": "info", | |
"text": "There are [{{ctx.payload.hits.total}}] Nginx 50x errors in the last minute." | |
} | |
}, | |
"send_email" : { | |
"transform": { | |
"script": { | |
"lang": "painless", | |
"inline": "[ 'total': ctx.payload.hits.total, 'top_urls': ctx.payload.aggregations.top_urls.buckets.collect(bucket -> [ 'url': bucket.key, 'errors': bucket.doc_count ]) ]" | |
} | |
}, | |
"email": { | |
"account": "ses_account", | |
"from" : "<REPLACE>", | |
"to" : "<REPLACE>", | |
"subject" : "[watcher] Nginx 500", | |
"body" : "There are [{{ctx.payload.total}}] Nginx 50x Errors in the last minute.\n\nTop URLs:\n\n{{#ctx.payload.top_urls}}* {{url}} ({{errors}} errors)\n{{/ctx.payload.top_urls}}\n\nMore info: http://localhost:5601/goto/<REPLACE>" | |
} | |
} | |
} | |
} |