Created
July 18, 2019 19:04
-
-
Save tvansteenburgh/ac306f2b8f4ed44764c2a18c1f08e878 to your computer and use it in GitHub Desktop.
Install and configure Telegraf to parse Juju Charm Store API logs and write metrics to Influxdb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Install and configure Telegraf to parse Juju Charm Store API logs | |
# and write metrics to Influxdb. | |
set -ex | |
# Install Telegraf | |
wget -qO- https://repos.influxdata.com/influxdb.key | sudo apt-key add - | |
source /etc/lsb-release | |
echo "deb https://repos.influxdata.com/${DISTRIB_ID,,} ${DISTRIB_CODENAME} stable" | sudo tee /etc/apt/sources.list.d/influxdb.list | |
sudo apt-get update -y | |
sudo apt-get install -y telegraf || true | |
sudo systemctl stop telegraf | |
# Configure Telegraf to: | |
# 1. Watch for new log files in /home/ubuntu/logs/** | |
# 2. Parse the logs using the rules defined here | |
# 3. Write the metric data to Influxdb | |
# | |
# More info about the Telegraf plugins used below can be found at | |
# https://github.com/influxdata/telegraf/tree/master/plugins | |
cat >/etc/telegraf/telegraf.conf << 'EOF' | |
[agent] | |
interval = "10ms" | |
logfile = "/var/log/telegraf/telegraf.log" | |
quiet = false | |
[[inputs.logparser]] | |
# The logparser plugin runs first and tokenizes the log line into fields, | |
# using the provided grok pattern(s). All subsequent plugins operate on fields. | |
files = ["/home/ubuntu/logs/**"] | |
from_beginning = true | |
# Exclude some fields we don't care about | |
tagexclude = ["path", "host"] | |
[inputs.logparser.grok] | |
measurement = "charmstore_api_request" | |
patterns = [ | |
"%{COMBINED_LOG_FORMAT} [^\"].*\"%{KVPAIRS:kv}\"" | |
] | |
custom_patterns = ''' | |
KVPAIRS [^"].* | |
SUCCESS_CODE 2.. | |
CHARMSTORE /(charmstore|v5)\S+ | |
COMMON_LOG_FORMAT %{CLIENT:client_ip} %{NOTSPACE:ident} %{NOTSPACE:auth} \[%{HTTPDATE:ts:ts-httpd}\] "(?:%{WORD:verb:tag} %{CHARMSTORE:request}(?: HTTP/%{NUMBER:http_version:float})?|%{DATA})" %{SUCCESS_CODE:resp_code:tag} (?:%{NUMBER:resp_bytes:int}|-) | |
COMBINED_LOG_FORMAT %{COMMON_LOG_FORMAT} "%{DATA:referrer}" "%{DATA:agent}" | |
''' | |
[[processors.strings]] | |
# Do some basic string replacements | |
order = 1 # run this plugin first | |
[[processors.strings.replace]] | |
field = "kv" | |
old = "," | |
new = "" | |
[[processors.strings.replace]] | |
field = "request" | |
old = "%2F" | |
new = "/" | |
[[processors.strings.replace]] | |
field = "request" | |
old = "%3A" | |
new = ":" | |
[[processors.parser]] | |
# Split each kv pair in the "kv" field out into its own field, where | |
# the key becomes the field name, and the value becomes the field value | |
order = 2 | |
parse_fields = ["kv"] | |
merge = "override" | |
data_format = "logfmt" | |
[[processors.regex]] | |
# Lastly, do regex processing on the fields. These run in the order in | |
# which they are defined. | |
order = 3 | |
namepass = ["charmstore_api_request"] | |
[[processors.regex.fields]] | |
# Apply pattern to the "request" field. If it matches, create a new | |
# "namespace" field with the value from the first match subgroup | |
key = "request" | |
pattern = '^[^~]*~([^/]*)/.*$' | |
replacement = "${1}" | |
result_key = "namespace" | |
[[processors.regex.fields]] | |
# If the "environment_uuid" field exists and has a value, create a new | |
# "request_type" field, with value "upgrade" | |
key = "environment_uuid" | |
pattern = '^.*$' | |
replacement = "upgrade" | |
result_key = "request_type" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create or update "request_type" field | |
# with value "download" | |
key = "request" | |
pattern = '.*archive(\?.*)?$' | |
replacement = "download" | |
result_key = "request_type" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create new "charm_or_bundle" field | |
# with value from the first match subgroup | |
key = "request" | |
pattern = '.*v5/([^/\?]*).*' | |
replacement = "${1}" | |
result_key = "charm_or_bundle" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create or update "charm_or_bundle" field | |
# with value from the third match subgroup | |
key = "request" | |
pattern = '.*id=(cs:)?(~[^/]*/)?([^&]*).*' | |
replacement = "${3}" | |
result_key = "charm_or_bundle" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create or update "charm_or_bundle" field | |
# with value from the first match subgroup | |
key = "request" | |
pattern = '.*v5/~[^/]*/([^/]*).*' | |
replacement = "${1}" | |
result_key = "charm_or_bundle" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create or update "charm_or_bundle" field | |
# with value from the first match subgroup | |
key = "request" | |
## https://github.com/juju/charmstore/blob/v5/internal/series/series.go#L38 | |
pattern = '.*v5/((bundle|oneiric|precise|quantal|raring|saucy|trusty|utopic|vivid|wily|xenial|yakkety|zesty|artful|bionic|cosmic|disco|win2012hvr2|win2012hv|win2012r2|win2012|win7|win8|win81|win10|win2016|win2016hv|win2016nano|centos7|kubernetes)/([^/]*)).*' | |
replacement = "${1}" | |
result_key = "charm_or_bundle" | |
[[processors.regex.fields]] | |
# If pattern matches on "request" field, create or update "charm_or_bundle" field | |
# with value from the first match subgroup | |
key = "request" | |
## https://github.com/juju/charmstore/blob/v5/internal/series/series.go#L38 | |
pattern = '.*v5/~[^/]*/((bundle|oneiric|precise|quantal|raring|saucy|trusty|utopic|vivid|wily|xenial|yakkety|zesty|artful|bionic|cosmic|disco|win2012hvr2|win2012hv|win2012r2|win2012|win7|win8|win81|win10|win2016|win2016hv|win2016nano|centos7|kubernetes)/([^/]*)).*' | |
replacement = "${1}" | |
result_key = "charm_or_bundle" | |
[[processors.regex.fields]] | |
# Parse the charm or bundle revision number into a new field | |
key = "charm_or_bundle" | |
pattern = '.*-(\d*)$' | |
replacement = "${1}" | |
result_key = "revision" | |
[[processors.regex.fields]] | |
# Parse the charm or bundle series into a new field | |
key = "charm_or_bundle" | |
pattern = '^(.*)/.*$' | |
replacement = "${1}" | |
result_key = "series" | |
[[processors.regex.fields]] | |
# Remove series from the charm or bundle name, since we have a series field | |
key = "charm_or_bundle" | |
pattern = '^([^/]*)/(.*)$' | |
replacement = "${2}" | |
[[processors.regex.fields]] | |
# Remove revision from charm or bundle name, since we have a revision field | |
key = "charm_or_bundle" | |
pattern = '^(.*)-(\d*)$' | |
replacement = "${1}" | |
[[processors.converter]] | |
order = 4 | |
# Remove the kv field, we don't need to store it in the db | |
fielddrop = ["kv"] | |
# Convert some fields to tags. In Influxdb, tags are indexed, fields are not | |
# (see https://docs.influxdata.com/influxdb/v1.7/concepts/key_concepts/#discussion). | |
[processors.converter.fields] | |
tag = ["cloud", "cloud_region", "provider", "controller_version", "namespace", "request_type", "charm_or_bundle", "revision", "series"] | |
# Convert the revision field to an integer | |
integer = ["revision"] | |
[processors.converter.tags] | |
# Convert the revision tag to an integer | |
integer = ["revision"] | |
[[outputs.influxdb]] | |
## The full HTTP or UDP endpoint URL for your InfluxDB instance. | |
urls = ["http://x.x.x.x:8086"] # required | |
username = "xxxxxxxxxx" | |
password = "xxxxxxxxxx" | |
## The target database for metrics (telegraf will create it if not exists). | |
database = "mydatabase" # required | |
## Write timeout (for the InfluxDB client), formatted as a string. | |
timeout = "5s" | |
EOF | |
sudo systemctl start telegraf | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment