Last active
May 14, 2022 13:46
-
-
Save duongphuhiep/9635d227c69868476427dbae7e6c6559 to your computer and use it in GitHub Desktop.
Configure fluentd 0.12 config to accept logs via HTTP inputs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
kind: ConfigMap | |
metadata: | |
name: fluentd-es-config | |
namespace: kube-system | |
labels: | |
addonmanager.kubernetes.io/mode: Reconcile | |
k8s-app: fluentd-es | |
kubernetes.io/minikube-addons: efk | |
data: | |
containers.input.conf: >- | |
# CRI Log Example: | |
# 2016-02-17T00:04:05.931087621Z stdout [info:2016-02-16T16:04:05.930-08:00] | |
Some log text here | |
<source> | |
type tail | |
path /var/log/containers/*.log | |
pos_file /var/log/es-containers.log.pos | |
time_format %Y-%m-%dT%H:%M:%S.%NZ | |
tag kubernetes.* | |
read_from_head true | |
format multi_format | |
<pattern> | |
format json | |
time_key time | |
time_format %Y-%m-%dT%H:%M:%S.%NZ | |
</pattern> | |
<pattern> | |
format /^(?<time>.+)\b(?<stream>stdout|stderr)\b(?<log>.*)$/ | |
time_format %Y-%m-%dT%H:%M:%S.%N%:z | |
</pattern> | |
</source> | |
forward.input.conf: |- | |
# Takes the messages sent over TCP | |
<source> | |
type forward | |
</source> | |
monitoring.conf: |- | |
# Prometheus Exporter Plugin | |
# input plugin that exports metrics | |
<source> | |
@type prometheus | |
</source> | |
<source> | |
@type monitor_agent | |
</source> | |
# input plugin that collects metrics from MonitorAgent | |
<source> | |
@type prometheus_monitor | |
<labels> | |
host ${hostname} | |
</labels> | |
</source> | |
# input plugin that collects metrics for output plugin | |
<source> | |
@type prometheus_output_monitor | |
<labels> | |
host ${hostname} | |
</labels> | |
</source> | |
# input plugin that collects metrics for in_tail plugin | |
<source> | |
@type prometheus_tail_monitor | |
<labels> | |
host ${hostname} | |
</labels> | |
</source> | |
output.conf: |- | |
# Enriches records with Kubernetes metadata | |
<filter kubernetes.**> | |
type kubernetes_metadata | |
</filter> | |
<match **> | |
type elasticsearch | |
log_level info | |
include_tag_key true | |
host elasticsearch-logging | |
port 9200 | |
logstash_format true | |
# Set the chunk limits. | |
buffer_chunk_limit 2M | |
buffer_queue_limit 8 | |
flush_interval 5s | |
# Never wait longer than 5 minutes between retries. | |
max_retry_wait 30 | |
# Disable the limit on the number of retries (retry forever). | |
disable_retry_limit | |
# Use multiple threads for processing. | |
num_threads 2 | |
</match> | |
system.input.conf: >- | |
# Example: | |
# 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state | |
[net.ipv4.ip_forward] at time 23:17:22.066081 | |
<source> | |
type tail | |
format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/ | |
time_format %Y-%m-%d %H:%M:%S | |
path /var/log/salt/minion | |
pos_file /var/log/es-salt.pos | |
tag salt | |
</source> | |
# Example: | |
# Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished | |
running startup script /var/run/google.startup.script | |
<source> | |
type tail | |
format syslog | |
path /var/log/startupscript.log | |
pos_file /var/log/es-startupscript.log.pos | |
tag startupscript | |
</source> | |
# Examples: | |
# time="2016-02-04T06:51:03.053580605Z" level=info msg="GET | |
/containers/json" | |
# time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No | |
such image: -f" statusCode=404 | |
<source> | |
type tail | |
format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ | |
path /var/log/docker.log | |
pos_file /var/log/es-docker.log.pos | |
tag docker | |
</source> | |
# Example: | |
# 2016/02/04 06:52:38 filePurge: successfully removed file | |
/var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal | |
<source> | |
type tail | |
# Not parsing this, because it doesn't have anything particularly useful to | |
# parse out of it (like severities). | |
format none | |
path /var/log/etcd.log | |
pos_file /var/log/es-etcd.log.pos | |
tag etcd | |
</source> | |
# Multi-line parsing is required for all the kube logs because very large | |
log | |
# statements, such as those that include entire object bodies, get split | |
into | |
# multiple lines by klog. | |
# Example: | |
# I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: | |
(13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537] | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/kubelet.log | |
pos_file /var/log/es-kubelet.log.pos | |
tag kubelet | |
</source> | |
# Example: | |
# I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for | |
kube-system/default-http-backend:http" (:31429/tcp) was open before and is | |
still needed | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/kube-proxy.log | |
pos_file /var/log/es-kube-proxy.log.pos | |
tag kube-proxy | |
</source> | |
# Example: | |
# I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: | |
(1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) | |
kubernetes/6a81b50] 127.0.0.1:38266] | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/kube-apiserver.log | |
pos_file /var/log/es-kube-apiserver.log.pos | |
tag kube-apiserver | |
</source> | |
# Example: | |
# I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists | |
and doesn't need update for service kube-system/kube-ui | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/kube-controller-manager.log | |
pos_file /var/log/es-kube-controller-manager.log.pos | |
tag kube-controller-manager | |
</source> | |
# Example: | |
# W0204 06:49:18.239674 7 reflector.go:245] | |
pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: | |
The event in requested index is outdated and cleared (the requested history | |
has been cleared [2578313/2577886]) [2579312] | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/kube-scheduler.log | |
pos_file /var/log/es-kube-scheduler.log.pos | |
tag kube-scheduler | |
</source> | |
# Example: | |
# I1104 10:36:20.242766 5 rescheduler.go:73] Running Rescheduler | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/rescheduler.log | |
pos_file /var/log/es-rescheduler.log.pos | |
tag rescheduler | |
</source> | |
# Example: | |
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from | |
path /etc/gce.conf | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/glbc.log | |
pos_file /var/log/es-glbc.log.pos | |
tag glbc | |
</source> | |
# Example: | |
# I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from | |
path /etc/gce.conf | |
<source> | |
type tail | |
format multiline | |
multiline_flush_interval 5s | |
format_firstline /^\w\d{4}/ | |
format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ | |
time_format %m%d %H:%M:%S.%N | |
path /var/log/cluster-autoscaler.log | |
pos_file /var/log/es-cluster-autoscaler.log.pos | |
tag cluster-autoscaler | |
</source> | |
# Logs from systemd-journal for interesting services. | |
<source> | |
type systemd | |
filters [{ "_SYSTEMD_UNIT": "docker.service" }] | |
pos_file /var/log/gcp-journald-docker.pos | |
read_from_head true | |
tag docker | |
</source> | |
<source> | |
type systemd | |
filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] | |
pos_file /var/log/gcp-journald-kubelet.pos | |
read_from_head true | |
tag kubelet | |
</source> | |
<source> | |
type systemd | |
filters [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }] | |
pos_file /var/log/gcp-journald-node-problem-detector.pos | |
read_from_head true | |
tag node-problem-detector | |
</source> | |
<source> | |
@type http | |
port 8424 | |
bind 0.0.0.0 | |
body_size_limit 32m | |
keepalive_timeout 10s | |
</source> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment