Last active
October 21, 2019 09:43
-
-
Save miticojo/fd5202b0d0ea4244bf7e47d4c139aa67 to your computer and use it in GitHub Desktop.
K8S - Centralized logging with ELK and Fluentd (kubernetes >= 1.6)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: elasticsearch-logging | |
namespace: kube-system | |
labels: | |
k8s-app: elasticsearch-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
--- | |
kind: ClusterRole | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
metadata: | |
name: elasticsearch-logging | |
labels: | |
k8s-app: elasticsearch-logging | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
rules: | |
- apiGroups: | |
- "" | |
resources: | |
- "services" | |
- "namespaces" | |
- "endpoints" | |
verbs: | |
- "get" | |
--- | |
kind: ClusterRoleBinding | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
metadata: | |
namespace: kube-system | |
name: elasticsearch-logging | |
labels: | |
k8s-app: elasticsearch-logging | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
subjects: | |
- kind: ServiceAccount | |
name: elasticsearch-logging | |
namespace: kube-system | |
apiGroup: "" | |
roleRef: | |
kind: ClusterRole | |
name: elasticsearch-logging | |
apiGroup: "" | |
--- | |
apiVersion: v1 | |
kind: ReplicationController | |
metadata: | |
name: elasticsearch-logging-v1 | |
namespace: kube-system | |
labels: | |
k8s-app: elasticsearch-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
spec: | |
replicas: 2 | |
selector: | |
k8s-app: elasticsearch-logging | |
version: v1 | |
template: | |
metadata: | |
labels: | |
k8s-app: elasticsearch-logging | |
version: v1 | |
kubernetes.io/cluster-service: "true" | |
spec: | |
serviceAccountName: elasticsearch-logging | |
containers: | |
- image: gcr.io/google_containers/elasticsearch:v2.4.1-2 | |
name: elasticsearch-logging | |
resources: | |
# need more cpu upon initialization, therefore burstable class | |
limits: | |
cpu: 1000m | |
requests: | |
cpu: 100m | |
ports: | |
- containerPort: 9200 | |
name: db | |
protocol: TCP | |
- containerPort: 9300 | |
name: transport | |
protocol: TCP | |
volumeMounts: | |
- name: es-persistent-storage | |
mountPath: /data | |
env: | |
- name: "NAMESPACE" | |
valueFrom: | |
fieldRef: | |
fieldPath: metadata.namespace | |
volumes: | |
- name: es-persistent-storage | |
emptyDir: {} | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: elasticsearch-logging | |
namespace: kube-system | |
labels: | |
k8s-app: elasticsearch-logging | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
kubernetes.io/name: "Elasticsearch" | |
spec: | |
ports: | |
- port: 9200 | |
protocol: TCP | |
targetPort: db | |
selector: | |
k8s-app: elasticsearch-logging | |
--- | |
apiVersion: v1 | |
kind: ServiceAccount | |
metadata: | |
name: fluentd-es | |
namespace: kube-system | |
labels: | |
k8s-app: fluentd-es | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
--- | |
kind: ClusterRole | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
metadata: | |
name: fluentd-es | |
labels: | |
k8s-app: fluentd-es | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
rules: | |
- apiGroups: | |
- "" | |
resources: | |
- "namespaces" | |
- "pods" | |
verbs: | |
- "get" | |
- "watch" | |
- "list" | |
--- | |
kind: ClusterRoleBinding | |
apiVersion: rbac.authorization.k8s.io/v1beta1 | |
metadata: | |
name: fluentd-es | |
labels: | |
k8s-app: fluentd-es | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
subjects: | |
- kind: ServiceAccount | |
name: fluentd-es | |
namespace: kube-system | |
apiGroup: "" | |
roleRef: | |
kind: ClusterRole | |
name: fluentd-es | |
apiGroup: "" | |
--- | |
apiVersion: extensions/v1beta1 | |
kind: DaemonSet | |
metadata: | |
name: fluentd-es-v1.22 | |
namespace: kube-system | |
labels: | |
k8s-app: fluentd-es | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
version: v1.22 | |
spec: | |
template: | |
metadata: | |
labels: | |
k8s-app: fluentd-es | |
kubernetes.io/cluster-service: "true" | |
version: v1.22 | |
# This annotation ensures that fluentd does not get evicted if the node | |
# supports critical pod annotation based priority scheme. | |
# Note that this does not guarantee admission on the nodes (#40573). | |
annotations: | |
scheduler.alpha.kubernetes.io/critical-pod: '' | |
spec: | |
serviceAccountName: fluentd-es | |
containers: | |
- name: fluentd-es | |
image: gcr.io/google_containers/fluentd-elasticsearch:1.23 | |
command: | |
- '/bin/sh' | |
- '-c' | |
- '/usr/sbin/td-agent $FLUENTD_ARGS' | |
env: | |
- name: FLUENTD_AGRS | |
value: -q | |
resources: | |
limits: | |
memory: 200Mi | |
requests: | |
cpu: 100m | |
memory: 200Mi | |
volumeMounts: | |
- name: varlog | |
mountPath: /var/log | |
- name: varlibdockercontainers | |
mountPath: /var/lib/docker/containers | |
readOnly: true | |
terminationGracePeriodSeconds: 30 | |
volumes: | |
- name: varlog | |
hostPath: | |
path: /var/log | |
- name: varlibdockercontainers | |
hostPath: | |
path: /var/lib/docker/containers | |
--- | |
apiVersion: extensions/v1beta1 | |
kind: Deployment | |
metadata: | |
name: kibana-logging | |
namespace: kube-system | |
labels: | |
k8s-app: kibana-logging | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
spec: | |
replicas: 1 | |
selector: | |
matchLabels: | |
k8s-app: kibana-logging | |
template: | |
metadata: | |
labels: | |
k8s-app: kibana-logging | |
spec: | |
containers: | |
- name: kibana-logging | |
image: gcr.io/google_containers/kibana:v4.6.1-1 | |
resources: | |
# keep request = limit to keep this container in guaranteed class | |
limits: | |
cpu: 100m | |
requests: | |
cpu: 100m | |
env: | |
- name: "ELASTICSEARCH_URL" | |
value: "http://elasticsearch-logging:9200" | |
- name: "KIBANA_BASE_URL" | |
value: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging" | |
ports: | |
- containerPort: 5601 | |
name: ui | |
protocol: TCP | |
--- | |
apiVersion: v1 | |
kind: Service | |
metadata: | |
name: kibana-logging | |
namespace: kube-system | |
labels: | |
k8s-app: kibana-logging | |
kubernetes.io/cluster-service: "true" | |
addonmanager.kubernetes.io/mode: Reconcile | |
kubernetes.io/name: "Kibana" | |
spec: | |
ports: | |
- port: 5601 | |
protocol: TCP | |
targetPort: ui | |
selector: | |
k8s-app: kibana-logging |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Do you have any instruction for how to use these functions?