Skip to content

Instantly share code, notes, and snippets.

@danehans

danehans/coredns_1.6.6_metrics_with_failed_upstream

Last active July 23, 2020 18:09
Show Gist options
  • Save danehans/efd3dc5b84a0db21dc908c9852c2a592 to your computer and use it in GitHub Desktop.
Save danehans/efd3dc5b84a0db21dc908c9852c2a592 to your computer and use it in GitHub Desktop.
Download ZIP
ocp_coredns_1.6.6_metrics
Raw
coredns_1.6.6_metrics_with_failed_upstream
# Verify resolution works with the configured "216.239.32.106" upstream:
sh-4.2$ nslookup resume.daneyon.com
Server: 172.30.0.10
Address: 172.30.0.10#53
resume.daneyon.com canonical name = danehans.github.io.
# Update the Corefile to use a bogus "216.239.32.1" upstream server and verify resolution now fails:
sh-4.2$ nslookup resume.daneyon.com
Server: 172.30.0.10
Address: 172.30.0.10#53
** server can't find resume.daneyon.com: SERVFAIL
# Check for the presence of metric `coredns_forward_healthcheck_failures_total`. It still does not exist:
sh-4.2$ curl -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IlNCRFRLMnZpLWh2QUY1ZG03YU1XMVYzUUJsaHVfZ0JXVXNqREw0OWJ2M00ifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi1nbHRsaCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjZhYzI2ZmJiLWNmZDgtNDVkMi1hZTc5LTg0NDIxNWYwNDdjZiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.GSLENM7fEURxhMAvfXDNNHRMRM9sN61BLEwYixHMMB44OWSCUEL3MN7XJEZ7Hjoof2nUptuaoSW7K7cPjNvE90jUgZcDNOpJV4Yu91-Rhy3gYnEB43xBPvIdbgr2tysSeYdEfPlukk_ZEznWNRaHbFa6nx9H4vadMi1Pae9he3FHIv8ytHkmd0Dho95E-VRlcGWbW6GyshGAUMeJltXKrV2rgGTzh40a6WgffuH0lFsEly5kQQdi1dE_CvLq8poSYDPo_GOCv4lkQVonSBSzcrcdDm8W9svEP6wIqkBpWIJr7Luf6yhjtt0YEvH5moAyq4V4KbyL7MZzFZCs6x5sEA" -k https://172.30.0.10:9154/metrics
# HELP coredns_build_info A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.
# TYPE coredns_build_info gauge
coredns_build_info{goversion="go1.13.4",revision="",version="1.6.6"} 1
# HELP coredns_cache_hits_total The count of cache hits.
# TYPE coredns_cache_hits_total counter
coredns_cache_hits_total{server="dns://:5353",type="denial"} 4802
coredns_cache_hits_total{server="dns://:5353",type="success"} 2057
# HELP coredns_cache_size The number of elements in the cache.
# TYPE coredns_cache_size gauge
coredns_cache_size{server="dns://:5353",type="denial"} 28
coredns_cache_size{server="dns://:5353",type="success"} 13
# HELP coredns_dns_request_count_total Counter of DNS requests made per zone, protocol and family.
# TYPE coredns_dns_request_count_total counter
coredns_dns_request_count_total{family="1",proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_request_count_total{family="1",proto="udp",server="dns://:5353",zone="."} 18077
# HELP coredns_dns_request_duration_seconds Histogram of the time (in seconds) each request took.
# TYPE coredns_dns_request_duration_seconds histogram
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.00025"} 15534
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.0005"} 15759
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.001"} 16697
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.002"} 17643
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.004"} 17805
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.008"} 17932
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.016"} 18026
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.032"} 18071
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.064"} 18077
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.128"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.256"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.512"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="1.024"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="2.048"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="4.096"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="8.192"} 18078
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="+Inf"} 18078
coredns_dns_request_duration_seconds_sum{server="dns://:5353",zone="."} 7.6443448269999905
coredns_dns_request_duration_seconds_count{server="dns://:5353",zone="."} 18078
# HELP coredns_dns_request_size_bytes Size of the EDNS0 UDP buffer in bytes (64K for TCP).
# TYPE coredns_dns_request_size_bytes histogram
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="100"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="200"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="300"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="400"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="511"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="1023"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="2047"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="4095"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="8291"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="16000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="32000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="48000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="64000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="+Inf"} 1
coredns_dns_request_size_bytes_sum{proto="tcp",server="dns://:5353",zone="."} 84
coredns_dns_request_size_bytes_count{proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="100"} 12528
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="200"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="300"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="400"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="511"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="1023"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="2047"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="4095"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="8291"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="16000"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="32000"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="48000"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="64000"} 18077
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="+Inf"} 18077
coredns_dns_request_size_bytes_sum{proto="udp",server="dns://:5353",zone="."} 1.507609e+06
coredns_dns_request_size_bytes_count{proto="udp",server="dns://:5353",zone="."} 18077
# HELP coredns_dns_request_type_count_total Counter of DNS requests per type, per zone.
# TYPE coredns_dns_request_type_count_total counter
coredns_dns_request_type_count_total{server="dns://:5353",type="A",zone="."} 9008
coredns_dns_request_type_count_total{server="dns://:5353",type="AAAA",zone="."} 8805
coredns_dns_request_type_count_total{server="dns://:5353",type="ANY",zone="."} 1
coredns_dns_request_type_count_total{server="dns://:5353",type="SRV",zone="."} 264
# HELP coredns_dns_response_rcode_count_total Counter of response status codes.
# TYPE coredns_dns_response_rcode_count_total counter
coredns_dns_response_rcode_count_total{rcode="NOERROR",server="dns://:5353",zone="."} 6838
coredns_dns_response_rcode_count_total{rcode="NXDOMAIN",server="dns://:5353",zone="."} 11240
# HELP coredns_dns_response_size_bytes Size of the returned response in bytes.
# TYPE coredns_dns_response_size_bytes histogram
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="100"} 0
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="200"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="300"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="400"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="511"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="1023"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="2047"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="4095"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="8291"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="16000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="32000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="48000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="64000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="+Inf"} 1
coredns_dns_response_size_bytes_sum{proto="tcp",server="dns://:5353",zone="."} 159
coredns_dns_response_size_bytes_count{proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="100"} 1809
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="200"} 13231
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="300"} 17265
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="400"} 17817
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="511"} 17817
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="1023"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="2047"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="4095"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="8291"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="16000"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="32000"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="48000"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="64000"} 18077
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="+Inf"} 18077
coredns_dns_response_size_bytes_sum{proto="udp",server="dns://:5353",zone="."} 3.331542e+06
coredns_dns_response_size_bytes_count{proto="udp",server="dns://:5353",zone="."} 18077
# HELP coredns_forward_request_count_total Counter of requests made per upstream.
# TYPE coredns_forward_request_count_total counter
coredns_forward_request_count_total{to="10.0.0.2:53"} 2454
coredns_forward_request_count_total{to="216.239.32.106:53"} 2
# HELP coredns_forward_request_duration_seconds Histogram of the time each request took.
# TYPE coredns_forward_request_duration_seconds histogram
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.00025"} 43
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.0005"} 150
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.001"} 1491
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.002"} 2067
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.004"} 2182
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.008"} 2329
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.016"} 2402
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.032"} 2447
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.064"} 2453
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.128"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.256"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.512"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="1.024"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="2.048"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="4.096"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="8.192"} 2454
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="+Inf"} 2454
coredns_forward_request_duration_seconds_sum{to="10.0.0.2:53"} 5.443270910000009
coredns_forward_request_duration_seconds_count{to="10.0.0.2:53"} 2454
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.00025"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.0005"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.001"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.002"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.004"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.008"} 0
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.016"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.032"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.064"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.128"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.256"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="0.512"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="1.024"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="2.048"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="4.096"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="8.192"} 2
coredns_forward_request_duration_seconds_bucket{to="216.239.32.106:53",le="+Inf"} 2
coredns_forward_request_duration_seconds_sum{to="216.239.32.106:53"} 0.028755777000000003
coredns_forward_request_duration_seconds_count{to="216.239.32.106:53"} 2
# HELP coredns_forward_response_rcode_count_total Counter of requests made per upstream.
# TYPE coredns_forward_response_rcode_count_total counter
coredns_forward_response_rcode_count_total{rcode="NOERROR",to="10.0.0.2:53"} 939
coredns_forward_response_rcode_count_total{rcode="NOERROR",to="216.239.32.106:53"} 2
coredns_forward_response_rcode_count_total{rcode="NXDOMAIN",to="10.0.0.2:53"} 1515
# HELP coredns_health_request_duration_seconds Histogram of the time (in seconds) each request took.
# TYPE coredns_health_request_duration_seconds histogram
coredns_health_request_duration_seconds_bucket{le="0.00025"} 0
coredns_health_request_duration_seconds_bucket{le="0.0005"} 11014
coredns_health_request_duration_seconds_bucket{le="0.001"} 12098
coredns_health_request_duration_seconds_bucket{le="0.002"} 12259
coredns_health_request_duration_seconds_bucket{le="0.004"} 12332
coredns_health_request_duration_seconds_bucket{le="0.008"} 12354
coredns_health_request_duration_seconds_bucket{le="0.016"} 12355
coredns_health_request_duration_seconds_bucket{le="0.032"} 12355
coredns_health_request_duration_seconds_bucket{le="0.064"} 12356
coredns_health_request_duration_seconds_bucket{le="0.128"} 12356
coredns_health_request_duration_seconds_bucket{le="0.256"} 12356
coredns_health_request_duration_seconds_bucket{le="0.512"} 12356
coredns_health_request_duration_seconds_bucket{le="1.024"} 12356
coredns_health_request_duration_seconds_bucket{le="2.048"} 12356
coredns_health_request_duration_seconds_bucket{le="4.096"} 12356
coredns_health_request_duration_seconds_bucket{le="8.192"} 12356
coredns_health_request_duration_seconds_bucket{le="+Inf"} 12356
coredns_health_request_duration_seconds_sum 6.272321745999985
coredns_health_request_duration_seconds_count 12356
# HELP coredns_kubernetes_dns_programming_duration_seconds Histogram of the time (in seconds) it took to program a dns instance.
# TYPE coredns_kubernetes_dns_programming_duration_seconds histogram
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.001"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.002"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.004"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.008"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.016"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.032"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.064"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.128"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.256"} 1
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.512"} 3
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="1.024"} 12
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="2.048"} 12
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="4.096"} 12
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="8.192"} 12
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="16.384"} 13
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="32.768"} 15
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="65.536"} 16
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="131.072"} 16
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="262.144"} 16
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="524.288"} 17
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="+Inf"} 32
coredns_kubernetes_dns_programming_duration_seconds_sum{service_kind="headless_with_selector"} 23187.974481579004
coredns_kubernetes_dns_programming_duration_seconds_count{service_kind="headless_with_selector"} 32
# HELP coredns_panic_count_total A metrics that counts the number of panics.
# TYPE coredns_panic_count_total counter
coredns_panic_count_total 0
# HELP coredns_plugin_enabled A metric that indicates whether a plugin is enabled on per server and zone basis.
# TYPE coredns_plugin_enabled gauge
coredns_plugin_enabled{name="cache",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="errors",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="forward",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="forward",server="dns://:5353",zone="daneyon.com."} 1
coredns_plugin_enabled{name="kubernetes",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="prometheus",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="prometheus",server="dns://:5353",zone="daneyon.com."} 1
# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 7.859e-06
go_gc_duration_seconds{quantile="0.25"} 1.1324e-05
go_gc_duration_seconds{quantile="0.5"} 1.2236e-05
go_gc_duration_seconds{quantile="0.75"} 1.8633e-05
go_gc_duration_seconds{quantile="1"} 0.000230493
go_gc_duration_seconds_sum 0.002684434
go_gc_duration_seconds_count 115
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 40
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.13.4"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 5.654856e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 4.67314424e+08
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 1.512576e+06
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 4.234684e+06
# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
# TYPE go_memstats_gc_cpu_fraction gauge
go_memstats_gc_cpu_fraction 4.48490122092631e-06
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 2.394112e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 5.654856e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 5.8056704e+07
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 8.298496e+06
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 29722
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 5.6180736e+07
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 6.63552e+07
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.5954461828058968e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 0
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 4.264406e+06
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 3472
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 16384
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 117368
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 163840
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 9.30216e+06
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 566392
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 753664
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 753664
# HELP go_memstats_sys_bytes Number of bytes obtained from system.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 7.1762168e+07
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 11
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 32.14
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 13
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 5.3334016e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.59543385137e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 5.08932096e+08
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes -1
Raw
coredns_1.6.6_metrics_with_working_upstream
# HELP coredns_build_info A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.
# TYPE coredns_build_info gauge
coredns_build_info{goversion="go1.13.4",revision="",version="1.6.6"} 1
# HELP coredns_cache_hits_total The count of cache hits.
# TYPE coredns_cache_hits_total counter
coredns_cache_hits_total{server="dns://:5353",type="denial"} 3422
coredns_cache_hits_total{server="dns://:5353",type="success"} 1327
# HELP coredns_cache_size The number of elements in the cache.
# TYPE coredns_cache_size gauge
coredns_cache_size{server="dns://:5353",type="denial"} 41
coredns_cache_size{server="dns://:5353",type="success"} 19
# HELP coredns_dns_request_count_total Counter of DNS requests made per zone, protocol and family.
# TYPE coredns_dns_request_count_total counter
coredns_dns_request_count_total{family="1",proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_request_count_total{family="1",proto="udp",server="dns://:5353",zone="."} 13248
# HELP coredns_dns_request_duration_seconds Histogram of the time (in seconds) each request took.
# TYPE coredns_dns_request_duration_seconds histogram
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.00025"} 11375
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.0005"} 11535
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.001"} 11905
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.002"} 12754
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.004"} 12993
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.008"} 13103
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.016"} 13217
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.032"} 13248
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.064"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.128"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.256"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="0.512"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="1.024"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="2.048"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="4.096"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="8.192"} 13249
coredns_dns_request_duration_seconds_bucket{server="dns://:5353",zone=".",le="+Inf"} 13249
coredns_dns_request_duration_seconds_sum{server="dns://:5353",zone="."} 5.973002514999999
coredns_dns_request_duration_seconds_count{server="dns://:5353",zone="."} 13249
# HELP coredns_dns_request_size_bytes Size of the EDNS0 UDP buffer in bytes (64K for TCP).
# TYPE coredns_dns_request_size_bytes histogram
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="100"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="200"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="300"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="400"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="511"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="1023"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="2047"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="4095"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="8291"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="16000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="32000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="48000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="64000"} 1
coredns_dns_request_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="+Inf"} 1
coredns_dns_request_size_bytes_sum{proto="tcp",server="dns://:5353",zone="."} 84
coredns_dns_request_size_bytes_count{proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="100"} 9132
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="200"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="300"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="400"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="511"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="1023"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="2047"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="4095"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="8291"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="16000"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="32000"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="48000"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="64000"} 13248
coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="+Inf"} 13248
coredns_dns_request_size_bytes_sum{proto="udp",server="dns://:5353",zone="."} 1.115584e+06
coredns_dns_request_size_bytes_count{proto="udp",server="dns://:5353",zone="."} 13248
# HELP coredns_dns_request_type_count_total Counter of DNS requests per type, per zone.
# TYPE coredns_dns_request_type_count_total counter
coredns_dns_request_type_count_total{server="dns://:5353",type="A",zone="."} 6573
coredns_dns_request_type_count_total{server="dns://:5353",type="AAAA",zone="."} 6460
coredns_dns_request_type_count_total{server="dns://:5353",type="ANY",zone="."} 1
coredns_dns_request_type_count_total{server="dns://:5353",type="SRV",zone="."} 215
# HELP coredns_dns_response_rcode_count_total Counter of response status codes.
# TYPE coredns_dns_response_rcode_count_total counter
coredns_dns_response_rcode_count_total{rcode="NOERROR",server="dns://:5353",zone="."} 5133
coredns_dns_response_rcode_count_total{rcode="NXDOMAIN",server="dns://:5353",zone="."} 8116
# HELP coredns_dns_response_size_bytes Size of the returned response in bytes.
# TYPE coredns_dns_response_size_bytes histogram
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="100"} 0
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="200"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="300"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="400"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="511"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="1023"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="2047"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="4095"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="8291"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="16000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="32000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="48000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="64000"} 1
coredns_dns_response_size_bytes_bucket{proto="tcp",server="dns://:5353",zone=".",le="+Inf"} 1
coredns_dns_response_size_bytes_sum{proto="tcp",server="dns://:5353",zone="."} 159
coredns_dns_response_size_bytes_count{proto="tcp",server="dns://:5353",zone="."} 1
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="0"} 0
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="100"} 1344
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="200"} 9707
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="300"} 12658
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="400"} 13037
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="511"} 13037
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="1023"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="2047"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="4095"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="8291"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="16000"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="32000"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="48000"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="64000"} 13248
coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:5353",zone=".",le="+Inf"} 13248
coredns_dns_response_size_bytes_sum{proto="udp",server="dns://:5353",zone="."} 2.443642e+06
coredns_dns_response_size_bytes_count{proto="udp",server="dns://:5353",zone="."} 13248
# HELP coredns_forward_request_count_total Counter of requests made per upstream.
# TYPE coredns_forward_request_count_total counter
coredns_forward_request_count_total{to="10.0.0.2:53"} 1820
# HELP coredns_forward_request_duration_seconds Histogram of the time each request took.
# TYPE coredns_forward_request_duration_seconds histogram
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.00025"} 60
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.0005"} 128
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.001"} 702
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.002"} 1361
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.004"} 1572
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.008"} 1678
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.016"} 1789
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.032"} 1819
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.064"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.128"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.256"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="0.512"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="1.024"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="2.048"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="4.096"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="8.192"} 1820
coredns_forward_request_duration_seconds_bucket{to="10.0.0.2:53",le="+Inf"} 1820
coredns_forward_request_duration_seconds_sum{to="10.0.0.2:53"} 4.356613313999996
coredns_forward_request_duration_seconds_count{to="10.0.0.2:53"} 1820
# HELP coredns_forward_response_rcode_count_total Counter of requests made per upstream.
# TYPE coredns_forward_response_rcode_count_total counter
coredns_forward_response_rcode_count_total{rcode="NOERROR",to="10.0.0.2:53"} 712
coredns_forward_response_rcode_count_total{rcode="NXDOMAIN",to="10.0.0.2:53"} 1108
# HELP coredns_health_request_duration_seconds Histogram of the time (in seconds) each request took.
# TYPE coredns_health_request_duration_seconds histogram
coredns_health_request_duration_seconds_bucket{le="0.00025"} 0
coredns_health_request_duration_seconds_bucket{le="0.0005"} 8260
coredns_health_request_duration_seconds_bucket{le="0.001"} 8863
coredns_health_request_duration_seconds_bucket{le="0.002"} 9027
coredns_health_request_duration_seconds_bucket{le="0.004"} 9152
coredns_health_request_duration_seconds_bucket{le="0.008"} 9223
coredns_health_request_duration_seconds_bucket{le="0.016"} 9235
coredns_health_request_duration_seconds_bucket{le="0.032"} 9235
coredns_health_request_duration_seconds_bucket{le="0.064"} 9235
coredns_health_request_duration_seconds_bucket{le="0.128"} 9235
coredns_health_request_duration_seconds_bucket{le="0.256"} 9235
coredns_health_request_duration_seconds_bucket{le="0.512"} 9235
coredns_health_request_duration_seconds_bucket{le="1.024"} 9235
coredns_health_request_duration_seconds_bucket{le="2.048"} 9235
coredns_health_request_duration_seconds_bucket{le="4.096"} 9235
coredns_health_request_duration_seconds_bucket{le="8.192"} 9235
coredns_health_request_duration_seconds_bucket{le="+Inf"} 9235
coredns_health_request_duration_seconds_sum 5.062270659000005
coredns_health_request_duration_seconds_count 9235
# HELP coredns_kubernetes_dns_programming_duration_seconds Histogram of the time (in seconds) it took to program a dns instance.
# TYPE coredns_kubernetes_dns_programming_duration_seconds histogram
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.001"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.002"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.004"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.008"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.016"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.032"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.064"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.128"} 0
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.256"} 3
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="0.512"} 13
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="1.024"} 21
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="2.048"} 26
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="4.096"} 26
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="8.192"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="16.384"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="32.768"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="65.536"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="131.072"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="262.144"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="524.288"} 27
coredns_kubernetes_dns_programming_duration_seconds_bucket{service_kind="headless_with_selector",le="+Inf"} 28
coredns_kubernetes_dns_programming_duration_seconds_sum{service_kind="headless_with_selector"} 21245.274901795005
coredns_kubernetes_dns_programming_duration_seconds_count{service_kind="headless_with_selector"} 28
# HELP coredns_panic_count_total A metrics that counts the number of panics.
# TYPE coredns_panic_count_total counter
coredns_panic_count_total 0
# HELP coredns_plugin_enabled A metric that indicates whether a plugin is enabled on per server and zone basis.
# TYPE coredns_plugin_enabled gauge
coredns_plugin_enabled{name="cache",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="errors",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="forward",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="kubernetes",server="dns://:5353",zone="."} 1
coredns_plugin_enabled{name="prometheus",server="dns://:5353",zone="."} 1
# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 9.813e-06
go_gc_duration_seconds{quantile="0.25"} 1.1229e-05
go_gc_duration_seconds{quantile="0.5"} 1.2746e-05
go_gc_duration_seconds{quantile="0.75"} 2.0656e-05
go_gc_duration_seconds{quantile="1"} 0.00078844
go_gc_duration_seconds_sum 0.002567517
go_gc_duration_seconds_count 87
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 40
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.13.4"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 6.22684e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 3.45534024e+08
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 1.508112e+06
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 3.121182e+06
# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started.
# TYPE go_memstats_gc_cpu_fraction gauge
go_memstats_gc_cpu_fraction 3.2859395839455554e-06
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 2.394112e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 6.22684e+06
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 5.791744e+07
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 8.43776e+06
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 34229
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 5.6008704e+07
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 6.63552e+07
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.5953771366796563e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 0
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 3.155411e+06
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 3472
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 16384
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 116688
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 147456
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 9.572992e+06
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 587240
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 753664
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 753664
# HELP go_memstats_sys_bytes Number of bytes obtained from system.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 7.1762168e+07
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 11
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 15.42
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 14
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 5.1245056e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.59536794294e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 4.41823232e+08
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes -1
Raw
corefile_with_upstream
$ oc get cm/dns-default -n openshift-dns -o yaml
apiVersion: v1
data:
Corefile: |
# test
daneyon.com:5353 {
forward . 216.239.32.106
prometheus :9153
reload
}
.:5353 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf {
policy sequential
}
cache 30
reload
}
kind: ConfigMap
Raw
ocp_coredns_v1.6.6_working_upstream_health_metrics
# Corefile needs prom and reload plugins for each server block
$ oc exec dns-default-4df9v -n openshift-dns -c dns -- cat /etc/coredns/Corefile
# test
daneyon.com:5353 {
forward . 216.239.32.106 216.239.32.1
prometheus :9153
reload
}
.:5353 {
errors
health
kubernetes cluster.local in-addr.arpa ip6.arpa {
pods insecure
upstream
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf {
policy sequential
}
cache 30
reload
}
# Curl the metrics endpoint:
sh-4.2$ curl -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImkxTE44SWJKVC12cFUzTG1zdzRrb05BNkI1SzA0bFlGLTkybUJqbU81YWMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi1wcmI2ciIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjA1NGRlZjliLTAwZWMtNDU5YS1iNzdmLWM4MDk3ODFmNmI3YSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.Dd4fku6-TVu6f60JXWdH7mlwPU-woxq_xx0jjIFcEI5O0zl82E5irkNaY-yTrpjrLHV-vrn1kdERJNLmo5t-b79YSHP6g2kyB9bDzXgioot1gnuA780jRE7KcDOgOuQqUfQU2OGajN-bqDdEYe80gfgOqVBLy00MFEpvssrqR5grsOCnpRWk0X1DbIykYrE8JVullB27AEIvmBZ4_PWhnDJgENHDoVvZfAdx9M9jgNkQMhviS6N-kQD9_htnIUxRFACPihS2IYQfhwyygFEWJ2h34hxvYYzorSt8In-DHJkjMo_iCScHUSvfQuFZWHkoNmL_VEsJPflfv_nyQ60nwA" -k https://10.131.0.3:9154/metrics | grep healthcheck
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 30816 0 30816 0 0 987k 0 --:--:-- --:--:-- --:--:-- 1003k
# HELP coredns_forward_healthcheck_failure_count_total Counter of the number of failed healthchecks.
# TYPE coredns_forward_healthcheck_failure_count_total counter
coredns_forward_healthcheck_failure_count_total{to="216.239.32.1:53"} 332
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment