Skip to content

Instantly share code, notes, and snippets.

@robert-moses
Forked from ThisIsQasim/README.md
Created January 12, 2022 17:04
Show Gist options
  • Save robert-moses/9723088ff1fd0f4ebea3c2432dab516a to your computer and use it in GitHub Desktop.
Save robert-moses/9723088ff1fd0f4ebea3c2432dab516a to your computer and use it in GitHub Desktop.
PiHole + Elasticsearch Ingest processing

NOTE - this was specifically built for a docker instance, with the Filebeat docker module collecting the logs.

filebeat.inputs:
- type: log
paths:
- "/var/log/pihole.log"
fields:
fileset.module: pihole
fields_under_root: true
max_backoff: 10s
pipeline: "pihole-logging"
processors:
- add_cloud_metadata: ~
- add_docker_metadata: ~
- drop_event:
when:
contains:
message: 'unifi'
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
# we need to do two when.contains here
hosts: ["10.3.0.10:9200"]
setup.ilm.enabled: false
PUT _ingest/pipeline/pihole-logging
{
"description" : "Pi-Hole Logging",
"processors": [
{
"grok": {
"field": "message",
"patterns": ["""%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: query\[%{NOTSPACE:query_record}\] %{NOTSPACE:query_target} from %{NOTSPACE:device_ip}""",
"""%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\:( gravity | )%{NOTSPACE:query_action} %{NOTSPACE:query_target} is %{NOTSPACE:query_answer}""",
"""%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: %{NOTSPACE:query_action} %{NOTSPACE:query_target} to %{NOTSPACE:dns_server}""",
"""%{SYSLOGTIMESTAMP:reqtimestamp} %{USER:program}\[%{NONNEGINT:pid}\]\: (DHCPREQUEST|DHCPACK|DHCPDISCOVER|DHCPOFFER|DHCPRELEASE)\(%{WORD:pihole_interface}\)( %{IP:device_ip})? %{COMMONMAC:device_mac}( %{NOTSPACE:device_name})?"""]
}
},
{
"date": {
"field": "reqtimestamp",
"target_field": "@timestamp",
"timezone" : "Asia/Karachi",
"formats": [
"MMM d HH:mm:ss",
"MMM dd HH:mm:ss"
]
}
},
{
"remove": {
"field": "reqtimestamp"
}
},
{
"remove": {
"field": "message"
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment