Created
October 23, 2018 20:34
-
-
Save cdahlqvist/c854d381538cd87f163a56fc6b4edb57 to your computer and use it in GitHub Desktop.
HTTP log replayer
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
input { | |
stdin {} | |
} | |
filter { | |
grok { | |
match => { "message" => [ '%{IP:ip}" %{GREEDYDATA:a}', | |
'%{IP:ip1}, %{IP:ip}" %{GREEDYDATA:a}' ] } | |
} | |
fingerprint { | |
method => "IPV4_NETWORK" | |
source => "ip" | |
target => "ip" | |
key => "20" | |
} | |
if [ip1] { | |
mutate { | |
add_field => { "masked" => '"%{ip1}, %{ip}" %{a}' } | |
} | |
} else { | |
mutate { | |
add_field => { "masked" => '"%{ip}" %{a}' } | |
} | |
} | |
geoip { | |
source => "ip" | |
} | |
} | |
output { | |
if [geoip][continent_code] == "NA" or [geoip][continent_code] == "SA" { | |
file { | |
path => "./access_us.log" | |
codec => line { format => "%{masked}"} | |
} | |
} else { | |
file { | |
path => "./access_row.log" | |
codec => line { format => "%{masked}"} | |
} | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import re | |
import time | |
import argparse | |
from datetime import datetime | |
def getKey(item): | |
return item[0] | |
def sort_tuple_list(list): | |
return sorted(list, key=getKey) | |
def timestamp_to_seconds_since_midnight(hh, mi, ss): | |
return (3600 * int(hh)) + (60 * int(mi)) + int(ss) | |
def build_data_map(data_file_path): | |
p1 = re.compile("^(.+)\[\d{2}.\w{3}.\d{4}:(.+) .+\](.*)$") | |
p2 = re.compile("(\d{2}):(\d{2}):(\d{2})") | |
m = {} | |
l = [] | |
recs = [line.rstrip('\n') for line in open(data_file_path)] | |
for rec in recs: | |
res1 = p1.search(rec) | |
res2 = p2.search(res1.group(2)) | |
secs = timestamp_to_seconds_since_midnight(res2.group(1), res2.group(2), res2.group(3)) | |
if secs in m: | |
m[secs].append((res1.group(1), res1.group(3))) | |
else: | |
m[secs] = [(res1.group(1), res1.group(3))] | |
return m | |
def get_current_utc_datetime(): | |
return datetime.utcnow() | |
def get_current_utc_timestamp(dt): | |
month = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] | |
return "%02d/%s/%04d:%02d:%02d:%02d +0000" % (dt.day, month[dt.month], dt.year, dt.hour, dt.minute, dt.second) | |
def get_offset_from_midnight(dt): | |
return (3600 * dt.hour) + (60 * dt.minute) + dt.second | |
def output_record(rec_str): | |
print(rec_str) | |
# Parse command line parameters | |
parser = argparse.ArgumentParser() | |
parser.add_argument('-f', help="Data file path", required=True) | |
args = parser.parse_args() | |
rec_map = build_data_map(args.f) | |
offset = 0 | |
# Loop and output records until interrupted | |
while (True): | |
time.sleep(0.5) | |
cdt = get_current_utc_datetime() | |
last_offset = offset | |
offset = get_offset_from_midnight(cdt) | |
ts = get_current_utc_timestamp(cdt) | |
if offset > last_offset and rec_map[offset]: | |
rec_list = rec_map[offset] | |
for rec in rec_list: | |
output_record("%s[%s]%s" % (rec[0], ts, rec[1])) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment