Skip to content

Instantly share code, notes, and snippets.

@ostronom
Created February 15, 2018 08:37
Show Gist options
  • Save ostronom/7593e58b6189b459c8b3b63e28c4ce12 to your computer and use it in GitHub Desktop.
Save ostronom/7593e58b6189b459c8b3b63e28c4ce12 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python2.7
import re
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Q
class RequestList(object):
def __init__(self, host, method_name):
#self.max_distance =
self.method_name = method_name
self.request_re = re.compile(r"^Request messageId (?P<requestId>(\d+)): (?P<rest>(.*))", re.U | re.I)
self.response_re = re.compile(r"^Response for messageId (?P<requestId>(\d+)): (?P<rest>(.*))", re.U | re.I)
level_query = Q("match", level=7)
app_query = Q("match", application="Dialog Big")
logger_query = Q("match", logger="im.dlg.server.session.RpcHandler")
self.base_query = Q("bool", must=[app_query, logger_query, level_query])
self.client = Elasticsearch(host)
self.requests = {}
def get_search(self, index):
return Search(using=self.client, index=index)\
.query(self.base_query)\
.sort('@timestamp')\
.source(['@timestamp', 'message', 'authId'])
def get_message_type_with_params(self, msg):
req = self.request_re.match(msg)
if req is not None:
return (True, req.groupdict())
res = self.response_re.match(msg)
if res is not None:
return (False, res.groupdict())
def insert(self, authId, requestId, is_start, ts):
print 'TS = ', ts, type(ts)
# self.requests.setdefault(authId, {})
# self.requests[authId].setdefault(requestId, [])
# # lookup nearest by ts counterpart for this item
# items = [x for x in self.requests[authId][requestId] if abs(x['ts'] - ts) < self.max_distance]
# for k, item in enumerate(items):
# if item.get('start') is None and is_start:
# item[]
def store_hit(self, hit):
result = self.get_message_type_with_params(hit.message)
if result is None:
return
start, data = result
if self.method_name not in data['rest']:
return
self.insert(i.authId, data['requestId'], start, hit['@timestamp'])
def scan_index(self, index):
for hit in self.get_search(index).scan():
self.store_hit(hit)
def scan(self):
for index in sorted(self.client.indices.get('dlg-servers-*')):
self.scan_index(index)
if __name__ == '__main__':
RequestList('localhost:9200', 'GetDifference').scan()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment