Created
December 21, 2020 16:38
-
-
Save karenc/e20bdcbc14d12ac06017987f96a2d846 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
from datetime import datetime | |
import glob | |
import os | |
import pathlib | |
import re | |
import subprocess | |
import sys | |
import tempfile | |
from urllib.parse import urlparse, urljoin | |
root_url = os.getenv('ROOT_URL', 'http://localhost:6868') | |
paths = [ | |
'/api/annot/', | |
'/view/', | |
'/view/imagesets/', | |
'/api/image/', | |
'/api/name/', | |
'/api/review/', | |
'/turk/', | |
'/turk/cameratrap/?imgsetid=None', | |
'/turk/detection/?imgsetid=None', | |
'/turk/annotation/?imgsetid=None', | |
'/turk/species/?imgsetid=None', | |
'/turk/viewpoint/?imgsetid=None', | |
'/turk/viewpoint3/?imgsetid=None', | |
'/turk/quality/?imgsetid=None', | |
'/turk/demographics/?imgsetid=None', | |
'/turk/part/type/?imgsetid=None', | |
'/turk/part/contour/?imgsetid=None', | |
'/turk/identification/lnbnn/?imgsetid=None', | |
'/turk/identification/graph/', | |
'/turk/identification/hardcase/', | |
] | |
def slugify(url): | |
path = urlparse(url).path | |
return re.sub('[^A-Za-z0-9%]+', '-', path).strip('-') | |
def get_url(path, dirpath): | |
url = urljoin(root_url, path) | |
fname = '{}.html'.format(os.path.join(dirpath, slugify(url))) | |
output = subprocess.check_output(['wget', '-O', fname, url], stderr=subprocess.STDOUT) | |
with open('wget.stdout', 'ab') as f: | |
f.write(output) | |
start_time = None | |
for line in output.splitlines(): | |
line = line.decode('utf-8') | |
time = re.search('^-?-?([0-9-]+ [0-9:]+)', line) | |
if time: | |
last_time = datetime.strptime(time.group(1), '%Y-%m-%d %H:%M:%S') | |
elif re.search('awaiting response... [0-9]+', line) and not start_time: | |
start_time = last_time | |
time_taken = (last_time - start_time).seconds | |
msg = '{} {} seconds'.format(url, time_taken) | |
if time_taken >=60: | |
msg += ' ({} minutes {} seconds)'.format(time_taken // 60, time_taken % 60) | |
print(msg) | |
with open(fname) as f: | |
if 'Traceback' not in f.read(): | |
pathlib.Path(fname).unlink() | |
if __name__ == '__main__': | |
print('ROOT_URL={}'.format(root_url)) | |
dirpath = tempfile.mkdtemp(dir='./') | |
for path in paths: | |
get_url(path, dirpath) | |
if glob.glob('{}/*'.format(dirpath)): | |
print('These files need to be checked manually:') | |
print('\n'.join(glob.glob('{}/*'.format(dirpath)))) | |
sys.exit(1) | |
else: | |
pathlib.Path(dirpath).rmdir() |
Using the sql/optimized-queries
branch:
karen.chan@cthulhu:~/flukebook-testing/wildbook-ia$ ./benchmark_urls.py
ROOT_URL=http://localhost:6868
http://localhost:6868/api/annot/ 5 seconds
http://localhost:6868/view/ 13 seconds
http://localhost:6868/view/imagesets/ 34 seconds
http://localhost:6868/api/image/ 1 seconds
http://localhost:6868/api/name/ 0 seconds
http://localhost:6868/api/review/ 0 seconds
http://localhost:6868/turk/ 0 seconds
http://localhost:6868/turk/cameratrap/?imgsetid=None 5 seconds
http://localhost:6868/turk/detection/?imgsetid=None 5 seconds
http://localhost:6868/turk/annotation/?imgsetid=None 13 seconds
http://localhost:6868/turk/species/?imgsetid=None 19 seconds
http://localhost:6868/turk/viewpoint/?imgsetid=None 13 seconds
http://localhost:6868/turk/viewpoint3/?imgsetid=None 13 seconds
http://localhost:6868/turk/quality/?imgsetid=None 13 seconds
http://localhost:6868/turk/demographics/?imgsetid=None 23 seconds
http://localhost:6868/turk/part/type/?imgsetid=None 12 seconds
http://localhost:6868/turk/part/contour/?imgsetid=None 10 seconds
http://localhost:6868/turk/identification/lnbnn/?imgsetid=None 6 seconds
http://localhost:6868/turk/identification/graph/ 327 seconds (5 minutes 27 seconds)
http://localhost:6868/turk/identification/hardcase/ 92 seconds (1 minutes 32 seconds)
These files need to be checked manually:
./tmpma1ccu1q/turk-identification-lnbnn.html
karen.chan@cthulhu:~/flukebook-testing/wildbook-ia$ ./benchmark_urls.py
ROOT_URL=http://localhost:6868
http://localhost:6868/api/annot/ 6 seconds
http://localhost:6868/view/ 13 seconds
http://localhost:6868/view/imagesets/ 35 seconds
http://localhost:6868/api/image/ 1 seconds
http://localhost:6868/api/name/ 0 seconds
http://localhost:6868/api/review/ 0 seconds
http://localhost:6868/turk/ 0 seconds
http://localhost:6868/turk/cameratrap/?imgsetid=None 6 seconds
http://localhost:6868/turk/detection/?imgsetid=None 6 seconds
http://localhost:6868/turk/annotation/?imgsetid=None 14 seconds
http://localhost:6868/turk/species/?imgsetid=None 19 seconds
http://localhost:6868/turk/viewpoint/?imgsetid=None 14 seconds
http://localhost:6868/turk/viewpoint3/?imgsetid=None 15 seconds
http://localhost:6868/turk/quality/?imgsetid=None 13 seconds
http://localhost:6868/turk/demographics/?imgsetid=None 26 seconds
http://localhost:6868/turk/part/type/?imgsetid=None 12 seconds
http://localhost:6868/turk/part/contour/?imgsetid=None 12 seconds
http://localhost:6868/turk/identification/lnbnn/?imgsetid=None 0 seconds
http://localhost:6868/turk/identification/graph/ 18 seconds
http://localhost:6868/turk/identification/hardcase/ 29 seconds
These files need to be checked manually:
./tmp8k9vwms1/turk-identification-lnbnn.html
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Using the
develop
branch:Using the
sql/exp-merge-4
branch: