Created
January 17, 2017 17:55
-
-
Save dyno/632846334d5de403cef5a89031e16731 to your computer and use it in GitHub Desktop.
drop-in replacement of file object line iterator with tqdm progress bar
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
from contextlib import contextmanager | |
from os.path import getsize, basename | |
from tqdm import tqdm | |
@contextmanager | |
def pbopen(filename): | |
total = getsize(filename) | |
pb = tqdm(total=total, unit="B", unit_scale=True, | |
desc=basename(filename), miniters=1, | |
ncols=80, ascii=True) | |
def wrapped_line_iterator(fd): | |
processed_bytes = 0 | |
for line in fd: | |
processed_bytes += len(line) | |
# update progress every MB. | |
if processed_bytes >= 1024 * 1024: | |
pb.update(processed_bytes) | |
processed_bytes = 0 | |
yield line | |
# finally | |
pb.update(processed_bytes) | |
pb.close() | |
with open(filename) as fd: | |
yield wrapped_line_iterator(fd) | |
if __name__ == "__main__": | |
import sys | |
filename = "whatever.txt" if len(sys.argv) == 1 else sys.argv[1] | |
with pbopen(filename) as f: | |
for line in f: | |
pass |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Just what was required, thanks