Created
May 29, 2016 21:58
-
-
Save ovaistariq/35d1b876d299006470d3085a764a371f to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| import hashlib | |
| import subprocess | |
| import time | |
| url = 'http://cachefly.cachefly.net/100mb.test' | |
| backup_file = 'backup.out' | |
| md5sum_file = 'backup.out.md5' | |
| gpg_file = 'backup.out.gpg' | |
| gpg_recipient = 'ovaistariq@gmail.com' | |
| keyring_file = '' | |
| chunk_size = 16*1024 | |
| class MD5Filter(object): | |
| def __init__(self, out_filename): | |
| self._hash = hashlib.md5() | |
| self._hash_filename = out_filename | |
| def processor(self, chunk): | |
| self._hash.update(chunk) | |
| def hash(self): | |
| return self._hash.hexdigest() | |
| def save(self): | |
| with open(self._hash_filename, 'w') as fh: | |
| fh.write(self.hash()) | |
| class GPGFilter(object): | |
| def __init__(self, out_filename, recipient, keyring): | |
| self._recipeint = recipient | |
| self._keyring = keyring | |
| self._gpg_file = out_filename | |
| self._gpg_process = self._create_gpg_process() | |
| def processor(self, chunk): | |
| self._gpg_process.stdin.write(chunk) | |
| def save(self): | |
| self._gpg_process.stdin.close() | |
| def _create_gpg_process(self): | |
| gpg_cmd = ['gpg2', | |
| '--trust-model', 'always', | |
| '-q', '--batch', '--yes', | |
| '--compress-algo', 'none', | |
| '--encrypt', | |
| '--recipient', self._recipeint] | |
| gpg = subprocess.Popen(gpg_cmd, stdin=subprocess.PIPE, | |
| stdout=open(self._gpg_file, 'wb')) | |
| return gpg | |
| class ChunkedStream(object): | |
| def __init__(self, url, chunk_size): | |
| self._url = url | |
| self._chunk_size = chunk_size | |
| response = requests.get(url, stream=True) | |
| self._stream = response.iter_content(chunk_size=chunk_size) | |
| self._current_chunk = None | |
| def __iter__(self): | |
| for chunk in self._stream: | |
| self._current_chunk = chunk | |
| yield self | |
| def process(self, processer_func): | |
| if self._current_chunk is not None: | |
| processer_func(self._current_chunk) | |
| return self | |
| start_time = time.time() | |
| md5_filter = MD5Filter(md5sum_file) | |
| gpg_filter = GPGFilter(gpg_file, gpg_recipient, keyring_file) | |
| chunked_stream = ChunkedStream(url, chunk_size) | |
| for chunk in chunked_stream: | |
| chunk.process(md5_filter.processor).process(gpg_filter.processor) | |
| md5_filter.save() | |
| gpg_filter.save() | |
| end_time = time.time() | |
| print("Time taken: %s seconds" % str(end_time - start_time)) | |
| print("MD5 checksum: %s" % md5_filter.hash()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment