Created
January 5, 2020 10:10
-
-
Save s-m-e/a930a49716ba65cfdc49d70d001818f1 to your computer and use it in GitHub Desktop.
Download & updating "www_logs" from Hetzner's web hosting service (for cron jobs)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# Python 3 | |
import datetime | |
import gzip | |
import os | |
import stat | |
import pysftp # pip install pysftp==0.2.8 | |
USER = 'username' | |
PWD = 'password' | |
HOST = 'www.domain.xy' | |
TARGET = '/path/to/data' | |
def _parse_remote(file_list): | |
return { | |
attr.filename: { | |
'fn': attr.filename, | |
'mtime': attr.st_mtime, | |
'size': attr.st_size, | |
} for attr in file_list | |
} | |
def _parse_local(file_list, local_path): | |
return { | |
fn: { | |
'fn': fn, | |
'mtime': attr.st_mtime, | |
'size': attr.st_size, | |
} for fn, attr in zip(file_list, (os.stat(os.path.join(local_path, item)) for item in file_list)) | |
} | |
def _update(sftp_handle, local_path): | |
remote_dict = _parse_remote(sftp_handle.listdir_attr()) | |
local_dict = _parse_local(os.listdir(local_path), local_path) | |
missing = remote_dict.keys() - local_dict.keys() | |
present = remote_dict.keys() & local_dict.keys() | |
updated = set() | |
for fn in present: | |
if local_dict[fn]['mtime'] != remote_dict[fn]['mtime'] or local_dict[fn]['size'] != remote_dict[fn]['size']: | |
updated.add(fn) | |
print(missing) | |
for fn in missing: | |
sftp_handle.get( | |
fn, | |
localpath = os.path.join(local_path, fn), | |
preserve_mtime = True | |
) | |
print(updated) | |
for fn in updated: | |
os.unlink(os.path.join(local_path, fn)) | |
sftp_handle.get( | |
fn, | |
localpath = os.path.join(local_path, fn), | |
preserve_mtime = True | |
) | |
def main(): | |
with pysftp.Connection(HOST, username = USER, password = PWD) as sftp: | |
with sftp.cd('www_logs'): | |
for domain_handle in sftp.listdir_attr(): | |
domain_path = os.path.join(TARGET, domain_handle.filename) | |
os.makedirs(domain_path, exist_ok = True) | |
print(domain_handle.filename) | |
with sftp.cd(domain_handle.filename): | |
_update(sftp, domain_path) | |
if __name__ == '__main__': | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment