Last active
April 7, 2020 07:20
-
-
Save 3096/6bd9a9c532d53adb7b68d78f9fc261a5 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import ftplib | |
import io | |
import os | |
import pprint | |
import sys | |
import time | |
import msgpack | |
DEFAULT_FTP_PORT = 5000 | |
SERVER_MSGPACK_DIR = "/prepo/" | |
LOCAL_MSGPACK_DIR_NAME = "msgpack" | |
LOCAL_EXPORT_DIR_NAME = "export" | |
if __name__ == "__main__": | |
ftp_addr = sys.argv[1] | |
out_dir = sys.argv[2] | |
# may be used as args | |
delete_from_server = True | |
keep_msgpack = True | |
# prep dirs | |
cur_export_dir = os.path.join(out_dir, f"{LOCAL_EXPORT_DIR_NAME}_{time.strftime('%Y%m%d_%H%M%S')}") | |
cur_msgpack_dir = os.path.join(out_dir, f"{LOCAL_MSGPACK_DIR_NAME}_{time.strftime('%Y%m%d_%H%M%S')}") | |
os.mkdir(cur_export_dir) | |
if keep_msgpack: | |
os.mkdir(cur_msgpack_dir) | |
# start | |
ftp = ftplib.FTP() | |
ftp.connect(ftp_addr, DEFAULT_FTP_PORT) | |
print(ftp.getwelcome()) | |
for name, facts in ftp.mlsd(path=SERVER_MSGPACK_DIR): | |
if facts["type"] != "file": | |
continue | |
print(name) | |
# get file | |
cur_buffer = io.BytesIO() | |
file_path = SERVER_MSGPACK_DIR + name | |
ftp.retrbinary(f"RETR {file_path}", cur_buffer.write) | |
cur_msgpack_content = cur_buffer.getbuffer().tobytes() | |
if keep_msgpack: | |
with open(os.path.join(cur_msgpack_dir, name), 'wb') as cur_out_file: | |
cur_out_file.write(cur_msgpack_content) | |
# try with removing tail bytes as it seems like sometimes there's garbage in the end | |
cur_dict = None | |
remove_tail_len = 0 | |
while cur_dict is None: | |
try: | |
cur_dict = msgpack.unpackb(cur_msgpack_content[:len(cur_msgpack_content) - remove_tail_len]) | |
except (msgpack.exceptions.ExtraData, ValueError): # | |
remove_tail_len += 1 | |
# write export | |
print(cur_dict) | |
with open(os.path.join(cur_export_dir, os.path.splitext(name)[0] + ".py"), 'w', | |
encoding='UTF-8') as cur_export_file: | |
pprint.pprint(cur_dict, stream=cur_export_file) | |
# clean up | |
if delete_from_server: | |
ftp.delete(file_path) | |
ftp.quit() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment