Last active
December 10, 2016 10:02
-
-
Save erichiggins/fd125afff7edf82bd7f7 to your computer and use it in GitHub Desktop.
Converts a directory of Python packages installed by pip into importable zip files for use on Google App Engine.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
""" | |
Convert a directory of pip-installed libraries into zip-imports for use on GAE. | |
Usage: | |
pip install -U <library_name> | |
pip freeze > requirements_dev.txt | |
pip install -U --egg --target <path_to_zips> -r requirements_dev.txt | |
python zipimportify.py <path_to_zips> | |
cd <path_to_zips> | |
ls | grep -v '\.zip' | xargs rm -rf | |
""" | |
import argparse | |
import json | |
import os | |
import subprocess | |
import sys | |
PIP_INFO_SUFFIX = '-info' | |
TOP_LEVEL = 'top_level.txt' | |
METADATA = 'metadata.json' | |
parser = argparse.ArgumentParser(description='Zip up and version pip-installed libraries.') | |
parser.add_argument('path', type=str) | |
def top_level(path): | |
with open(os.path.join(path, TOP_LEVEL), 'r') as fp: | |
line = fp.readline().strip() | |
# Note(eric): Packages like PyYaml have top_level lines like "__yaml" that should be skipped. | |
while line.startswith('_'): | |
line = fp.readline().strip() | |
return line | |
def version(path): | |
with open(os.path.join(path, METADATA), 'r') as fp: | |
metadata = json.load(fp) | |
return metadata['version'] | |
def zip_dir(dir_to_zip, filename, cwd=None): | |
return subprocess.Popen(['zip', '-qr0', filename, dir_to_zip], cwd=cwd, universal_newlines=True) | |
def zip_files(files_to_zip, filename, cwd=None): | |
return subprocess.Popen(['zip', '-qr0', filename, '.', '-i', files_to_zip], cwd=cwd, universal_newlines=True) | |
def main(args): | |
# Everything in the directory. | |
all_files = set(os.listdir(args.path)) | |
# Everything that ends with "-info". | |
info_names = set([x for x in all_files if x.endswith(PIP_INFO_SUFFIX)]) | |
print len(info_names), '-info files:' | |
print info_names | |
# Create paths by prepending the directory. | |
info_paths = set([os.path.join(args.path, x) for x in info_names]) | |
print len(info_paths), 'info paths:' | |
print info_paths | |
# Every package version that matches a subdirectory name. | |
versions = {top_level(x): version(x) for x in info_paths} | |
print len(versions.keys()), 'versions:' | |
print versions | |
packages = set([x for x in versions.iterkeys() if os.path.isdir(os.path.join(args.path, x))]) | |
print len(packages), 'packages:' | |
print packages | |
non_dir_packages = set(versions.iterkeys()) - packages | |
print len(non_dir_packages), 'non_dir_packages:' | |
print non_dir_packages | |
# Create a zip for each package. | |
print 'zip paths:' | |
for path in packages: | |
zip_path = path + '-' + versions[path] + '.zip' | |
print path, '>', zip_path | |
zip_dir(path, zip_path, cwd=args.path) | |
# Create a zip for each non-directory package. | |
print 'zip paths:' | |
for path in non_dir_packages: | |
zip_path = path + '-' + versions[path] + '.zip' | |
path = path + '.py*' | |
print path, '>', zip_path | |
zip_files(path, zip_path, cwd=args.path) | |
if __name__ == '__main__': | |
args = parser.parse_args() | |
main(args) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment