Created
February 22, 2014 11:09
-
-
Save gauravtoshniwal1989/9152171 to your computer and use it in GitHub Desktop.
Management command to upload only files that have been changed to S3
The S3 bucket is assumed to be like:
Bucket
|-static
|-media
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from django.core.management.base import BaseCommand | |
| from django.conf import settings | |
| from storages.backends.s3boto import S3BotoStorage | |
| from boto.s3.connection import S3Connection | |
| from boto.s3.key import Key | |
| import os | |
| from hashlib import md5 | |
| from subprocess import call | |
| # Assuming the project structure is like this: | |
| # BASE_DIR | |
| # |- .git | |
| # |- PROJECT_DIR | |
| # | |- APPS_DIR | |
| # | |- settings.py | |
| # | |- STATIC_DIR (static) | |
| # |-STATIC_ROOT(static) | |
| # |-VIRTUALENV_DIR | |
| # In the settings file: | |
| # PROJECT_PATH = os.path.dirname(os.path.abspath(__file__)) | |
| # BASE_DIR = os.path.dirname(os.path.dirname(__file__)) | |
| class Command(BaseCommand): | |
| args = '' | |
| help = 'Closes the specified poll for voting' | |
| can_import_settings = True | |
| def __init__(self,*args, **kwargs): | |
| super(Command, self).__init__(*args, **kwargs) | |
| conn = S3Connection(settings.AWS_ACCESS_KEY_ID,settings.AWS_SECRET_ACCESS_KEY) | |
| self.bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME) | |
| def handle(self, *args, **options): | |
| bucket_dict = {} | |
| for k in self.bucket.get_all_keys(prefix="static"): | |
| # k.etag returns u'"xxxxxxxxxxx30bb87cbdac24f55xxxx"' hence strip the quotes from left and right | |
| bucket_dict[k.name] = k.etag.rstrip('"').lstrip('"') | |
| # pass | |
| static_dir = os.path.join(settings.PROJECT_PATH,'static') | |
| os.chdir(settings.BASE_DIR) # Git root | |
| for root, dirs, files in os.walk(static_dir): | |
| for f in files: | |
| fullpath = os.path.join(root,f) | |
| key = fullpath.replace(settings.PROJECT_PATH+"/","") | |
| if bucket_dict.get(key,None) == md5(open(fullpath).read()).hexdigest(): | |
| # if the key exists on the server side already and its hash is equal to the hash of local file | |
| # go to the next file | |
| continue | |
| else: | |
| # Check if the key exists | |
| if self.bucket.get_key(key) is None: | |
| # Check if the key is ignored in the local repository | |
| if not fullpath.endswith(".gitignore") or bool(call(["git","check-ignore","%s"%(fullpath)])): | |
| # If not (that is check-ignore returns True | |
| # Create the key | |
| key_to_upload_to = self.bucket.new_key(key) | |
| else: | |
| # Else go to the next file | |
| continue | |
| else: | |
| # Get the existing key if it already exists | |
| key_to_upload_to = self.bucket.get_key(key) | |
| uploaded_bytes = key_to_upload_to.set_contents_from_filename(fullpath) | |
| print ("Uploaded %s bytes of %s to S3"%(uploaded_bytes,fullpath)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment