Created
June 25, 2012 04:39
-
-
Save bobveznat/2986559 to your computer and use it in GitHub Desktop.
NVZ uploader
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto | |
import cStringIO | |
import gzip | |
import hashlib | |
import logging | |
import os | |
import sys | |
from boto.s3.connection import S3Connection | |
from boto.s3.key import Key | |
dir_name = 'nicholas.veznat.com' | |
base_path = 'nicholas.veznat.com' | |
c = S3Connection() | |
b = c.create_bucket(dir_name) | |
content_types = { | |
'html': 'text/html', | |
'css': 'text/css', | |
'js': 'text/javascript', | |
} | |
boto.set_file_logger('boto', 'nvz_upload.out', level=logging.DEBUG) | |
def process_dir(dir_path): | |
for filename in os.listdir(dir_path): | |
complete_path = os.path.join(dir_path, filename) | |
if os.path.isdir(complete_path): | |
complete_path = complete_path + '/' | |
process_dir(complete_path) | |
file_id = None | |
continue | |
else: | |
hasher = hashlib.new('sha256') | |
with open(complete_path, 'r') as f: | |
file_contents = f.read() | |
file_id = hashlib.new('sha256', file_contents).hexdigest() | |
key_name = complete_path.replace(base_path, '') | |
existing_key = b.get_key(key_name) | |
if existing_key: | |
old_file_id = existing_key.get_metadata('file_id') | |
if old_file_id == file_id: | |
print 'hashes match', complete_path | |
continue | |
headers={'Cache-Control': 'max-age=900'} | |
if complete_path[-3:] not in ('jpg', 'png', 'gif'): | |
compressed_file = cStringIO.StringIO() | |
zipper = gzip.GzipFile(fileobj=compressed_file, mode='wb') | |
zipper.write(file_contents) | |
zipper.close() | |
compressed_file.seek(0) | |
file_contents = compressed_file.read() | |
headers['Content-Encoding'] = 'gzip' | |
extension = complete_path[complete_path.rfind('.') + 1:] | |
headers['Content-Type'] = content_types[extension] | |
compressed = True | |
else: | |
compressed = False | |
def start_progress(): | |
sys.stderr.write('\n') | |
def upload_progress(bytes_done, bytes_total): | |
compressed_str = '' | |
if compressed: | |
compressed_str = '(compressed)' | |
sys.stderr.write('\r%s%s: %s / %s' % ( | |
complete_path, compressed_str, bytes_done, bytes_total)) | |
sys.stderr.flush() | |
def finish_progress(new_key): | |
sys.stderr.write('\nFinished: %r\n' % (new_key,)) | |
start_progress() | |
new_key = Key(b, key_name) | |
headers['x-amz-meta-file_id'] = file_id | |
if compressed: | |
new_key.set_contents_from_string( | |
file_contents, | |
policy='public-read', | |
headers=headers, | |
cb=upload_progress, | |
num_cb=20, | |
) | |
else: | |
with open(complete_path, 'r') as f: | |
new_key.set_contents_from_file( | |
f, | |
policy='public-read', | |
headers=headers, | |
cb=upload_progress, | |
num_cb=20 | |
) | |
finish_progress(new_key) | |
process_dir(dir_name) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment