Last active
August 29, 2015 14:04
-
-
Save kobrinartem/e1ff4f3b385544515e50 to your computer and use it in GitHub Desktop.
s3_safe_sync
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
AWS_ACCESS_KEY = 'XXXXXXXXXXX' | |
AWS_SECRET_KEY = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXX' | |
import os | |
import random, string | |
import boto | |
from boto.s3.connection import S3Connection | |
import hashlib | |
from subprocess import call | |
name_bucket = 'pica8' | |
prefix = 'test1' | |
name_key_file = 'key.txt' | |
key_file_s3 = prefix + name_key_file | |
backup_folder = '/Backups/' | |
key_file = backup_folder + name_key_file | |
key_file_s3 = prefix+'/'+name_key_file | |
conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) | |
bucket = conn.get_bucket(name_bucket) | |
#get size of folder or file on s3 | |
#Example: print get_size(bucket, 'test_folder') | |
#Example: print get_size(bucket, 'test_folder/testfile.txt') | |
def get_size(bucket, prefix): | |
size = 0 | |
for key in bucket.list(prefix): | |
size += key.size | |
return size | |
#get dictionary with md5sum of files on bucket on s3 | |
#Example: get_md5_s3(bucket)[u'i-9a4b699c/duplicity-full.20140704T010001Z.vol1.difftar.gz'] | |
def get_md5_s3(bucket, file_s3): | |
bucket_dict = {} | |
rs_keys = bucket.get_all_keys() | |
for key_val in rs_keys: | |
bucket_dict[key_val.name] = key_val.etag.rstrip('"').lstrip('"') | |
try: | |
print "Note: md5sum of key file on s3: "+bucket_dict[file_s3] | |
return bucket_dict[file_s3] | |
except: | |
print "Note: Key file don't exist on s3" | |
#ge md5sum of local file | |
def get_md5(filePath): | |
with open(filePath, 'rb') as fh: | |
m = hashlib.md5() | |
while True: | |
data = fh.read(8192) | |
if not data: | |
break | |
m.update(data) | |
return m.hexdigest() | |
# Create key file | |
# Example: create_key_file('/Backups/key.txt') | |
def create_key_file(path): | |
if not os.path.exists(path): | |
f = open(path, 'w+') | |
f.write('Don\'t remove and change the file \n') | |
f.write(''.join(random.sample(string.digits+string.letters, 48)) + '\n') | |
f.close() | |
print "Note: created file "+path | |
#run command for sync to s3 | |
def sync_to_s3(backup_folder=backup_folder,name_bucket=name_bucket,prefix=prefix): | |
call(["nice", "-n", "15", "/usr/bin/s3cmd", "sync", backup_folder, "s3://"+name_bucket+"/"+prefix+"/"]) | |
print "Note: Sync is done" | |
#try to get md5sum of file on s3 and if local file exist function compare md5sum both of files | |
def match_key_files(file_local, file_s3): | |
try: | |
md5_file_s3 = get_md5_s3(bucket, file_s3) | |
except KeyError: | |
return False | |
if os.path.isfile(file_local): | |
print "Note: md5sum of key file on local server: "+ get_md5(file_local) | |
else: | |
print "Note: key file on local server don't exist " | |
if os.path.isfile(file_local) and get_md5(file_local) == md5_file_s3: | |
print "Note: Key files match" | |
return True | |
else: | |
return False | |
# main function | |
def main(bucket, prefix, key_file_s3, key_file): | |
size_folder = get_size(bucket, prefix) | |
print "size of folder on s3: ", size_folder | |
if size_folder == 0: | |
create_key_file(key_file) | |
sync_to_s3() # run rsync | |
return 0 | |
else: | |
if match_key_files(key_file, key_file_s3): | |
print "Finished checking and running sync" | |
sync_to_s3() # run rsync | |
return 0 | |
else: | |
print "Error: key files don't match for local folder and for s3 folder or folder on s3 is not empty and don't have key file" | |
pass # sent alert | |
main(bucket, prefix, key_file_s3, key_file) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment