Skip to content

Instantly share code, notes, and snippets.

@helmetwearer
Created February 4, 2016 16:58
Show Gist options
  • Save helmetwearer/71b1f0aae329e0949450 to your computer and use it in GitHub Desktop.
Save helmetwearer/71b1f0aae329e0949450 to your computer and use it in GitHub Desktop.
class S3PickleAndLocalCache(object):
def __init__(self, local_on=True):
self.bucket_name = settings.S3_CACHE_BUCKET
self.conn = boto.s3.connection.S3Connection(
settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY
)
try:
self.bucket = self.conn.get_bucket(self.bucket_name, validate=True)
except boto.exception.S3ResponseError:
self.bucket = self.conn.create_bucket(self.bucket_name)
self.local_on = local_on
if self.local_on:
self.local_cache = cache.get_cache('filebased')
def clean_key(self, key):
return key.replace('/','-').replace(' ', '-')
def get(self, key):
# try local first because network calls are expensive
if self.local_on:
k = self.local_cache.get(self.clean_key(key))
if k is not None:
return k
# try s3 to see if another machine has
k = self.bucket.get_key(key)
if k is None:
return None
try:
val = pickle.loads(k.get_contents_as_string())
except Exception as e:
print e
print '-----'
print '-----'
print '-----'
print '-----'
print 'cache error, retrying'
print '-----'
print '-----'
print '-----'
print '-----'
sleep(5)
try:
k = self.bucket.get_key(key)
val = pickle.loads(k.get_contents_as_string())
print 'success on retry'
except Exception as e:
print 'failed retry'
return None
# set to local to avoid further network calls
if self.local_on:
self.local_cache.set(self.clean_key(key), val)
return val
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment