Skip to content

Instantly share code, notes, and snippets.

@mpapi

mpapi/cached.py

Created May 1, 2011 22:05
Show Gist options
  • Save mpapi/950918 to your computer and use it in GitHub Desktop.
Save mpapi/950918 to your computer and use it in GitHub Desktop.
Download ZIP
cached: Python contextmanager for cached URL fetches
Raw
cached.py
from contextlib import closing, contextmanager
from urllib2 import urlopen
import hashlib
import os
import time
@contextmanager
def cached(url, expires=0, root='/tmp'):
'''
Returns the contents of the URL through a file-based cache. If "expires" is
greater than zero, the contents will be re-fetched from the URL after that
many seconds.
'''
checksum = hashlib.md5(url).hexdigest()
cache_file = os.path.join(root, checksum)
if os.path.exists(cache_file) and time.time() - os.path.getctime(cache_file) < expires:
with open(cache_file, 'r') as cache_fh:
yield cache_fh.read()
else:
with closing(urlopen(url)) as url_fh:
data = url_fh.read()
with open(cache_file, 'w') as cache_fh:
cache_fh.write(data)
yield data
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment