Created
October 5, 2010 16:00
-
-
Save jehiah/611787 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pylibmc | |
import Queue | |
import logging | |
import functools | |
""" | |
This is a transparent pool library that wraps a pylibmc client | |
from MemcachePool import mc | |
mc.get(key) | |
mc.set(key, value) | |
""" | |
import settings | |
def servers(): | |
return settings.get('memcached') | |
server_pool = Queue.Queue(maxsize=50) # of max cached connections; not max # of open connections | |
def newConnection(): | |
"""setup a new memcached connection""" | |
if callable(servers): | |
return pylibmc.Client(servers()) | |
else: | |
return pylibmc.Client(servers) | |
def pool(method): | |
""" | |
this decorator implements a self growing pool and returns | |
connections to it | |
If there is not a connection available in the pool create one | |
If we can't return the connection to the pool, drop it | |
""" | |
@functools.wraps(method) | |
def wrapper(*args, **kwargs): | |
try: | |
conn = server_pool.get_nowait() | |
except Queue.Empty: | |
conn = newConnection() | |
val = None | |
try: | |
## method is wrapperFunction in PoolClient | |
key = args[0] | |
if not isinstance(key, (str, list, tuple)): | |
logging.warning('%r must be _utf8' % key) | |
try: | |
# convert to utf8 on the fly since memcached chokes on unicode objects | |
args = [_utf8(arg) for arg in args] | |
val = method(conn, *args, **kwargs) | |
except: | |
logging.exception('failed memcached call %s %s' % (str(args), str(kwargs))) | |
val = None | |
finally: | |
try: | |
server_pool.put_nowait(conn) | |
except Queue.Full: | |
pass | |
return val | |
return wrapper | |
class PoolClient(object): | |
""" | |
This is a wrapper class that passes all methods on to the actual client | |
getting a connection from the pool first, and returning it there after | |
""" | |
def __getattribute__(self,name): | |
@pool | |
def wrapperFunction(*args, **kwargs): | |
## called by acquireAndRelease; | |
## args[0] == pylibmc.Client | |
## name == method to call | |
## args[1:] the original method arguments | |
## debug: | |
# print "memcache", name, args[1] | |
return getattr(args[0], name)(*args[1:], **kwargs) | |
return wrapperFunction | |
mc = PoolClient() | |
def _utf8(s): | |
if isinstance(s, unicode): | |
return s.encode("utf-8") | |
return s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment