Created
January 20, 2014 12:00
-
-
Save aliles/8518861 to your computer and use it in GitHub Desktop.
API endpoint templating using Jinja2 templates and Twisted.Web.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
"""API endpoint templating using Jinja2. | |
Rendering of Jinja2 templates, with external actions, using Twisted.Web. | |
Combining external actions for Jinja2 [1] with Twisted's deferreds inside the | |
Twisted's web framework. Jinja2 templates are iteratively processed [2] to | |
render an API response [3]. The template prepares, executes and formats the | |
response from an HTTP request. | |
When run, the application looks up an email address provided as a GET paramater | |
on Have I Been Pwned, returning a text response with the names of datasets that | |
have leaked private information registered with the address. For example: | |
http://localhost:8080/pwned?email=foo@foo.com | |
This example will return the text "Adobe Gawker Stratfor". | |
[1] https://gist.github.com/aliles/8417271 | |
[2] https://gist.github.com/aliles/8454244 | |
[3] https://gist.github.com/aliles/8471660 | |
[4] https://haveibeenpwned.com/ | |
""" | |
import os | |
import random | |
import string | |
import sys | |
from twisted.internet import defer, reactor, task, threads | |
from twisted.python import log | |
from twisted.web.resource import Resource | |
from twisted.web.server import NOT_DONE_YET, Site | |
import jinja2 | |
import jinja2.ext | |
import requests | |
class HTTPExtension(jinja2.ext.Extension): | |
"""HTTP method extension for Jinja2 templates. | |
{% http METHOD URL into NAME as TYPE %} | |
Defers control out of template to the calling scope for exection of HTTP | |
request. A Mutable object is also passed out to allow external scope to | |
pass result back into template context. | |
""" | |
tags = set(['http']) | |
mutable = type('Mutable', (object,), {'__slots__': ('value',)}) | |
def __init__(self, environment): | |
super(HTTPExtension, self).__init__(environment) | |
def parse(self, parser): | |
proxy = "_" + "".join(random.choice(string.letters) for i in range(16)) | |
lineno = parser.stream.next().lineno | |
method = parser.stream.expect('name').value | |
url = parser.parse_expression() | |
_ = parser.stream.expect('name:into') | |
target = parser.stream.next().value | |
_ = parser.stream.expect('name:as') | |
content = parser.stream.expect('name').value | |
return [ | |
# Creates new mutable proxy object inside template context. | |
jinja2.nodes.Assign( | |
jinja2.nodes.Name(proxy, 'store'), | |
jinja2.nodes.Call( | |
self.attr('mutable', lineno=lineno), | |
[], [], None, None)).set_lineno(lineno), | |
# Pass control out to calling scope. | |
jinja2.nodes.CallBlock( | |
self.call_method('action', [ | |
jinja2.nodes.Name(proxy, 'load'), | |
jinja2.nodes.Const(method), | |
url, | |
jinja2.nodes.Const(content) | |
]), [], [], []).set_lineno(lineno), | |
# Retrives value and assigns it to final result target name. | |
jinja2.nodes.Assign( | |
jinja2.nodes.Name(target, 'store'), | |
jinja2.nodes.Getattr( | |
jinja2.nodes.Name(proxy, 'load'), 'value', 'load') | |
).set_lineno(lineno) | |
] | |
def action(self, proxy, method, url, content, caller): | |
"""Defer HTTP request and processed to background thread""" | |
def closure(): | |
request = getattr(requests, method) | |
response = request(url) | |
data = getattr(response, content) | |
if callable(data): | |
data = data() | |
proxy.value = data | |
deferred = threads.deferToThread(closure) | |
return deferred | |
class Endpoint(Resource): | |
"""API endpoint web resource for Twisted web applications. | |
Uses Jinja2 as a templating engine for API endpoints, iteratively | |
processing the template. | |
""" | |
isLeaf = True | |
def __init__(self, source, environment): | |
self.source = source | |
self.environment = environment | |
self.template = environment.from_string(source) | |
def render_GET(self, request): | |
"""Process GET requests for endpoint resource. | |
Iteratively processes the Jinja2 template, returning the templates | |
output in chucnked responses to the client. | |
""" | |
deferred = self._render(self.template, request) | |
deferred.addCallback(lambda _: request.finish()) | |
deferred.addErrback(log.err) | |
return NOT_DONE_YET | |
def _iterate(self, iterator, request): | |
"""Consume next value from iterator | |
If the next value is not a deferred, process the value with process() and | |
return a new deferred for the next value. | |
If the next value is a deferred add process() as a callback followed by an | |
anonymous function for a new deferred to process the next value. | |
""" | |
try: | |
value = iterator.next() | |
if not isinstance(value, defer.Deferred): | |
request.write(value.encode('utf8')) | |
return task.deferLater(reactor, 0, self._iterate, iterator, request) | |
else: | |
value.addCallback(lambda _: task.deferLater(reactor, 0, self._iterate, iterator, request)) | |
return value | |
except StopIteration: | |
return | |
def _render(self, template, request): | |
"""Create new iterator for rendering template with context""" | |
iterator = template.generate({'request': request}) | |
deferred = task.deferLater(reactor, 0, self._iterate, iter(iterator), request) | |
return deferred | |
if __name__ == '__main__': | |
# Example of Jinja2 template code. | |
source = """{% set email = request.args.email[0] %} | |
{% set url = 'http://haveibeenpwned.com/api/breachedaccount/' + email|urlencode %} | |
{% http get url into result as json %} | |
{% for site in result %} | |
{{ site }} | |
{% endfor %} | |
""" | |
# Create Environment, with the HTTP extension, and load sample template | |
# code. | |
env = jinja2.Environment(trim_blocks=True, lstrip_blocks=True, | |
extensions=[HTTPExtension]) | |
# Construct resource structure for Twisted web application. | |
root = Resource() | |
root.putChild("pwned", Endpoint(source, env)) | |
factory = Site(root) | |
# Listen on port 8080 for HTTP requests and start the Twisted reactor. | |
reactor.listenTCP(8080, factory) | |
reactor.run() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment