-
-
Save nhoad/8b861a52ffe6c9c088d3 to your computer and use it in GitHub Desktop.
jake!
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python2 | |
import argparse | |
import contextlib | |
import logging | |
import re | |
import os | |
import sys | |
import types | |
from StringIO import StringIO | |
from parsimonious.grammar import Grammar | |
log = logging.getLogger(__name__) | |
# FIXME: add escaped grave | |
STR_GRAMMAR = r''' | |
script = block* | |
block = code / text | |
code = grave text grave | |
text = ~"[A-Z 0-9\s\!\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\]\^\"\_\{\|\}\~\\\\]+"i | |
grave = "`" | |
''' | |
GRAMMAR = Grammar(STR_GRAMMAR) | |
class Config: | |
comment_char = '#' | |
dump_lookups = True | |
dynamic_config_callable_name = 'jake_get_config' | |
namespace_template_separator = '~~~' | |
verbose = False | |
strip_extra_whitespace = True | |
lint = False | |
dump_debug = False | |
@contextlib.contextmanager | |
def redir_stdout(): | |
old = sys.stdout | |
new = StringIO() | |
sys.stdout = new | |
yield new | |
sys.stdout = old | |
class Template(object): | |
def __init__(self, template_path, namespace=None): | |
self.namespace = namespace or Namespace() | |
template = open(template_path).read() | |
try: | |
self.namespace_str, self.template = template.split( | |
'\n{}\n'.format(Config.namespace_template_separator)) | |
except ValueError: | |
self.template = template | |
self.namespace_str = '' | |
@classmethod | |
def render(cls, name): | |
template = cls(name) | |
template.execute_namespace() | |
return template.render_template() | |
def run(self): | |
self.execute_namespace() | |
self.execute_template() | |
def execute_namespace(self): | |
if self.namespace_str: | |
namespace = self.namespace | |
exec self.namespace_str in namespace.module_namespace | |
if namespace.config is None and Config.dynamic_config_callable_name in self.namespace.module_namespace: | |
config_callable = self.namespace.module_namespace[Config.dynamic_config_callable_name] | |
namespace.config = config_callable() | |
def create_script(self): | |
script = StringIO() | |
for is_code, text in self.transform(): | |
if is_code: | |
# gross approximation for printing values when necessary | |
if len(text.split('\n')) == 1: | |
script.write('\nprint ({} or ""),\n'.format(text)) | |
else: | |
script.write(text) | |
else: | |
script.write('\nprint """{}""",\n'.format(text)) | |
return script.getvalue() | |
def execute_script(self, script): | |
if Config.dump_debug: | |
sys.stderr.write(script) | |
sys.stderr.flush() | |
exec script in self.namespace | |
def lint_script(self, script): | |
script = self.namespace_str + script | |
import subprocess | |
try: | |
# we ignore all E-class warnings because they're non-fatal, | |
# style-based things. | |
proc = subprocess.Popen( | |
['flake8', '--ignore', 'E', '-'], | |
stdin=subprocess.PIPE, stdout=subprocess.PIPE, | |
stderr=subprocess.STDOUT | |
) | |
except OSError as e: | |
print('# Could not lint source: {}\n'.format(str(e))) | |
return | |
stdout, stderr = proc.communicate(script) | |
if stdout: | |
for line in stdout.splitlines(): | |
m = re.match(r".* undefined name '(.+)'", line) | |
if m: | |
name, = m.groups(0) | |
if name in self.namespace.lookups: | |
continue | |
else: | |
try: | |
self.namespace[name] | |
except KeyError: | |
self.namespace.lookups.pop(-1) | |
else: | |
self.namespace.lookups.pop(-1) | |
continue | |
name, line = line.split(':', 1) | |
print('# {}'.format(line)) | |
print('') | |
def render_template(self): | |
self.namespace.freeze() | |
script = self.create_script() | |
with redir_stdout() as config: | |
if Config.lint: | |
self.lint_script(script) | |
self.execute_script(script) | |
output = config.getvalue() | |
pretty_output = self.dump_pretty_output(output) | |
return pretty_output | |
def dump_pretty_output(self, output): | |
pretty_output = StringIO() | |
# prefix all the dynamic values that were looked up as comments at the | |
# top of the file. | |
if Config.dump_lookups: | |
for name in sorted(self.namespace.lookups): | |
value = self.namespace[name] | |
if isinstance(value, types.FunctionType): | |
continue | |
pretty_output.write('%s %s = %s\n' % (Config.comment_char, name, value)) | |
pretty_output.write('\n') | |
if Config.strip_extra_whitespace: | |
output = re.sub(r' \n', '\n', output) | |
output = re.sub(r' ', ' ', output) | |
output = re.sub(r'\n\n+', '\n\n', output) | |
pretty_output.write(output) | |
return pretty_output.getvalue() | |
def transform(self): | |
parsed = GRAMMAR.parse(self.template) | |
for block in parsed: | |
assert len(block.children) == 1 | |
expr_name = block.children[0].expr_name | |
text = block.text | |
if expr_name == 'code': | |
yield True, text[1:-1] | |
else: | |
yield False, text | |
class Namespace(dict): | |
def __init__(self, config=None): | |
self.config = config | |
self.module_namespace = {} | |
self.local_namespace = {} | |
self._frozen = False | |
self.lookups = [] | |
def freeze(self): | |
self._frozen = True | |
def __getitem__(self, key): | |
self.lookups.append(key) | |
try: | |
if isinstance(self.config, dict): | |
value = self.config[key] | |
else: | |
value = getattr(self.config, key) | |
except (AttributeError, KeyError) as e: | |
if self.config is not None: | |
log.debug("Could not retrieve %s from config object", key) | |
else: | |
return value | |
if key in self.module_namespace: | |
value = self.module_namespace[key] | |
elif key in self.local_namespace: | |
value = self.local_namespace[key] | |
else: | |
raise KeyError(key) | |
return value | |
def __setitem__(self, key, value): | |
if self._frozen: | |
self.module_namespace[key] = value | |
else: | |
self.local_namespace[key] = value | |
def parse_args(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument('templates', metavar='template', nargs='+', | |
help='Paths to templates to generate.') | |
parser.add_argument('--no-dump-lookups', action='store_false', | |
default=True, | |
help="If given, don't dump variable lookups to beginning of config file as comments") | |
parser.add_argument('--verbose', action='store_true', | |
help="Debugging output. Does not affect template output") | |
parser.add_argument('--namespace-template-separator', type=str, | |
default=Config.namespace_template_separator, | |
help="Change separator between namespace and template separator.") | |
parser.add_argument('--comment-char', type=str, | |
default=Config.comment_char, | |
help="Change comment character to use for dumped variable name lookups.") | |
parser.add_argument('--dynamic-config-callable-name', type=str, | |
default=Config.dynamic_config_callable_name, | |
help="Change name of the callable to look for for retrieving dynamic configuration.") | |
parser.add_argument('--output-to-dir', | |
default=None, | |
help="Rather than write to stdout, write templates to given directory. Filenames will be that of the template") | |
parser.add_argument('--no-strip-extra-whitespace', | |
action='store_false', | |
default=Config.strip_extra_whitespace, | |
help="Don't remove trailing whitespace, extra newlines and double spaces.") | |
parser.add_argument('--lint', | |
action='store_true', | |
default=Config.lint, | |
help="Lint script before it is executed. Warnings will be inserted toward the top of the file") | |
parser.add_argument('--dump-debug', | |
action='store_true', | |
default=Config.dump_debug, | |
help="Dump script that is about to be executed.") | |
return parser.parse_args() | |
def main(): | |
options = parse_args() | |
Config.comment_char = options.comment_char | |
Config.dynamic_config_callable_name = options.dynamic_config_callable_name | |
Config.dump_lookups = options.no_dump_lookups | |
Config.namespace_template_separator = options.namespace_template_separator | |
Config.verbose = options.verbose | |
Config.strip_extra_whitespace = options.no_strip_extra_whitespace | |
Config.lint = options.lint | |
Config.dump_debug = options.dump_debug | |
for filename in options.templates: | |
output = Template.render(filename) | |
if options.output_to_dir: | |
output_path = os.path.join(options.output_to_dir, os.path.basename(filename)) | |
with open(output_path, 'w') as f: | |
f.write(output) | |
else: | |
sys.stdout.write(output) | |
if __name__ == '__main__': | |
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
$ ./bin/jake examples/squid.conf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class jake_get_config: | |
direct_proxy_enabled = False | |
transparent_proxy_enabled = False | |
smp_enabled = True | |
cache_mem = 256000 | |
disk_caches = [] | |
ssl_bump_server_first = False | |
ssl_bump_enabled = False | |
import subprocess | |
def directive(command, *args, **kwargs): | |
text = '%s %s %s' % (command, ' '.join(map(str, args)), ' '.join('%s=%s' % kv for kv in kwargs.iteritems())) | |
print text | |
def http_port(port, *args, **kwargs): | |
args = set(args) | |
if kwargs.pop('ssl_bump', False): | |
args.add('ssl-bump') | |
kwargs['generate-host-certificates'] = 'on' | |
kwargs['cert'] = '/path/to/cert' | |
kwargs['key'] = '/path/to/key' | |
https = kwargs.pop('https', False) | |
return directive('http_port' if not https else 'https_port', port, *args, **kwargs) | |
def https_port(port, *args, **kwargs): | |
return http_port(port, https=True, *args, **kwargs) | |
def workers(enabled): | |
if enabled: | |
cpu_count = len(subprocess.check_output(['grep', 'processor', '/proc/cpuinfo']).splitlines()) | |
return directive('workers', max(cpu_count/2, 1)) | |
def cache_dir(config): | |
if not config.enabled: | |
return | |
args = [] | |
if config.cache_type in ('ufs', 'aufs', 'diskd'): | |
args.append('/var/cache/ufs/%d' % config.pk) | |
args += ['%s' % (config.cache_size / 1024), '16', '256'] | |
else: | |
args.append('/var/cache/rock/%d' % config.pk) | |
args.append('%s' % (config.cache_size / 1024)) | |
if config.no_store: | |
args.append('no-store') | |
kwargs = { | |
'min-size': config.min_value_object_size, | |
'min-size': config.max_value_object_size, | |
} | |
return directive('cache_dir', config.cache_type, *args, **kwargs) | |
def ssl_bump(server_first, enabled): | |
if not enabled: | |
return | |
if server_first: | |
mode = 'server-first' | |
else: | |
mode = 'client-first' | |
return directive('ssl_bump', mode) | |
~~~ | |
acl localnet src 10.0.0.0/8 | |
acl localnet src 172.16.0.0/12 | |
acl localnet src 192.168.0.0/16 | |
acl localnet src fc00::/7 | |
acl localnet src fe80::/10 | |
acl SSL_ports port 443 | |
acl Safe_ports port 80 | |
acl Safe_ports port 443 | |
acl CONNECT method CONNECT | |
http_access deny !Safe_ports | |
http_access deny CONNECT !SSL_ports | |
http_access allow localnet | |
http_access allow localhost | |
http_access allow localhost manager | |
http_access deny manager | |
# INSERT RULES HERE | |
http_access deny all | |
cache_effective_user proxy | |
coredump_dir /var/cache/squid | |
` | |
if direct_proxy_enabled: | |
http_port(direct_proxy_port, ssl_bump=direct_ssl) | |
if transparent_proxy_enabled: | |
http_port(65080, 'intercept') | |
if transparent_ssl: | |
https_port(65443, 'intercept', ssl_bump=transparent_ssl) | |
` | |
`workers(smp_enabled)` | |
cache_mem `cache_mem` bytes | |
` | |
for dir in disk_caches: | |
if smp_enabled and dir.cache_type != 'rock': | |
continue | |
cache_dir(dir) | |
` | |
url_rewriter_program /usr/libexec/url_rewriter | |
url_rewrite_access allow all | |
`ssl_bump(ssl_bump_server_first, enabled=ssl_bump_enabled)` | |
refresh_pattern ^ftp: 1440 20% 10080 | |
refresh_pattern ^gopher: 1440 0% 1440 | |
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 | |
refresh_pattern . 0 20% 4320 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# cache_mem = 256000 | |
# direct_proxy_enabled = False | |
# disk_caches = [] | |
# smp_enabled = True | |
# ssl_bump_enabled = False | |
# ssl_bump_server_first = False | |
# transparent_proxy_enabled = False | |
acl localnet src 10.0.0.0/8 | |
acl localnet src 172.16.0.0/12 | |
acl localnet src 192.168.0.0/16 | |
acl localnet src fc00::/7 | |
acl localnet src fe80::/10 | |
acl SSL_ports port 443 | |
acl Safe_ports port 80 | |
acl Safe_ports port 443 | |
acl CONNECT method CONNECT | |
http_access deny !Safe_ports | |
http_access deny CONNECT !SSL_ports | |
http_access allow localnet | |
http_access allow localhost | |
http_access allow localhost manager | |
http_access deny manager | |
# INSERT RULES HERE | |
http_access deny all | |
cache_effective_user proxy | |
coredump_dir /var/cache/squid | |
workers 4 | |
cache_mem 256000 bytes | |
url_rewriter_program /usr/libexec/url_rewriter | |
url_rewrite_access allow all | |
refresh_pattern ^ftp: 1440 20% 10080 | |
refresh_pattern ^gopher: 1440 0% 1440 | |
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 | |
refresh_pattern . 0 20% 4320 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment