Switch to a Python API to write nginx confs

dploi_fabric/nginx.py

@@ -1,26 +1,120 @@
 import StringIO
+from django.utils.text import slugify
 from fabric.decorators import task
 from fabric.api import run, env, put
+from dploi_fabric.toolbox.nginx_dsl import Nginx, prettify
 
-from dploi_fabric.toolbox.template import render_template
 from dploi_fabric.utils import config
 
 
 @task(alias="reload")
 def reload_nginx():
     run('sudo /etc/init.d/nginx reload')
-
-@task
-def update_config_file():
-    output = ""
-    template_name = 'templates/nginx/nginx.conf'
+
+def _default_proxy_pass_config(section, upstream_name, ssl):
+    section.config('proxy_pass', 'http://%s' % upstream_name)
+    section.config('proxy_redirect', 'off')
+    section.config('proxy_set_header', 'Host', '$host')
+    section.config('proxy_set_header', 'X-Real-IP', '$remote_addr')
+    section.config('proxy_set_header', 'X-Forwarded-For', '$proxy_add_x_forwarded_for')
+    if ssl:
+        section.config('proxy_set_header', 'X-Forwarded-Protocol', 'https')
+        section.config('proxy_set_header', 'X-Forwarded-SSL', 'on')
+    else:
+        section.config('proxy_set_header', 'X-Forwarded-Protocol', 'http')
+        section.config('proxy_set_header', 'X-Forwarded-SSL', 'off')
+    section.config('client_body_buffer_size', '128k')
+    section.config('proxy_connect_timeout', 90)
+    section.config('proxy_send_timeout', 90)
+    section.config('proxy_read_timeout', 90)
+    section.config('proxy_buffer_size', '4k')
+    section.config('proxy_buffers', 4, '32k')
+    section.config('proxy_busy_buffers_size', '64k')
+    section.config('proxy_temp_file_writer_size', '64k')
+
+def render_config():
+    full = []
     for site, site_config in config.sites.items():
-        context_dict = site_config
-        context_dict.update({
-            'domains': " ".join(site_config.deployment.get("domains")[site]),
-            'www_processes': [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"],
-        })
+        conf = Nginx()
+        domains = site_config.deployment.get("domains")[site]
+        upstream_name = slugify(u" ".join(domains))
+        with conf.section('upstream', upstream_name) as upstream:
+            for process in [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"]:
+                upstream.config('server', 'unix:%s' % process['socket'], 'fail_timeout=0')
+
+        deployment = site_config['deployment']
+
+        listen = '%s:%s' % (deployment['bind_ip'], '443' if deployment['ssl'] else '80')
+
+        with conf.server(listen, *domains) as server:
+            if deployment['ssl']:
+                server.config('ssl', 'on')
+                server.config('ssl_certificate', deployment['ssl_cert_path'])
+                server.config('ssl_certificate_key', deployment['ssl_key_path'])
+
+            server.config('access_log', '%s../log/nginx/access.log' % deployment['path'])
+            server.config('error_log', '%s../log/nginx/error.log' % deployment['path'])
+
+            with server.section('location', '/') as root:
+                _default_proxy_pass_config(root, upstream_name, deployment['ssl'])
+                for key, value in site_config['nginx'].items():
+                    root.config(key, value)
+
+            for location, max_body_size in deployment['big_body_endpoints']:
+                with server.section('location', location) as big_body_endpoint:
+                    _default_proxy_pass_config(big_body_endpoint, upstream_name, deployment['ssl'])
+                    big_body_endpoint.config('client_max_body_size', max_body_size)
+
+            for url, relpath in site_config['static'].items():
+                with server.section('location', url) as static:
+                    static.config('access_log', 'off')
+                    static.config('alias', '%s%s' % (deployment['path'], relpath))
 
-        output += render_template(template_name, context_dict)
+            for url, relpath in site_config['sendfile'].items():
+                with server.section('location', url) as sendfile:
+                    sendfile.config('internal')
+                    sendfile.config('alias', '%s%s' % (deployment['path'], relpath))
+
+            for redirect in deployment['url_redirect']:
+                server.config('rewrite', redirect['source'], redirect['destination'], redirect.get('options', 'permanent'))
+
+            if deployment['basic_auth']:
+                server.config('auth_basic', '"Restricted"')
+                server.config('auth_basic_user_file', site_config['basic_auth_path'])
+
+            for codes, filename, root in deployment['static_error_pages']:
+                location_name = '/%s' % filename
+                args = tuple(codes) + (location_name,)
+                server.config('error_page', *args)
+
+                with server.section('location', '=', location_name) as static_error_page:
+                    static_error_page.config('root', root)
+                    static_error_page.config('allow', 'all')
+
+        if deployment['ssl']:
+            with conf.server('%s:80' % deployment['bind_ip'], *domains) as http_server:
+                http_server.config('rewrite', '^(.*)', 'https://$host$1', 'permanent')
+
+        for redirect in deployment['domains_redirect']:
+            with conf.server('%s:80' % deployment['bind_ip'], redirect['domain']) as domain_redirect:
+                domain_redirect.config('rewrite', '^(.*)', 'http://%s$1' % redirect['destination_domain'], 'permanent')
+                domain_redirect.config('access_log', '%s../log/nginx/access.log' % deployment['path'])
+                domain_redirect.config('error_log', '%s../log/nginx/error.log' % deployment['path'])
+
+            if deployment['ssl']:
+                with conf.server('%s:443' % deployment['bind_ip'], redirect['domain']) as secure_domain_redirect:
+                    secure_domain_redirect.config('ssl', 'on')
+                    secure_domain_redirect.config('ssl_certificate', deployment['ssl_cert_path'])
+                    secure_domain_redirect.config('ssl_certificate_key', deployment['ssl_key_path'])
+                    secure_domain_redirect.config('rewrite', '^(.*)', 'http://%s$1' % redirect['destination_domain'], 'permanent')
+                    secure_domain_redirect.config('access_log', '%s../log/nginx/access.log' % deployment['path'])
+                    secure_domain_redirect.config('error_log', '%s../log/nginx/error.log' % deployment['path'])
+        deployment['postprocess_nginx_conf'](conf)
+        full.extend(prettify(conf))
+    return '\n'.join(full)
+
+@task
+def update_config_file():
+    output = render_config()
     put(StringIO.StringIO(output), '%(path)s/../config/nginx.conf' % env)
     reload_nginx()

dploi_fabric/toolbox/nginx_dsl.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+from contextlib import contextmanager
+
+
+class Config(object):
+    """
+    A configuration directive
+    """
+    def __init__(self, *keys):
+        self.keys = map(str, keys)
+        self.values = []
+
+    def set_values(self, values):
+        self.values = map(str, values)
+
+    def render(self):
+        return ['%s;' % ' '.join(self.keys + self.values)]
+
+
+class Nginx(object):
+    """
+    Configuration section (or root)
+    """
+    # multi key configs are configurations which don't have a unique directive (first argument)
+    # the number indicates how many arguments together should form the cache key
+    _multi_key_configs = {
+        'proxy_set_header': 2,
+        'add_header': 2,
+        'server': 2,
+    }
+
+    def __init__(self, *args):
+        self._args = map(str, args)
+        self._children = []
+        self._cache = {}
+
+    def _make_child(self, cls, *args):
+        """
+        Add a child class cls instantiating it with *args, or return it from cache
+        """
+        if args in self._cache:
+            return self._cache[args]
+        child = cls(*args)
+        self._children.append(child)
+        self._cache[args] = child
+        return child
+
+    def _remove(self, cache_key):
+        """
+        Remove a child
+        """
+        child = self._cache.pop(cache_key)
+        self._children.remove(child)
+
+    def render(self):
+        """
+        Render the section (return as list of strings)
+        """
+        lines = []
+        if self._args:
+            lines.append('%s {' % ' '.join(self._args))
+        for child in self._children:
+            lines.extend(child.render())
+        if self._args:
+            lines.append('}')
+        return lines
+
+    @contextmanager
+    def server(self, listen, *names):
+        """
+        Add a 'server' section.
+
+        This is a special case and not used by Nginx.section because it takes no arguments, which means we have to use
+        some tricks with the cache keys.
+        """
+        yield self._make_child(Server, 'server', listen, *names)
+
+    @contextmanager
+    def section(self, name, first_arg, *other_args):
+        """
+        Add a section <name>.
+
+        first_arg is not in *args because this function requires at least two arguments to effectively cache it without
+        causing duplicate cache keys.
+        """
+        if name == 'server':
+            raise TypeError("Use Nginx.server() instead of Nginx.section('server')")
+        yield self._make_child(Nginx, name, first_arg, *other_args)
+
+    def remove_section(self, name, first_arg, *other_args):
+        """
+        Remove a section.
+        """
+        if name == 'server':
+            raise TypeError("Cannot remove server")
+        cache_key = (name, first_arg) + other_args
+        self._remove(cache_key)
+
+    def _config_cache_key_values(self, args):
+        """
+        Split a tuple of args into its cache key and values (to be used for Config)
+        """
+        cache_length = self._multi_key_configs.get(args[0], 1)
+        cache_key = args[:cache_length]
+        values = args[cache_length:]
+        return cache_key, values
+
+    def config(self, *args):
+        """
+        Add a configuration directive
+        """
+        cache_key, values = self._config_cache_key_values(args)
+        child = self._make_child(Config, *cache_key)
+        child.set_values(values)
+        return child
+
+    def get_config(self, *args):
+        """
+        Get a (cached) config directive value.
+
+        Nginx.config can't be used because if it's only called with the cache key (directive), it will set the value
+        to nothing.
+        """
+        return self._cache[args].values
+
+    def remove_config(self, *args):
+        """
+        Remove a configuration directive.
+        """
+        cache_key, values = self._config_cache_key_values(args)
+        self._remove(cache_key)
+
+
+class Server(Nginx):
+    """
+    See Nginx.server for the reason this is a special case.
+    """
+    def __init__(self, _, listen, *names):
+        super(Server, self).__init__('server')
+        self.config('listen', listen)
+        self.config('server_name', *names)
+
+
+def prettify(nginx):
+    """
+    Indents the sections and adds empty lines after a section is closed.
+    """
+    indent = 0
+    for line in nginx.render():
+        if line.startswith('}'):
+            indent -= 1
+        yield '%s%s' % ('    '  * indent, line)
+        if line.startswith('}'):
+            yield ''
+        if line.endswith('{'):
+            indent += 1
+

dploi_fabric/utils.py

@@ -55,8 +55,8 @@ class Configuration(object):
 
         },
         'nginx': {
-            'client_max_body_size': '10m'
-        }
+            'client_max_body_size': '10m',
+        },
     }
     def load_sites(self, config_file_content=None, env_dict=None):
         """
@@ -261,8 +261,8 @@ def deployment(self, site, env_dict):
             'generated_settings_path': posixpath.join(env_dict.get("path"), "_gen/settings.py"),
 
             # New settings
-            'domains_redirect': env_dict.get('domains_redirect'),
-            'url_redirect': env_dict.get('url_redirect'),
+            'domains_redirect': env_dict.get('domains_redirect', []),
+            'url_redirect': env_dict.get('url_redirect', []),
 
             'basic_auth': env_dict.get('basic_auth', False),
             'basic_auth_path': os.path.join(env_dict.get("path"), env_dict.get('basic_auth_path', None) or ""),
@@ -274,6 +274,7 @@ def deployment(self, site, env_dict):
             'static_error_pages': env_dict.get('static_error_pages', []),
             'big_body_endpoints': env_dict.get('big_body_endpoints', []),
             'home': '/home/%s' %  env_dict.get("user"),
+            'postprocess_nginx_conf': env_dict.get('postprocess_nginx_conf', lambda conf: None),
         }
 
         if not env_dict.get("databases"):