Rewrite to use traefik instead of nginx

nginx proved more complicated than hoped for, primarily due to
lack of hot reloading. The two blockers were:

1. nginx won't start without SSL certs existing, but we won't get
   them until after nginx starts to use the webroot challenge!
2. When certificates get renewed, nginx needs a reload.

We could have fixed these, but I realized the reason we were not
using traefik for this was that it needs persistent disk to put its
let's encrypt config in. Since that is no longer a problem due to
the secret sync, I switched us over to traefik instead. *Much*
cleaner, simpler and straightforward!!!

images/autocertbot/Dockerfile

@@ -1,5 +1,5 @@
-FROM python:3.7-buster
+FROM python:3.7-alpine
 
-RUN pip install --no-cache certbot kubernetes
+RUN pip install --no-cache kubernetes
 
 COPY autocert.py /usr/local/bin/autocert.py

images/autocertbot/autocert.py

@@ -16,6 +16,7 @@
 lets us use the webroot challenge with certbot.
 """
 import sys
+import os
 import subprocess
 import argparse
 import time
@@ -25,18 +26,6 @@
 import logging
 from kubernetes import client, config
 
-def compress_dir(path):
-    """
-    Compress directory at 'path' to a tar.gz & return it.
-
-    Paths stored in the tarball are relative to the base directory -
-    so /etc/letsencrypt/account/ is stored as account/
-    """
-    compressed_stream = io.BytesIO()
-    with tarfile.open(fileobj=compressed_stream, mode='w:gz') as tf:
-        tf.add(path, arcname='.')
-    return compressed_stream.getvalue()
-
 def update_secret(namespace, secret_name, key, value):
     """
     Update a secret object's key with the value
@@ -61,6 +50,8 @@ def update_secret(namespace, secret_name, key, value):
             raise
     # Value should be base64'd string
     new_value = base64.standard_b64encode(value).decode()
+    if secret.data is None:
+        secret.data = {}
     if new_value != secret.data.get(key):
         secret.data[key] = base64.standard_b64encode(value).decode()
         v1.patch_namespaced_secret(namespace=namespace, name=secret_name, body=secret)
@@ -80,6 +71,8 @@ def get_secret_value(namespace, secret_name, key):
             # Secret doesn't exist
             return None
         raise
+    if secret.data is None or key not in secret.data:
+        return None
     return base64.standard_b64decode(secret.data[key])
 
 def setup_logging():
@@ -91,28 +84,25 @@ def setup_logging():
 def main():
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
-        'secret_name',
-        help='Name of kubernetes secret to store certificates in'
+        '--namespace',
+        help='Namespace to operate in'
     )
+
     argparser.add_argument(
-        'email',
-        help='Contact email to pass to letsencrypt'
+        'action',
+        choices=['load', 'watch-save']
     )
+
     argparser.add_argument(
-        'domains',
-        help='List of domains to get certificates for',
-        nargs='+'
+        'secret_name'
     )
 
     argparser.add_argument(
-        '--namespace',
-        help='Namespace to operate in'
+        'key',
     )
 
     argparser.add_argument(
-        '--test-cert',
-        help='Get test certificates from the staging server',
-        action='store_true'
+        'path',
     )
 
     args = argparser.parse_args()
@@ -124,31 +114,21 @@ def main():
             with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace") as f:
                 args.namespace = f.read().strip()
         except FileNotFoundError:
-            print("Can not determin a namespace, must be explicitly set with --namespace", file=sys.stderr)
+            print("Can not determine a namespace, must be explicitly set with --namespace", file=sys.stderr)
             sys.exit(1)
 
-    current_dir = get_secret_value(args.namespace, args.secret_name, 'letsencrypt.tar.gz')
-    if current_dir:
-        with tarfile.open(fileobj=io.BytesIO(current_dir), mode='r:gz') as tf:
-            tf.extractall('/etc/letsencrypt')
-
-    certbot_args = [
-        'certbot',
-        'certonly', '--webroot', '-n', '--agree-tos',
-        '-m', args.email,
-        '-w', '/usr/share/nginx/html'
-    ] + [f'-d={d}' for d in args.domains]
-
-    if args.test_cert:
-        certbot_args.append('--test-cert')
-        logging.info("Using Let's Encrypt Staging server")
-
-    while True:
-        logging.info(f"Calling certbot: {' '.join(certbot_args)}")
-        subprocess.check_call(certbot_args)
-        letsencrypt_dir = compress_dir('/etc/letsencrypt')
-        update_secret(args.namespace, args.secret_name, 'letsencrypt.tar.gz', letsencrypt_dir)
-        time.sleep(30)
+    if args.action == 'load':
+        value = get_secret_value(args.namespace, args.secret_name, args.key)
+        if value:
+            with open(args.path, 'wb') as f:
+                f.write(value)
+                os.fchmod(f.fileno(), 0o600)
+    else:
+        while True:
+            if os.path.exists(args.path):
+                with open(args.path, 'rb') as f:
+                    update_secret(args.namespace, args.secret_name, args.key, f.read())
+            time.sleep(30)
 
 if __name__ == '__main__':
     main()

jupyterhub/templates/proxy/autohttps/configmap.yaml

@@ -4,47 +4,66 @@
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: nginx-proxy-config
+  name: traefik-proxy-config
   labels:
     {{- include "jupyterhub.labels" . | nindent 4 }}
 data:
-  proxy.conf: |
-    map $http_upgrade $connection_upgrade {
-        default upgrade;
-        ''      close;
-    }
-
-    server {
-        listen 80;
-
-        # Serve let's encrypt challenges from the disk shared with autocertbot
-        location /.well-known/acme-challenge {
-            root   /usr/share/nginx/html/;
-        }
-
-        location / {
-          # Redirect everything to HTTPS
-          return 301 https://$host$request_uri;
-        }
-    }
-
-    {{- range $host := .Values.proxy.https.hosts }}
-    server {
-        listen 443 ssl;
-        server_name {{ $host }};
-
-        ssl_certificate /etc/letsencrypt/live/{{ $host }}/fullchain.pem;
-        ssl_certificate_key /etc/letsencrypt/live/{{ $host }}/privkey.pem;
-
-        location / {
-          proxy_pass   http://proxy-http:8000/;
-          proxy_set_header Host $host;
-          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-
-          # websocket headers
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection $connection_upgrade;
-        }
-    }
-    {{- end }}
+  traefik.toml: |
+    # traefik.toml file template
+    defaultEntryPoints = ["http", "https"]
+
+    logLevel = "INFO"
+    # log errors, which could be proxy errors
+    [accessLog]
+    format = "json"
+    [accessLog.filters]
+    statusCodes = ["500-999"]
+
+    [accessLog.fields.headers]
+    [accessLog.fields.headers.names]
+    Authorization = "redact"
+    Cookie = "redact"
+    Set-Cookie = "redact"
+    X-Xsrftoken = "redact"
+
+    [respondingTimeouts]
+    idleTimeout = "10m0s"
+
+    [entryPoints]
+      [entryPoints.http]
+      address = ":80"
+
+      [entryPoints.https]
+      address = ":443"
+
+    [wss]
+    protocol = "http"
+
+    [certificatesResolvers.le.acme]
+      email = "[email protected]"
+      storage = "/etc/acme/acme.json"
+      caServer = "https://acme-staging-v02.api.letsencrypt.org/directory"
+      [certificatesResolvers.le.acme.httpChallenge]
+        # used during the challenge
+        entryPoint = "http"
+    [providers]
+      [providers.file]
+        filename = '/etc/traefik/dynamic.toml'
+
+  dynamic.toml: |
+    [http.routers]
+      [http.routers.chp]
+        rule = "PathPrefix(`/`)"
+        service = "chp"
+          [http.routers.chp.tls]
+            certResolver = "le"
+            {{- range $host := .Values.proxy.https.hosts }}
+            [[http.routers.chp.tls.domains]]
+            main = "{{ $host }}"
+            {{- end}}
+
+    [http.services]
+      [http.services.chp.loadBalancer]
+        [[http.services.chp.loadBalancer.servers]]
+          url = "http://proxy-http:8000/"
 {{- end }}

jupyterhub/templates/proxy/autohttps/deployment.yaml

@@ -18,7 +18,7 @@ spec:
         {{- include "jupyterhub.matchLabels" . | nindent 8 }}
         hub.jupyter.org/network-access-proxy-http: "true"
       annotations:
-        checksum/config-map: {{ include (print .Template.BasePath "/proxy/autohttps/configmap-nginx.yaml") . | sha256sum }}
+        checksum/config-map: {{ include (print .Template.BasePath "/proxy/autohttps/configmap.yaml") . | sha256sum }}
     spec:
       {{- if .Values.rbac.enabled }}
       serviceAccountName: autohttps
@@ -30,33 +30,42 @@ spec:
       nodeSelector: {{ toJson .Values.proxy.nodeSelector }}
       {{- include "jupyterhub.coreAffinity" . | nindent 6 }}
       volumes:
-        - name: webroot
-          emptyDir: {}
         - name: certificates
           emptyDir: {}
-        - name: nginx-config
+        - name: traefik-config
           configMap:
-            name: nginx-proxy-config
+            name: traefik-proxy-config
       initContainers:
       - name: volume-mount-hack-why-god-still
         image: busybox
         command:
         - /bin/sh
         - -c
-        - chmod 0755 /usr/share/nginx/html /etc/letsencrypt
+        - chmod 0755 /etc/acme
+        volumeMounts:
+          - name: certificates
+            mountPath: /etc/acme
+      - name: load-acme
+        image: "{{ .Values.proxy.autocertbot.image.name }}:{{ .Values.proxy.autocertbot.image.tag }}"
+        {{- with .Values.proxy.autocertbot.image.pullPolicy }}
+        imagePullPolicy: {{ . }}
+        {{- end }}
+        command: ["/usr/local/bin/autocert.py", "load", "proxy-public-tls-acme", "acme.json", "/etc/acme/acme.json"]
+        env:
+        # We need this to get logs immediately
+        - name: PYTHONUNBUFFERED
+          value: "True"
         volumeMounts:
-          - name: webroot
-            mountPath: /usr/share/nginx/html
           - name: certificates
-            mountPath: /etc/letsencrypt
+            mountPath: /etc/acme
       containers:
-        - name: nginx
-          image: "{{ .Values.proxy.nginx.image.name }}:{{ .Values.proxy.nginx.image.tag }}"
-          {{- with .Values.proxy.nginx.image.pullPolicy }}
+        - name: traefik
+          image: "{{ .Values.proxy.traefik.image.name }}:{{ .Values.proxy.traefik.image.tag }}"
+          {{- with .Values.proxy.traefik.image.pullPolicy }}
           imagePullPolicy: {{ . }}
           {{- end }}
           resources:
-            {{- .Values.proxy.nginx.resources | toYaml | trimSuffix "\n" | nindent 12 }}
+            {{- .Values.proxy.traefik.resources | toYaml | trimSuffix "\n" | nindent 12 }}
           ports:
             - name: http
               containerPort: 80
@@ -65,32 +74,21 @@ spec:
               containerPort: 443
               protocol: TCP
           volumeMounts:
-            - name: webroot
-              mountPath: /usr/share/nginx/html
+            - name: traefik-config
+              mountPath: /etc/traefik
             - name: certificates
-              mountPath: /etc/letsencrypt
-            - name: nginx-config
-              mountPath: /etc/nginx/conf.d/
+              mountPath: /etc/acme
         - name: certbot
           image: "{{ .Values.proxy.autocertbot.image.name }}:{{ .Values.proxy.autocertbot.image.tag }}"
           {{- with .Values.proxy.autocertbot.image.pullPolicy }}
           imagePullPolicy: {{ . }}
           {{- end }}
-          command: ["/usr/local/bin/autocert.py"]
+          command: ["/usr/local/bin/autocert.py", "watch-save", "proxy-public-tls-acme", "acme.json", "/etc/acme/acme.json"]
           env:
           # We need this to get logs immediately
           - name: PYTHONUNBUFFERED
             value: "True"
-          args:
-          - --test-cert
-          - {{ .Release.Name }}-https-certbot-dir
-          - [email protected]
-          {{- range $host := .Values.proxy.https.hosts }}
-          - {{ $host }}
-          {{- end }}
           volumeMounts:
-            - name: webroot
-              mountPath: /usr/share/nginx/html
             - name: certificates
-              mountPath: /etc/letsencrypt
+              mountPath: /etc/acme
 {{- end }}

jupyterhub/values.yaml

@@ -153,6 +153,13 @@ proxy:
     proxyBodySize: 64m
     hstsIncludeSubdomains: 'false'
     resources: {}
+  traefik:
+    image:
+      name: traefik
+      tag: v2.1
+    proxyBodySize: 64m
+    hstsIncludeSubdomains: 'false'
+    resources: {}
   autocertbot:
     image:
       name: jupyterhub/k8s-autocertbot