support internal ssl

[kreuzert]

Unfortunately, I couldn't make this work in a production environment (Kubernetes Cluster, one traefik deployment, one etcd deployment, one jupyterhub with internal ssl). While at first glance everything looks fine, we're hitting two types of SSL Errors, which we don't see with a nginx, chp + jupyterhub setup.

At the moment, I don't fully understand the reason for that. I assume traefik reuses connections that tornado has already closed.

Details on Traefik setup:

(click to open)

Proxy Class used in this setup (click to open)

import json

from jupyterhub_traefik_proxy import traefik_utils as traefik_utils_orig
from jupyterhub_traefik_proxy.etcd import TraefikEtcdProxy
from jupyterhub_traefik_proxy.kv_proxy import TKvProxy
from tornado.httpclient import HTTPClientError
from traitlets import Bool
from traitlets import Dict
from traitlets import List
from traitlets import Unicode


class SSLTKvProxy(TKvProxy):
    traefik_tls_options = Dict(
        default_value=None,
        allow_none=True,
        config=True,
        help="A dictionary of traefik TLS options to apply to services when using internal SSL. This can be used to customize the TLS settings used by traefik when communicating with backends over SSL. Example: {"options": "default"}
        ",
    )
    traefik_router_middlewares = List(
        default_value=[],
        allow_none=True,
        config=True,
        help="A list of traefik middleware names to add to each router for retrying requests. This can be used to improve reliability when using traefik with backends that may intermittently fail.",
    )

    skip_hub_route = Bool(
        False,
        config=True,
        help="If True, skip adding a route for the hub itself",
    )

    skip_services_route = Bool(
        False,
        config=True,
        help="If True, skip adding a route for services",
    )

    traefik_http_servers_transport = Unicode(
        "",
        allow_none=True,
        config=True,
        help="The name of the servers transport to use for internal SSL",
    )

    traefik_alias_prefix = Unicode(
        "",
        config=True,
        help="""The alias prefix to use for traefik services.

        This is used to namespace the services created by traefik,
        to avoid conflicts with other services running in the same
        environment.
        """,
    )

    traefik_enforce_host_in_rules = Unicode(
        "",
        config=True,
        help="""
        Optional configuration to enforce a specific host in all generated traefik rules.
        Allows running multiple JupyterHub instances behind the same traefik proxy.
        """,
    )

    def generate_rule(self, routespec):
        rule = traefik_utils_orig.generate_rule(routespec)
        if (not rule.startswith("Host")) and self.traefik_enforce_host_in_rules:
            return f"Host(`{self.traefik_enforce_host_in_rules}`) && {rule}"
        else:
            return rule

    def generate_alias(self, routespec, kind=""):
        alias = traefik_utils_orig.generate_alias(routespec, kind)
        if self.traefik_alias_prefix:
            return f"{self.traefik_alias_prefix}_{alias}"
        else:
            return alias

    async def add_route(self, routespec, target, data=None):
        if data.get("hub", False) and self.skip_hub_route:
            self.log.info("Skipping addition of hub route as configured to do so")
            return
        if routespec.startswith("/services/") and self.skip_services_route:
            self.log.info("Skipping addition of services route as configured to do so")
            return
        await super().add_route(routespec, target, data)

    async def _check_for_traefik_service(self, routespec, kind):
        """Check for an expected router or service in the Traefik API.

        This is used to wait for traefik to load configuration
        from a provider
        """
        # expected e.g. 'service' + '_' + routespec @ file
        routespec = self.validate_routespec(routespec)
        expected = self.generate_alias(routespec, kind) + "@" + self.provider_name
        path = f"/api/http/{kind}s/{expected}"
        try:
            resp = await self._traefik_api_request(path)
            json.loads(resp.body)
        except HTTPClientError as e:
            if e.code == 404:
                self.log.debug(
                    "Traefik route for %s: %s not yet in %s", routespec, expected, kind
                )
                return False
            self.log.exception(f"Error checking traefik api for {kind} {routespec}")
            return False
        except Exception:
            self.log.exception(f"Error checking traefik api for {kind} {routespec}")
            return False

        # found the expected endpoint
        return True

    def _dynamic_config_for_route(self, routespec, target, data):
        """Returns two dicts, which will be used to update traefik configuration for a given route

        (traefik_config, jupyterhub_config) -
            where traefik_config is traefik dynamic_config to be merged,
            and jupyterhub_config is jupyterhub-specific data to be stored elsewhere
            (implementation-specific) and associated with the route
        """

        service_alias = self.generate_alias(routespec, "service")
        router_alias = self.generate_alias(routespec, "router")
        rule = self.generate_rule(routespec)
        # dynamic config to deep merge
        traefik_config = {
            "http": {
                "routers": {},
                "services": {},
            },
        }

        jupyterhub_config = {
            "routes": {},
        }
        traefik_config["http"]["routers"][router_alias] = {
            "service": service_alias,
            "rule": rule,
            "entryPoints": [self.traefik_entrypoint],
        }
        if self.traefik_router_middlewares:
            traefik_config["http"]["routers"][router_alias][
                "middlewares"
            ] = self.traefik_router_middlewares
        traefik_config["http"]["services"][service_alias] = {
            "loadBalancer": {"servers": [{"url": target}], "passHostHeader": True}
        }
        if self.traefik_tls_options:
            traefik_config["http"]["routers"][router_alias][
                "tls"
            ] = self.traefik_tls_options
        if self.app.internal_ssl:
            if self.traefik_http_servers_transport:
                traefik_config["http"]["services"][service_alias]["loadBalancer"][
                    "serversTransport"
                ] = self.traefik_http_servers_transport

        # Add the data node to a separate top-level node, so traefik doesn't see it.
        # key needs to be key-value safe (no '/')
        # store original routespec, router, service aliases for easy lookup
        jupyterhub_config["routes"][router_alias] = {
            "data": data,
            "routespec": routespec,
            "target": target,
            "router": router_alias,
            "service": service_alias,
        }
        return traefik_config, jupyterhub_config

    def _keys_for_route(self, routespec):
        service_alias = self.generate_alias(routespec, "service")
        router_alias = self.generate_alias(routespec, "router")
        traefik_keys = (
            ["http", "routers", router_alias],
            ["http", "services", service_alias],
        )
        jupyterhub_keys = (["routes", router_alias],)
        return traefik_keys, jupyterhub_keys

    async def get_all_routes(self):
        if self._start_future and not self._start_future.done():
            await self._start_future

        jupyterhub_config = await self._get_jupyterhub_dynamic_config()

        all_routes = {}
        for _, route in jupyterhub_config.get("routes", {}).items():
            if self.traefik_alias_prefix and (
                not route.get("router", "").startswith(self.traefik_alias_prefix)
                and not route.get("service", "").startswith(self.traefik_alias_prefix)
            ):
                # not our route
                continue
            all_routes[route["routespec"]] = {
                "routespec": route["routespec"],
                "data": route.get("data", {}),
                "target": route["target"],
            }
        return all_routes

    async def get_route(self, routespec):
        routespec = self.validate_routespec(routespec)
        router_alias = self.generate_alias(routespec, "router")
        route_key = self.kv_separator.join(
            [self.kv_jupyterhub_prefix, "routes", router_alias]
        )
        route = await self._kv_get_tree(route_key)
        if not route:
            return None
        return {key: route[key] for key in ("routespec", "data", "target")}


class SSLTraefikEtcdProxy(SSLTKvProxy, TraefikEtcdProxy):
    pass

JupyterHub config: (click to open)

c.SSLTraefikEtcdProxy.etcd_url = "http://etcd.etcd.svc:2379"
c.SSLTraefikEtcdProxy.should_start = False
c.SSLTraefikEtcdProxy.skip_hub_route = True
c.SSLTraefikEtcdProxy.skip_services_route = False
c.SSLTraefikEtcdProxy.enable_setup_dynamic_config = False
c.SSLTraefikEtcdProxy.traefik_http_servers_transport = f"{namespace}-serverstransport@kubernetescrd"
c.SSLTraefikEtcdProxy.traefik_router_middlewares = [] # tried different buffering middlewares here
c.SSLTraefikEtcdProxy.traefik_api_url = "http://traefik-internal.traefik.svc:8080"
c.SSLTraefikEtcdProxy.traefik_api_entrypoint = "traefik"
c.SSLTraefikEtcdProxy.traefik_entrypoint = "web"
c.SSLTraefikEtcdProxy.traefik_alias_prefix = namespace
c.SSLTraefikEtcdProxy.traefik_enforce_host_in_rules = <my.domain.com>  

KubernetesCRD Traefik Resources:

ServersTransport: (click to open)

apiVersion: traefik.io/v1alpha1
kind: ServersTransport
  name: serverstransport
  namespace: <namespace>
spec:
  certificatesSecrets:
  - proxy-client-tls
  disableHTTP2: true
  insecureSkipVerify: true
  rootCAs:
  - secret: proxy-client-tls
  

IngressRoute (click to open)

apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
  name: jupyterhub
  namespace: namespace
spec:
  entryPoints:
  - web
  routes:
  - kind: Rule
    match: Host(`my.domain.com`) && PathPrefix(`/`)
    priority: 1
    services:
    - name: hub
      namespace: namespace
      port: 8081
      scheme: https
      serversTransport: serverstransport
  tls:
    secretName: traefik-letsencrypt # managed via CertManager

Errors:

(click to expand)

On JupyterHub:

logger=JupyterHub levelno=30 levelname=WARNING file=/usr/local/lib/python3.12/site-packages/tornado/iostream.py line=779 function=_handle_read : error on read: [SSL: SHUTDOWN_WHILE_IN_INIT] shutdown while in init (_ssl.c:2580)

On JupyterLab, e.g. when opening a XPra Proxy

2025-11-16 10:20:17,987 - ServerApp - ERROR - Uncaught exception GET /user/username/servername/xprahtml5/js/lib/web-streams-ponyfill.es6.js (10.42.15.30)
HTTPServerRequest(protocol='https', host='jupyter-staging.jsc.fz-juelich.de', method='GET', uri='/user/username/servername/xprahtml5/js/lib/web-streams-ponyfill.es6.js', version='HTTP/1.1', remote
_ip='10.42.15.30')
Traceback (most recent call last):
  File "/p/software/default/stages/2025/software/tornado/6.4.1-GCCcore-13.3.0/lib/python3.12/site-packages/tornado/web.py", line 1790, in _execute
    result = await result
             ^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/websocket.py", line 101, in get
    return await self.http_get(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/handlers.py", line 817, in http_get
    return await ensure_async(self.proxy(self.port, path))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server/2.15.0-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_core/utils/__init__.py", line 198, in ensure_async
    result = await obj
             ^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/handlers.py", line 990, in proxy
    return await ensure_async(super().proxy(port, path))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server/2.15.0-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_core/utils/__init__.py", line 198, in ensure_async
    result = await obj
             ^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/handlers.py", line 814, in proxy
    return await ensure_async(super().proxy(port, path))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server/2.15.0-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_core/utils/__init__.py", line 198, in ensure_async
    result = await obj
             ^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/handlers.py", line 384, in proxy
    return await self._proxy_buffered(host, port, proxied_path, body, client)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/jupyter-server-proxy/20250303-GCCcore-13.3.0/lib/python3.12/site-packages/jupyter_server_proxy/handlers.py", line 475, in _proxy_buffered
    response = await client.fetch(req, raise_error=False)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/tornado/6.4.1-GCCcore-13.3.0/lib/python3.12/site-packages/tornado/iostream.py", line 773, in _handle_read
    pos = self._read_to_buffer_loop()
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/tornado/6.4.1-GCCcore-13.3.0/lib/python3.12/site-packages/tornado/iostream.py", line 750, in _read_to_buffer_loop
    if self._read_to_buffer() == 0:
       ^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/tornado/6.4.1-GCCcore-13.3.0/lib/python3.12/site-packages/tornado/iostream.py", line 861, in _read_to_buffer
    bytes_read = self.read_from_fd(buf)
                 ^^^^^^^^^^^^^^^^^^^^^^
  File "/p/software/default/stages/2025/software/tornado/6.4.1-GCCcore-13.3.0/lib/python3.12/site-packages/tornado/iostream.py", line 1116, in read_from_fd
    return self.socket.recv_into(buf, len(buf))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
OSError: [Errno 22] Invalid argument  

Previous message (click to open)

PR for #191

I have not added tests so far. In the tests TraefikProxy is created without a MockHub including app.internal_ssl.

This PR adds support for 3 cases:

1. JupyterHub starts the Proxy (TraefikProxy.should_start=True)
2. JupyterHub does not start the Proxy but adds dynamic config (TraefikProxy.enable_setup_dynamic_config=True)
3. JupyterHub does not start the Proxy and does not add dynamic (TraefikProxy.enable_setup_dynamic_config=False) (internal-ssl certificates have to already exist)

All tested with redis as provider (already running locally)

1. Start Proxy

jupyterhub_config.py

from jupyterhub_traefik_proxy.redis import TraefikRedisProxy

c.JupyterHub.internal_ssl = True
c.JupyterHub.proxy_class = TraefikRedisProxy
c.TraefikRedisProxy.redis_url = "redis://localhost:6379"
c.TraefikRedisProxy.redis_password = "...redispassword..."

from jupyterhub.auth import DummyAuthenticator
c.JupyterHub.authenticator_class = DummyAuthenticator

from jupyterhub.spawner import LocalProcessSpawner
c.JupyterHub.spawner_class = LocalProcessSpawner

Run JupyterHub, Hub available at http://localhost:8000

2. Running Proxy, with dynamic config

Start traefik:
traefik --entryPoints.traefik.address=:8080/tcp --entryPoints.http.address=:8000/tcp --ping=true --log.level=INFO --api.insecure=true --providers.redis --providers.redis.endpoints=localhost:6379 --providers.redis.password=<password> --providers.redis.rootkey=traefik

jupyterhub_config.py:

from jupyterhub_traefik_proxy.redis import TraefikRedisProxy

c.JupyterHub.internal_ssl = True
c.JupyterHub.proxy_class = TraefikRedisProxy
c.TraefikRedisProxy.redis_url = "redis://localhost:6379"
c.TraefikRedisProxy.redis_password = "<password>"
c.TraefikRedisProxy.should_start = False
c.TraefikRedisProxy.enable_setup_dynamic_config = True
c.TraefikRedisProxy.traefik_api_url = "http://localhost:8080"
c.TraefikRedisProxy.traefik_api_entrypoint = "traefik"

from jupyterhub.auth import DummyAuthenticator
c.JupyterHub.authenticator_class = DummyAuthenticator

from jupyterhub.spawner import LocalProcessSpawner
c.JupyterHub.spawner_class = LocalProcessSpawner

TraefikProxy._setup_traefik_dynamic_config sets http.serversTransports to the absolut c.JupyterHub.internal_certs_location (code). Therefore traefik + JupyterHub must run on the same machine (or traefik must use the same pathes for the proxy-client certificates)

3. Running Proxy, configure serversTransports externally

In this example I've used Traefik FileProvider.:

Setup yml File with serverTransports configuration:
/home/ubuntu/traefik_dynamic/transport.yml

http:
  ServersTransports:
    someTransportName:
      certificates:
        - certfile: /traefik_path_to_certificates/proxy-client/proxy-client.crt
          keyfile: /traefik_path_to_certificates/proxy-client/proxy-client.key
      rootCAs:
        - /traefik_path_to_certificates/proxy-client-ca_trust.crt

Start traefik (notice file provider)
traefik --entryPoints.traefik.address=:8080/tcp --entryPoints.http.address=:8000/tcp --ping=true --log.level=INFO --api.insecure=true --providers.redis --providers.redis.endpoints=localhost:6379 --providers.redis.password=<password> --providers.redis.rootkey=traefik --providers.file --providers.file.directory=/path/to/traefikprovider/dynamic --providers.file.watch=true

jupyterhub_config.py:
c.TraefikRedisProxy.traefik_http_servers_transport and .enable_setup_dynamic_config are important)

from jupyterhub_traefik_proxy.redis import TraefikRedisProxy

c.JupyterHub.internal_ssl = True
c.JupyterHub.proxy_class = TraefikRedisProxy
c.TraefikRedisProxy.redis_url = "redis://localhost:6379"
c.TraefikRedisProxy.redis_password = "...redispassword..."
c.TraefikRedisProxy.should_start = False
c.TraefikRedisProxy.enable_setup_dynamic_config = False
c.TraefikRedisProxy.traefik_http_servers_transport = "someTransportName@file"
c.TraefikRedisProxy.traefik_api_url = "http://localhost:8080"
c.TraefikRedisProxy.traefik_api_entrypoint = "traefik"

from jupyterhub.auth import DummyAuthenticator
c.JupyterHub.authenticator_class = DummyAuthenticator

from jupyterhub.spawner import LocalProcessSpawner
c.JupyterHub.spawner_class = LocalProcessSpawner