diff --git a/.env.prod.example b/.env.prod.example index 675af68..c59e4fc 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -8,7 +8,7 @@ # # Replace every CHANGE_ME with a strong random value (`openssl rand -hex 24`; # the Zitadel masterkey needs exactly 32 characters) and auth.example.com -# with the real hostname the edge (e.g. Musify's nginx) terminates TLS for. +# with the real hostname the shared edge (edge/) terminates TLS for. # --- PostgreSQL --------------------------------------------------------------- POSTGRES_USER=postgres diff --git a/.gitignore b/.gitignore index 34c9a37..03c6d42 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ zitadel/.output/ # OS Thumbs.db .DS_Store + +# Edge runtime state (Let's Encrypt account, certificates, ACME challenges) +edge/certbot/ diff --git a/README.md b/README.md index 3976444..1172a40 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,34 @@ with an example value that assumes these repos happen to sit as sibling folders. If yours don't, change it. Not versioned, and regenerated whenever the Zitadel volume is reset. +## The public edge + +`edge/compose.yml` is the only thing that publishes host ports 80/443: one +nginx that terminates TLS for every project's domains (Gitea, Musify, ...) +and proxies to their containers by name, plus a certbot container that +renews the Let's Encrypt certificates (webroot) and lets nginx reload them +every 6 hours. It joins `infra-net` and the `gitea_gitea` network (both +external, created by the stacks that own them). + +```bash +docker compose -f edge/compose.yml up -d +``` + +- **Vhosts** live in `edge/conf.d/` (one file per project; `00-common.conf` + has the shared resolver). Every upstream is a `set $upstream ...`, resolved + per request, so a project's stack being down never stops the edge from + starting. Shared bits are in `edge/snippets/`. Check a change with + `docker exec edge-nginx nginx -t`, then `docker exec edge-nginx nginx -s reload`. +- **Certificates** are state, not versioned (`edge/certbot/`). To issue one + for new hostnames (their DNS has to already point at the host): + + ```bash + docker compose -f edge/compose.yml run --rm --entrypoint certbot certbot certonly --webroot -w /var/www/certbot -d host1.example.com -d host2.example.com --non-interactive --agree-tos -m you@example.com + ``` + + Then add the `server` blocks pointing at + `/etc/letsencrypt/live//`. + ## Resetting ```bash @@ -111,6 +139,7 @@ re-creates its piece on the next `up`, since both are idempotent. ``` Infrastructure/ ├─ compose.yml # postgres, zitadel, seaweedfs, rabbitmq, jaeger: the prod shape +├─ edge/ # shared public nginx + certbot (prod only): compose.yml, conf.d/, snippets/ ├─ compose.dev.yml # dev overlay: publishes ports, adds pgAdmin ├─ .env.example / .env.prod.example ├─ postgres-init/ # creates the per-project application databases diff --git a/edge/compose.yml b/edge/compose.yml new file mode 100644 index 0000000..9199f3d --- /dev/null +++ b/edge/compose.yml @@ -0,0 +1,51 @@ +# Shared public edge: the only thing that publishes host ports 80/443. One +# nginx terminates TLS for every project's domains (Gitea, Musify, ...) and +# proxies to their containers by name, over the networks below. Certificates +# come from Let's Encrypt (webroot), renewed by the certbot service. +# +# docker compose -f edge/compose.yml up -d +# +# Requires `infra-net` (created by ../compose.yml) and `gitea_gitea` (created +# by the Gitea stack) to already exist. + +name: edge + +networks: + infra-net: + name: infra-net + external: true + gitea: + name: gitea_gitea + external: true + +services: + nginx: + image: nginx:1.27-alpine + container_name: edge-nginx + # The subshell reloads nginx every 6h so renewed certificates get picked up. + command: + - /bin/sh + - -c + - "(while :; do sleep 6h; nginx -s reload; done) & exec nginx -g 'daemon off;'" + ports: + - "80:80" + - "443:443" + volumes: + - ./conf.d:/etc/nginx/conf.d:ro + - ./snippets:/etc/nginx/snippets:ro + - ./certbot/www:/var/www/certbot:ro + - ./certbot/conf:/etc/letsencrypt:ro + networks: [gitea, infra-net] + restart: unless-stopped + + certbot: + image: certbot/certbot + container_name: edge-certbot + entrypoint: + - /bin/sh + - -c + - "trap exit TERM; while :; do certbot renew --webroot -w /var/www/certbot --quiet; sleep 12h & wait $${!}; done" + volumes: + - ./certbot/conf:/etc/letsencrypt + - ./certbot/www:/var/www/certbot + restart: unless-stopped diff --git a/edge/conf.d/00-common.conf b/edge/conf.d/00-common.conf new file mode 100644 index 0000000..f8da9c6 --- /dev/null +++ b/edge/conf.d/00-common.conf @@ -0,0 +1,8 @@ +# Docker's embedded DNS. Every upstream below is a `set $upstream ...` so +# nginx resolves it per request and starts even if a project's stack is down. +resolver 127.0.0.11 valid=10s; + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} diff --git a/edge/conf.d/gitea.conf b/edge/conf.d/gitea.conf new file mode 100644 index 0000000..8f933cb --- /dev/null +++ b/edge/conf.d/gitea.conf @@ -0,0 +1,24 @@ +server { + listen 80; + listen [::]:80; + server_name gitea.ikerdemo.cat; + include /etc/nginx/snippets/acme-redirect.conf; +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name gitea.ikerdemo.cat; + client_max_body_size 512M; + + ssl_certificate /etc/letsencrypt/live/gitea.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/gitea.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://gitea:3000; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + } +} diff --git a/edge/conf.d/musify.conf b/edge/conf.d/musify.conf new file mode 100644 index 0000000..a5163cd --- /dev/null +++ b/edge/conf.d/musify.conf @@ -0,0 +1,102 @@ +# One certificate (live/musify.ikerdemo.cat) covers the five hostnames. +server { + listen 80; + listen [::]:80; + server_name musify.ikerdemo.cat musifyapi.ikerdemo.cat + musifystream.ikerdemo.cat auth.ikerdemo.cat s3.ikerdemo.cat; + include /etc/nginx/snippets/acme-redirect.conf; +} + +# Web player. Uploads are posted to its own form actions. +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name musify.ikerdemo.cat; + client_max_body_size 1024m; + + ssl_certificate /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://web-player:3000; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name musifyapi.ikerdemo.cat; + + ssl_certificate /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://api:5111; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + } +} + +# Audio bytes. Buffering off keeps range requests (seeking) responsive. +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name musifystream.ikerdemo.cat; + + ssl_certificate /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://gateway:8081; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + proxy_buffering off; + } +} + +# Zitadel, shared with the other projects (hence the generic hostname). +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name auth.ikerdemo.cat; + + ssl_certificate /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://zitadel:8080; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + grpc_set_header Host $host; + } +} + +# SeaweedFS S3: presigned PUT/GET straight from the browser. +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name s3.ikerdemo.cat; + client_max_body_size 1024m; + + ssl_certificate /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem; + include /etc/nginx/snippets/tls.conf; + + location / { + set $upstream http://seaweedfs:8333; + proxy_pass $upstream; + include /etc/nginx/snippets/proxy.conf; + proxy_request_buffering off; + } +} diff --git a/edge/snippets/acme-redirect.conf b/edge/snippets/acme-redirect.conf new file mode 100644 index 0000000..835a835 --- /dev/null +++ b/edge/snippets/acme-redirect.conf @@ -0,0 +1,7 @@ +location /.well-known/acme-challenge/ { + root /var/www/certbot; +} + +location / { + return 301 https://$host$request_uri; +} diff --git a/edge/snippets/proxy.conf b/edge/snippets/proxy.conf new file mode 100644 index 0000000..56bd34f --- /dev/null +++ b/edge/snippets/proxy.conf @@ -0,0 +1,7 @@ +proxy_http_version 1.1; +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $connection_upgrade; diff --git a/edge/snippets/tls.conf b/edge/snippets/tls.conf new file mode 100644 index 0000000..e79c9dd --- /dev/null +++ b/edge/snippets/tls.conf @@ -0,0 +1,2 @@ +ssl_protocols TLSv1.2 TLSv1.3; +ssl_ciphers HIGH:!aNULL:!MD5;