diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 8fa6424..af0d417 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -81,6 +81,7 @@ jobs: - name: Pull latest master run: | set -euo pipefail + git config --global --add safe.directory "$DEPLOY_PATH" cd "$DEPLOY_PATH" git fetch origin master git reset --hard origin/master @@ -91,26 +92,24 @@ jobs: cd "$DEPLOY_PATH" $COMPOSE up -d wait-for-postgres keys-init migrate - # zitadel-init/seaweedfs-init verify themselves through the PUBLIC - # auth./s3. URLs (see deploy/compose.yml), so they only succeed once a - # real domain is registered and nginx is serving it (see the TODO - # below and deploy/README.md). Tolerated for now so a red step here - # doesn't block the rest of the deploy; drop the `|| true` once that's - # in place, so a real regression here fails the job again. + # Both jobs verify themselves through the public auth./s3. URLs, served + # by the shared edge nginx (Infrastructure's edge/), and are idempotent. - name: Create OIDC app and S3 bucket run: | + set -euo pipefail cd "$DEPLOY_PATH" - $COMPOSE up -d seaweedfs-init zitadel-init || echo "expected to fail until the production domain + nginx edge exist" + $COMPOSE up -d seaweedfs-init zitadel-init + for job in musify-seaweedfs-init musify-zitadel-init; do + code=$(docker wait "$job") + if [ "$code" != "0" ]; then + docker logs "$job" + echo "$job exited with $code" + exit 1 + fi + done - name: Build and start apps run: | set -euo pipefail cd "$DEPLOY_PATH" $COMPOSE --profile apps up -d --build api worker gateway web-player - - # TODO once a domain is registered and pointed at this server: merge - # Musify's vhosts (deploy/nginx/templates) into the existing - # gitea-nginx config instead of running the `nginx` service from - # compose.prod.yml, since gitea-nginx already owns host ports 80/443 - # for gitea.ikerdemo.cat. See "Continuous deployment" in - # deploy/README.md. diff --git a/.gitignore b/.gitignore index f17bdc8..067b398 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,4 @@ Thumbs.db # Deploy secrets and runtime state deploy/keys/ deploy/secrets/ -deploy/nginx/certs/ deploy/zitadel/.output/ diff --git a/deploy/.env.prod.example b/deploy/.env.prod.example index e98fb5a..4b69699 100644 --- a/deploy/.env.prod.example +++ b/deploy/.env.prod.example @@ -7,8 +7,8 @@ # subdomains), # 2. replace every CHANGE_ME with a strong random value # (`openssl rand -hex 24`), -# 3. point each *_DOMAIN below at the server, proxied/TLS-terminated the -# same way. +# 3. point each domain below at the server and add its vhost to the shared +# edge (Infrastructure's edge/conf.d). # # Requires ../../../Infrastructure running first, on the same Docker host, # with its own `infra-net` network already created. See @@ -18,14 +18,9 @@ DOTNET_ENVIRONMENT=Production # --- Domains ------------------------------------------------------------- -# Each *_DOMAIN drives its own nginx vhost (deploy/nginx/templates); the -# PUBLIC_*_URL/S3_PUBLIC_URL below must agree with it (https://). -WEB_DOMAIN=musify.example.com -API_DOMAIN=musifyapi.example.com -STREAM_DOMAIN=musifystream.example.com -AUTH_DOMAIN=auth.example.com -S3_DOMAIN=s3.example.com - +# Public URLs. They must match the vhosts of the shared edge nginx +# (Infrastructure's edge/conf.d), which is what serves them; the domains are +# independent, they don't need to share a root. PUBLIC_WEB_URL=https://musify.example.com PUBLIC_API_URL=https://musifyapi.example.com PUBLIC_STREAM_URL=https://musifystream.example.com @@ -71,6 +66,4 @@ SESSION_SECRET=CHANGE_ME AUTH_ALLOW_INSECURE_HTTP=false # --- Misc -------------------------------------------------------------------- -AUTH_REQUIRE_HTTPS=true -# Largest request body nginx accepts (audio uploads go through the web player). -MAX_UPLOAD_SIZE=1024m +AUTH_REQUIRE_HTTPS=true \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md index 26522a2..e7be331 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -13,7 +13,7 @@ wherever you actually cloned it. |---|---| | `compose.yml` | Musify's own services, including the one-shot bootstrap jobs. No published ports, no domains, no environment-specific values. | | `compose.dev.yml` | Development overlay: publishes Musify's ports to the host. | -| `compose.prod.yml` | Production overlay: adds the nginx edge and joins Infrastructure's `infra-net` network. | +| `compose.prod.yml` | Production overlay: joins Infrastructure's `infra-net` network. | | `.env` / `.env.prod` | The values compose interpolates. Not versioned; templates in `.env.example` / `.env.prod.example`. | The four applications (`api`, `worker`, `gateway`, `web-player`) sit behind @@ -113,31 +113,26 @@ SeaweedFS is done from the `Infrastructure` repo instead. ## Production -On a server with a domain: +Musify doesn't run its own nginx: the shared edge in Infrastructure +(`edge/`) owns host ports 80/443, terminates TLS for every project's domains +and proxies to Musify's containers over `infra-net`. On a server with +domains: 1. Bring up the `Infrastructure` repo first, with its own `.env.prod` filled in. It creates the `infra-net` network this stack joins. -2. From this repository's root: +2. Bring up its `edge/` (see Infrastructure's README) and add Musify's + vhosts and certificate there (`edge/conf.d/musify.conf`). The five + public hostnames (web, api, stream, auth, s3) are independent: they + don't need to share a root domain. +3. From this repository's root: ```sh cp deploy/.env.prod.example deploy/.env.prod ``` -Fill it in: - -1. Replace every `*.example.com` with your actual domains and every - `CHANGE_ME` with a random value (`openssl rand -hex 24`). Each of - `WEB_DOMAIN`/`API_DOMAIN`/`STREAM_DOMAIN`/`AUTH_DOMAIN`/`S3_DOMAIN` is - independent — they don't need to share a root domain (e.g. - `musify.example.com` + `musifyapi.example.com`, or sibling subdomains of - a domain whose DNS provider doesn't allow multi-level subdomains). - `chmod 600 deploy/.env.prod`. -2. Drop **`deploy/nginx/certs/origin.pem` and `origin.key`**, the origin - certificate nginx serves. The stack assumes Cloudflare in *Full (strict)* - mode in front of it, so a Cloudflare Origin Certificate is enough; any - certificate valid for the five hostnames works. -3. Point the five `*_DOMAIN` values at the server, proxied and - TLS-terminated the same way. + Replace every `*.example.com` with your domains (they must match the + edge's vhosts) and every `CHANGE_ME` with a random value + (`openssl rand -hex 24`). `chmod 600 deploy/.env.prod`. Then: @@ -146,68 +141,45 @@ docker compose --env-file deploy/.env.prod -f deploy/compose.yml -f deploy/compo docker compose --env-file deploy/.env.prod -f deploy/compose.yml -f deploy/compose.prod.yml --profile apps up -d --build ``` -The first command starts Musify's bootstrap jobs and the nginx edge; the -second builds and starts the applications, already wired to the OIDC client -ids the first one produced. Both are idempotent, so running them again -deploys a new version (`... --profile apps up -d --build` alone is enough for -an update once the stack already exists). +The first command starts Musify's bootstrap jobs (`zitadel-init` and +`seaweedfs-init` reach Zitadel/SeaweedFS through the public `auth.`/`s3.` +URLs, so the edge has to be serving them by then); the second builds and +starts the applications, already wired to the OIDC client ids the first one +produced. Both are idempotent, so running them again deploys a new version +(`... --profile apps up -d --build` alone is enough for an update once the +stack already exists). -Only nginx publishes ports (80/443). Every other Musify service, and all of -Infrastructure's, is reachable only on the internal `infra-net` network. +Nothing in this stack publishes a port; everything is reachable only on the +internal `infra-net` network, through the edge. ### Continuous deployment `.gitea/workflows/ci-cd.yml` runs the backend and web-player checks on every push/PR, and on a successful push to `master` deploys automatically. The -deploy job runs on `gitea-runner`, the existing self-hosted Gitea Actions -runner on the production host itself (`gitea.ikerdemo.cat`, in -`~/gitea/docker-compose.yml`), which also carries a `production:host` -label registered directly in `~/gitea/runner-data/.runner` (act_runner -reads `labels` from that file on every start, not from -`GITEA_RUNNER_LABELS`, once a runner is already registered). Running in -native mode means the label's steps execute inside the runner container -itself rather than spinning up a fresh one — that container has -`/var/run/docker.sock` mounted, so `docker`/`docker compose` there still -control the host's real Docker daemon (sibling containers, "Docker outside -of Docker"). Its compose service also bind-mounts `/home/ubuntu/deploys` -into itself at the *same* path, which matters because compose's own -relative bind mounts (`./keys`, `./nginx/...`) are resolved by the host -daemon against that literal path — mounting it anywhere else would silently -bind the wrong (empty) directory. +deploy job runs on `gitea-runner`, the self-hosted Gitea Actions runner on +the production host (`~/gitea/docker-compose.yml`), configured by +`~/gitea/runner-data/config.yaml` (`CONFIG_FILE`): job containers are pinned +to the `gitea_gitea` network so `actions/checkout` can resolve `gitea`, +`ubuntu-latest` is `node:22` (JS actions need Node inside the job +container), and a `production:host` label runs steps natively. + +Native mode means the label's steps execute inside the runner container +itself rather than a fresh one. That container has `/var/run/docker.sock` +mounted, so `docker`/`docker compose` there still control the host's real +Docker daemon (sibling containers, "Docker outside of Docker"). Its compose +service also bind-mounts `/home/ubuntu/deploys` into itself at the *same* +path, which matters because compose's own relative bind mounts (`./keys`) +are resolved by the host daemon against that literal path; mounting it +anywhere else would silently bind the wrong (empty) directory. `DEPLOY_PATH` (hardcoded in the workflow as `/home/ubuntu/deploys/musify`) is a persistent clone on that host, reused instead of a fresh checkout each run since `deploy/.env.prod` is untracked and only lives there. `/home/ubuntu/deploys/SharedServices` is the sibling clone of the -`Infrastructure`/`SharedServices` repo, already running (`infra-net` -created); its `.env.prod` shares the Postgres/RabbitMQ/S3 credentials with -Musify's own `deploy/.env.prod` by design (see that repo's README). - -**Pending DNS on the registered domains:** `deploy/.env.prod`'s five -`*_DOMAIN` values are set (`musify.ikerdemo.cat`, `musifyapi.ikerdemo.cat`, -`musifystream.ikerdemo.cat`, `auth.ikerdemo.cat`, `s3.ikerdemo.cat` — -siblings, not `api.musify.ikerdemo.cat`, because the DNS provider here -doesn't allow multi-level subdomains), but no DNS record points any of them -at the server yet, so: - -- `zitadel-init` and `seaweedfs-init` (they verify themselves through the - public `auth.`/`s3.` URLs) fail every deploy until those two resolve and - are actually reachable — the workflow tolerates this on purpose, see its - comments. Until then, `api`/`worker` build and start but can't do - anything OIDC- or storage-dependent (`AUTH_CLIENT_ID`/`WEB_CLIENT_ID`/ - `WEB_CLIENT_SECRET` in `.env.prod` stay blank); `gateway`/`web-player` - are unaffected. -- The `nginx` service from `compose.prod.yml` is intentionally **not** - brought up by the deploy job: it would try to publish host ports 80/443, - which `gitea-nginx` already owns for `gitea.ikerdemo.cat` (certbot). Once - the five DNS records above exist, merge Musify's vhosts - (`deploy/nginx/templates`) into `~/gitea/nginx/conf.d` instead — same - nginx, additional `server_name` blocks proxying to Musify's containers - over `infra-net` — rather than running a second nginx that would conflict - on those ports. That also needs a certificate valid for the 5 hostnames - (the existing setup uses certbot/Let's Encrypt, not the Cloudflare Origin - cert `compose.prod.yml`'s nginx assumes — reconcile whichever way when - that's set up). +`Infrastructure` repo (shared services + `edge/`), already running; its +`.env.prod` shares the Postgres/RabbitMQ/S3 credentials with Musify's own +`deploy/.env.prod` by design (see that repo's README). The edge is deployed +from that repo, not from this pipeline. --- @@ -218,13 +190,9 @@ deploy/ ├─ compose.yml # Musify's own services: wait-for-postgres, keys-init, │ # migrate, seaweedfs-init, zitadel-init, api/worker/gateway/web-player ├─ compose.dev.yml # dev overlay: host ports -├─ compose.prod.yml # prod overlay: nginx edge, joins infra-net +├─ compose.prod.yml # prod overlay: joins infra-net ├─ .env.example / .env.prod.example -├─ keys/ # RS256 stream-ticket key pair (generated) -└─ nginx/ - ├─ templates/default.conf.template # vhosts, rendered with the domain by envsubst - ├─ snippets/proxy.conf # proxy headers shared by every vhost - └─ certs/ # origin.pem + origin.key (not versioned) +└─ keys/ # RS256 stream-ticket key pair (generated) ``` The application images are built from `backend/Musify.*.Dockerfile` (build diff --git a/deploy/compose.prod.yml b/deploy/compose.prod.yml index 623a4e5..53e497d 100644 --- a/deploy/compose.prod.yml +++ b/deploy/compose.prod.yml @@ -1,11 +1,10 @@ name: musify-prod -# Production overlay: adds the nginx edge and joins Infrastructure's shared -# `infra-net` network so this project's containers can reach Postgres, -# Zitadel, SeaweedFS, RabbitMQ and Jaeger by container name. None of them -# run as part of this compose project anymore (see -# ../../../Infrastructure/README.md, which must already be up). Nothing but -# nginx publishes a port; every other service is reachable only on infra-net. +# Production overlay: joins Infrastructure's shared `infra-net` network so +# this project's containers can reach Postgres, Zitadel, SeaweedFS, RabbitMQ +# and Jaeger by container name, and so the shared edge nginx (Infrastructure's +# `edge/`, which owns host ports 80/443 and terminates TLS for every domain) +# can reach api, gateway and web-player. Nothing here publishes a port. # # docker compose --env-file deploy/.env.prod -f deploy/compose.yml -f deploy/compose.prod.yml up -d # docker compose --env-file deploy/.env.prod -f deploy/compose.yml -f deploy/compose.prod.yml --profile apps up -d --build @@ -20,35 +19,3 @@ services: zitadel-init: volumes: - ./.env.prod:/workspace/.env - - # Every vhost proxies through `set $upstream ...; proxy_pass $upstream;` - # with the embedded Docker DNS resolver (see nginx/templates), so nginx - # does not need api/gateway/web-player to exist at its own startup, only - # at request time. That is what lets it start in the FIRST command - # (profile-gated services aren't part of that invocation yet) and still be - # reachable at the auth domain for zitadel-init to provision through. - nginx: - image: nginx:1.27-alpine - container_name: musify-nginx - environment: - # The entrypoint renders /etc/nginx/templates/*.template with envsubst. - # The filter keeps it from eating nginx's own $host / $remote_addr. - MUSIFY_WEB_DOMAIN: ${WEB_DOMAIN} - MUSIFY_API_DOMAIN: ${API_DOMAIN} - MUSIFY_STREAM_DOMAIN: ${STREAM_DOMAIN} - MUSIFY_AUTH_DOMAIN: ${AUTH_DOMAIN} - MUSIFY_S3_DOMAIN: ${S3_DOMAIN} - MUSIFY_MAX_UPLOAD_SIZE: ${MAX_UPLOAD_SIZE:-1024m} - NGINX_ENVSUBST_FILTER: ^MUSIFY_ - ports: - - "80:80" - - "443:443" - volumes: - - ./nginx/templates:/etc/nginx/templates:ro - - ./nginx/snippets:/etc/nginx/snippets:ro - - ./nginx/certs:/etc/nginx/certs:ro - - nginx_cache:/var/cache/nginx - restart: unless-stopped - -volumes: - nginx_cache: diff --git a/deploy/nginx/snippets/proxy.conf b/deploy/nginx/snippets/proxy.conf deleted file mode 100644 index 934e050..0000000 --- a/deploy/nginx/snippets/proxy.conf +++ /dev/null @@ -1,9 +0,0 @@ -# Shared proxy settings for every Musify vhost. -proxy_http_version 1.1; -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto https; -proxy_set_header X-Forwarded-Host $host; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $connection_upgrade; diff --git a/deploy/nginx/templates/default.conf.template b/deploy/nginx/templates/default.conf.template deleted file mode 100644 index 91d7ec8..0000000 --- a/deploy/nginx/templates/default.conf.template +++ /dev/null @@ -1,109 +0,0 @@ -# Musify edge. The nginx entrypoint renders this with envsubst into -# /etc/nginx/conf.d/default.conf (replacing the image's stock file) using the -# MUSIFY_* variables set in compose.prod.yml. Each vhost has its own, -# independent domain (see deploy/.env.prod.example) — they don't need to -# share a root domain. -# -# Cloudflare (Full strict) terminates the public edge; this box only needs to -# serve the Cloudflare Origin Certificate in nginx/certs/. Each vhost proxies to -# a compose service by name through the embedded Docker DNS resolver, so nginx -# re-resolves it instead of caching the IP of a container that may be recreated. - -resolver 127.0.0.11 valid=10s; - -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -server { - listen 80; - listen [::]:80; - server_name ${MUSIFY_WEB_DOMAIN} ${MUSIFY_API_DOMAIN} - ${MUSIFY_AUTH_DOMAIN} ${MUSIFY_STREAM_DOMAIN} ${MUSIFY_S3_DOMAIN}; - return 301 https://$host$request_uri; -} - -# Web player (SvelteKit, adapter-node). Uploads are posted to its own form -# actions, so it needs the large body size too. -server { - listen 443 ssl; - http2 on; - server_name ${MUSIFY_WEB_DOMAIN}; - client_max_body_size ${MUSIFY_MAX_UPLOAD_SIZE}; - - ssl_certificate /etc/nginx/certs/origin.pem; - ssl_certificate_key /etc/nginx/certs/origin.key; - - location / { - set $upstream http://web-player:3000; - proxy_pass $upstream; - include /etc/nginx/snippets/proxy.conf; - } -} - -server { - listen 443 ssl; - http2 on; - server_name ${MUSIFY_API_DOMAIN}; - - ssl_certificate /etc/nginx/certs/origin.pem; - ssl_certificate_key /etc/nginx/certs/origin.key; - - location / { - set $upstream http://api:5111; - proxy_pass $upstream; - include /etc/nginx/snippets/proxy.conf; - } -} - -# Audio bytes. Buffering off keeps range requests (seeking) responsive. -server { - listen 443 ssl; - http2 on; - server_name ${MUSIFY_STREAM_DOMAIN}; - - ssl_certificate /etc/nginx/certs/origin.pem; - ssl_certificate_key /etc/nginx/certs/origin.key; - - location / { - set $upstream http://gateway:8081; - proxy_pass $upstream; - include /etc/nginx/snippets/proxy.conf; - proxy_buffering off; - } -} - -server { - listen 443 ssl; - http2 on; - server_name ${MUSIFY_AUTH_DOMAIN}; - - ssl_certificate /etc/nginx/certs/origin.pem; - ssl_certificate_key /etc/nginx/certs/origin.key; - - location / { - set $upstream http://zitadel:8080; - proxy_pass $upstream; - include /etc/nginx/snippets/proxy.conf; - grpc_set_header Host $host; - } -} - -# Presigned S3 PUT/GET straight from the browser. -server { - listen 443 ssl; - http2 on; - server_name ${MUSIFY_S3_DOMAIN}; - client_max_body_size ${MUSIFY_MAX_UPLOAD_SIZE}; - - ssl_certificate /etc/nginx/certs/origin.pem; - ssl_certificate_key /etc/nginx/certs/origin.key; - - location / { - set $upstream http://seaweedfs:8333; - proxy_pass $upstream; - include /etc/nginx/snippets/proxy.conf; - proxy_request_buffering off; - } -}