Merge pull request 'Docker audit fixes' (#1) from docker-audit-fixes into master

Reviewed-on: #1

.env.example

@@ -3,7 +3,12 @@
 #
 #   cp .env.example .env
 
-# --- PostgreSQL (shared instance, one database per project) -----------------
+# Address the dev ports are published on. 127.0.0.1 keeps them off the LAN.
+# Use 0.0.0.0 only if your hosts file maps host.docker.internal to the LAN IP
+# (then the browser can't reach 127.0.0.1-only ports through that name).
+DEV_BIND_ADDRESS=127.0.0.1
+
+# --- PostgreSQL (shared instance; superuser for this stack only) -------------
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_HOST_PORT=5432
@@ -17,16 +22,21 @@ ZITADEL_MASTERKEY=MasterkeyNeedsToHave32Characters
 ZITADEL_EXTERNAL_DOMAIN=host.docker.internal
 ZITADEL_EXTERNAL_PORT=8080
 ZITADEL_EXTERNAL_SECURE=false
+ZITADEL_PUBLIC_SCHEME=http
+# Login V2 UI (zitadel-login). In dev it has its own port; the URL has no
+# trailing slash. Only read when the Zitadel instance is created.
+ZITADEL_LOGIN_HOST_PORT=3900
+ZITADEL_LOGIN_BASE_URL=http://host.docker.internal:3900/ui/v2/login
 ZITADEL_DATABASE_PASSWORD=zitadel
 # Console login: <username>@zitadel.<external domain>
 ZITADEL_ADMIN_USERNAME=admin
 ZITADEL_ADMIN_PASSWORD=Password1!
 
-# --- SeaweedFS (S3) ----------------------------------------------------------
+# --- SeaweedFS (S3; admin identity for this stack only) ---------------------
 S3_ACCESS_KEY=admin_access_key
 S3_SECRET_KEY=admin_secret_key
 
-# --- RabbitMQ -----------------------------------------------------------------
+# --- RabbitMQ (administrator for this stack only) ---------------------------
 RABBITMQ_DEFAULT_USER=guest
 RABBITMQ_DEFAULT_PASS=guest
 
@@ -34,3 +44,23 @@ RABBITMQ_DEFAULT_PASS=guest
 PGADMIN_DEFAULT_EMAIL=admin@infra.dev
 PGADMIN_DEFAULT_PASSWORD=admin
 PGADMIN_HOST_PORT=5050
+
+# --- Per-project credentials (see provision/) ---------------------------------
+# Each project gets its own Postgres role/database, RabbitMQ user/vhost and S3
+# identity limited to its bucket; the admin accounts above stay with this
+# stack. Each project's own .env copies its values from here. An empty value
+# means that project doesn't get that service.
+MUSIFY_DB_PASSWORD=musify
+MUSIFY_RABBITMQ_PASSWORD=musify
+MUSIFY_S3_ACCESS_KEY=musify_access_key
+MUSIFY_S3_SECRET_KEY=musify_secret_key
+
+PING_DB_PASSWORD=ping
+PING_RABBITMQ_PASSWORD=
+PING_S3_ACCESS_KEY=ping_access_key
+PING_S3_SECRET_KEY=ping_secret_key
+
+DOCKERMANAGER_DB_PASSWORD=dockermanager
+DOCKERMANAGER_RABBITMQ_PASSWORD=
+DOCKERMANAGER_S3_ACCESS_KEY=
+DOCKERMANAGER_S3_SECRET_KEY=

.env.prod.example

@@ -10,7 +10,7 @@
 # the Zitadel masterkey needs exactly 32 characters) and auth.example.com
 # with the real hostname the shared edge (edge/) terminates TLS for.
 
-# --- PostgreSQL ---------------------------------------------------------------
+# --- PostgreSQL (superuser for this stack only) -----------------------------
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=CHANGE_ME
 
@@ -20,14 +20,41 @@ ZITADEL_EXTERNAL_DOMAIN=auth.example.com
 ZITADEL_EXTERNAL_PORT=443
 ZITADEL_EXTERNAL_SECURE=true
 ZITADEL_TLS_MODE=external
+ZITADEL_PUBLIC_SCHEME=https
+# Login V2 UI, routed by the edge under the same hostname. No trailing slash.
+ZITADEL_LOGIN_BASE_URL=https://auth.example.com/ui/v2/login
 ZITADEL_DATABASE_PASSWORD=CHANGE_ME
-ZITADEL_ADMIN_USERNAME=admin
+ZITADEL_ORG_NAME=Iker Demo
+ZITADEL_ADMIN_USERNAME=admin@ikerdemo.cat
+ZITADEL_ADMIN_FIRST_NAME=Iker Demo
+ZITADEL_ADMIN_LAST_NAME=Admin
+ZITADEL_ADMIN_DISPLAY_NAME=Iker Demo Admin
+# Must pass Zitadel's password policy (upper, lower, number, symbol, 8+).
 ZITADEL_ADMIN_PASSWORD=CHANGE_ME
 
-# --- SeaweedFS (S3) ----------------------------------------------------------
+# --- SeaweedFS (S3; admin identity for this stack only) ---------------------
 S3_ACCESS_KEY=CHANGE_ME
 S3_SECRET_KEY=CHANGE_ME
 
-# --- RabbitMQ -----------------------------------------------------------------
+# --- RabbitMQ (administrator for this stack only) ---------------------------
 RABBITMQ_DEFAULT_USER=CHANGE_ME
 RABBITMQ_DEFAULT_PASS=CHANGE_ME
+
+# --- Per-project credentials (see provision/) ---------------------------------
+# Each project's own .env.prod copies its values from here. S3 keys: only
+# A-Z a-z 0-9 _ + / = . - (hex is fine). Empty = that project doesn't get
+# that service.
+MUSIFY_DB_PASSWORD=CHANGE_ME
+MUSIFY_RABBITMQ_PASSWORD=CHANGE_ME
+MUSIFY_S3_ACCESS_KEY=CHANGE_ME
+MUSIFY_S3_SECRET_KEY=CHANGE_ME
+
+PING_DB_PASSWORD=CHANGE_ME
+PING_RABBITMQ_PASSWORD=
+PING_S3_ACCESS_KEY=CHANGE_ME
+PING_S3_SECRET_KEY=CHANGE_ME
+
+DOCKERMANAGER_DB_PASSWORD=CHANGE_ME
+DOCKERMANAGER_RABBITMQ_PASSWORD=
+DOCKERMANAGER_S3_ACCESS_KEY=
+DOCKERMANAGER_S3_SECRET_KEY=

.gitattributes

@@ -0,0 +1,3 @@
+# Shell scripts run inside Linux containers: a CRLF checkout (Windows,
+# core.autocrlf) would break them.
+*.sh text eol=lf

README.md

@@ -8,8 +8,7 @@ provisioning, and keeping in sync) its own copy, they run once, here.
 
 This repo doesn't assume anything about where the other projects are
 checked out. Each one just needs a local path to this repo's checkout (see
-"The Zitadel admin PAT" below), which you set once in that project's own
+"The Zitadel PAT" below), which you set once in that project's own `.env`.
-`.env`.
 
 Each project keeps its own independent, fast `docker compose up` for the
 things that actually are its own: API/worker/gateway containers, database
@@ -23,7 +22,7 @@ needs to know a consuming project exists.
 
 | Here (shared) | Stays in each project |
 |---|---|
-| PostgreSQL (one instance, one database per project) | The project's own EF Core migrations, connection pooling config |
+| PostgreSQL (one instance, one database and role per project) | The project's own EF Core migrations, connection pooling config |
 | Zitadel (OIDC provider) | The project's own OIDC app registration (`zitadel-init`), JWT validation config |
 | SeaweedFS (S3 storage) | The project's own bucket (`seaweedfs-init`), key layout |
 | RabbitMQ (broker) | The project's own exchanges/queues/consumers |
@@ -37,8 +36,10 @@ cp .env.example .env
 docker compose -f compose.yml -f compose.dev.yml up -d
 ```
 
-That starts Postgres, Zitadel, SeaweedFS, RabbitMQ and Jaeger, with their
+That starts Postgres, Zitadel (and its login UI), SeaweedFS, RabbitMQ and Jaeger, with their
-ports published to `localhost`. Add `--profile tools` to also start pgAdmin.
+ports published on `127.0.0.1` only (`DEV_BIND_ADDRESS`), so the dev
+credentials aren't reachable from the LAN. Add `--profile tools` to also
+start pgAdmin.
 
 In production, the base `compose.yml` file **is** the production shape
 already. There are no published ports, and everything is reachable only on
@@ -52,14 +53,38 @@ docker compose --env-file .env.prod -f compose.yml up -d
 
 A consuming project's own production compose joins `infra-net` as an
 external network to reach `postgres`, `zitadel`, `seaweedfs`, `rabbitmq` and
-`jaeger` by container name (see Musify's `deploy/compose.prod.yml`).
+`jaeger` by name (see Musify's `deploy/compose.prod.yml`).
+
+Every project on `infra-net` shares one DNS namespace, and Docker answers a
+service name with *every* container that has it, round-robin. Two projects
+each with a service called `api` would get each other's traffic. So anything
+that crosses projects (the edge, or one project calling into this stack)
+should use the unique `container_name` (`infra-postgres`, `musify-api`, ...),
+never a generic service name.
+
+### RabbitMQ and its hostname
+
+RabbitMQ keeps its data under its node name, `rabbit@<hostname>`, so the
+container has a fixed `hostname: rabbitmq`. Without it every recreated
+container (an image update, an `.env` change) came up with a new hostname
+and an empty broker. A deployment that ran before this was fixed has its
+data under the old random name; carry the definitions (users, vhosts,
+permissions, queues, exchanges) over when applying the change, ideally with
+the queues drained, since queued messages aren't part of the export:
+
+```bash
+docker exec infra-rabbitmq rabbitmqctl export_definitions /var/lib/rabbitmq/defs.json
+docker compose --env-file .env.prod -f compose.yml up -d rabbitmq
+docker exec infra-rabbitmq rabbitmqctl import_definitions /var/lib/rabbitmq/defs.json
+```
 
 ### Ports (development)
 
 | Service | Port | Notes |
 |---|---|---|
-| PostgreSQL | `5432` | one instance, one database per project |
+| PostgreSQL | `5432` | one instance, one database and role per project |
 | Zitadel | `8080` | OIDC/OAuth2, console at `/ui/console` |
+| Zitadel login | `3900` | Login V2 UI at `/ui/v2/login` (under `8080`'s hostname in prod) |
 | SeaweedFS | `8333` / `8888` / `9333` | S3 API / filer / master |
 | RabbitMQ | `5672` / `15672` | AMQP / management UI |
 | Jaeger | `16686` / `4317` / `4318` | UI / OTLP gRPC / OTLP HTTP |
@@ -72,28 +97,102 @@ running (in Docker or from the IDE). No shared Docker network is needed in
 development, which is what keeps every project's own `up` independent and
 fast, with this stack as the only prerequisite.
 
-### Shared secrets
+### Per-project credentials
 
-`POSTGRES_USER`/`POSTGRES_PASSWORD`, `S3_ACCESS_KEY`/`S3_SECRET_KEY`,
+No project gets an admin account. `provision` (a one-shot that runs on
-`RABBITMQ_DEFAULT_USER`/`RABBITMQ_DEFAULT_PASS` and `PUBLIC_AUTH_URL` (i.e.
+every `up`, see `provision/`) gives each project listed in `compose.yml`'s
-`http://host.docker.internal:${ZITADEL_EXTERNAL_PORT}` in dev) are the values
+`x-projects` its own credentials, from the `<PROJECT>_*` settings in the env
-every consuming project's own `.env.example` copies and documents as "must
+file:
-match this repo's `.env`". This file is the source of truth for them.
 
-### The Zitadel admin PAT
+| Service | What the project gets | Setting |
+|---|---|---|
+| PostgreSQL | role `<project>`, owner of database `<project>_db`; no other role can connect to it, and it can't connect to any other | `<PROJECT>_DB_PASSWORD` |
+| RabbitMQ | user `<project>`, only on vhost `<project>` | `<PROJECT>_RABBITMQ_PASSWORD` |
+| SeaweedFS | S3 identity limited to the project's bucket (it may create that bucket, nothing else) | `<PROJECT>_S3_ACCESS_KEY` / `_SECRET_KEY` |
+| Zitadel | service account `<project>-provisioner`, owner of the project's own Zitadel project only; its PAT is in `zitadel/.output/<project>/zitadel.pat` | (the project name, in `compose.yml`) |
 
-`zitadel-init` (in the compose file of whichever project runs it first)
+An empty setting means that project doesn't use that service. Each
-writes a service-account personal access token to `zitadel/.output/admin-sa.pat`
+project's own `.env.example` copies its values from here and says so; this
-on first init. Every consuming project mounts that same folder read-only, via
+repo's env file is the source of truth. Passwords are re-applied on every
-its own `ZITADEL_ADMIN_PAT_DIR` setting, to call the Zitadel management API
+`up`, so rotating one is: change it here and in the project, `up` both.
-and create its own project/apps.
+
+The admin accounts (`POSTGRES_*`, `RABBITMQ_DEFAULT_*`, `S3_ACCESS_KEY` /
+`S3_SECRET_KEY`, `zitadel/.output/admin-sa.pat`) stay with this stack.
+
+Two limits worth knowing: the Zitadel `PROJECT_OWNER` role can still read
+the organization's users (every project's users are in the one
+organization); and S3 keys must only use `A-Z a-z 0-9 _ + / = . -` (they go
+into SeaweedFS's JSON config as they are).
+
+**Adding a project**: add it to `x-projects`, add its `<PROJECT>_*` lines
+to `provision`'s and `seaweedfs`'s environment in `compose.yml` (with its
+bucket and Zitadel project name), put its values in `.env.example` /
+`.env.prod.example`, and `up`.
+
+**Moving an existing deployment over**: fill in the new `<PROJECT>_*`
+values and `up`. `provision` hands over the existing databases (and every
+table, sequence, type, view and function in them) to each project's role,
+reuses the existing Zitadel projects and apps, and leaves the buckets as
+they are. Then switch each project's env file to its own credentials. The
+only thing that doesn't carry over is RabbitMQ: a project now uses its own
+vhost, so drain its queues on `/` first; its consumers recreate their
+exchanges and queues in the new vhost.
+
+### One Zitadel for every project
+
+Every project signs its users in against the same instance and the same
+organization (the first one, `ZITADEL_ORG_NAME`), each with its own Zitadel
+project for its apps. So there is one account per person, not one per
+project: someone who registers in Musify can sign in to Ping with the same
+account, and a session in one app is reused by the others (single sign-on),
+because none of the projects requires a per-project grant or role to sign
+in. Keep it that way when adding a project: point its issuer at this
+Zitadel (`PUBLIC_AUTH_URL`), and don't create a separate organization for it.
+
+### The Zitadel PAT
+
+Each project's `zitadel-init` mounts **its own** folder,
+`zitadel/.output/<project>/`, read-only (via that project's
+`ZITADEL_PAT_DIR` setting) and uses the PAT in it to create its OIDC apps in
+its own Zitadel project. It can't see or change any other project's.
 
 That setting has no universally correct value. It's a path (relative or
 absolute) to wherever *you* cloned this repo on your machine, from the
 consuming project's `deploy/` folder. Each project's `.env.example` ships
 with an example value that assumes these repos happen to sit as sibling
-folders. If yours don't, change it. Not versioned, and regenerated whenever
+folders. If yours don't, change it. Not versioned; `provision` issues a new
-the Zitadel volume is reset.
+PAT whenever the one there stops working (e.g. after a Zitadel volume reset).
+
+### The Zitadel login (Login V2)
+
+Zitadel v4 serves its sign-in pages from a separate container,
+`zitadel-login` (Next.js), under `/ui/v2/login`. Zitadel redirects every
+login there, so it has to be running for anyone to sign in. It calls the
+Zitadel API with its own service account (`login-client`), whose PAT Zitadel
+writes on init to the `zitadel_bootstrap` volume, which only `zitadel-login`
+mounts.
+
+- **Dev**: no proxy, so the login has its own port (`ZITADEL_LOGIN_HOST_PORT`,
+  `3900`) and `ZITADEL_LOGIN_BASE_URL` points the browser at it.
+- **Prod**: the edge routes `/ui/v2/login` on Zitadel's hostname to
+  `infra-zitadel-login`, and everything else to Zitadel.
+- `zitadel` and `zitadel-login` are pinned to the same version; bump both
+  together.
+
+`ZITADEL_LOGIN_BASE_URL` and the `login-client` PAT are only applied when the
+instance is created. On an instance created before `zitadel-login` existed:
+
+1. In the console, create a service user `login-client`, give it the
+   instance role **IAM Login Client**, and create a PAT for it.
+2. Copy it into the volume and restart the login:
+
+   ```bash
+   docker cp login-client.pat infra-zitadel:/bootstrap/login-client.pat
+   docker restart infra-zitadel-login
+   ```
+
+3. If the instance still uses the old login, set Login V2 as required with
+   base URI `${ZITADEL_LOGIN_BASE_URL}/` in the instance's feature settings.
 
 ## The public edge
 
@@ -111,13 +210,23 @@ docker compose -f edge/compose.yml up -d
 - **Vhosts** live in `edge/conf.d/` (one file per project; `00-common.conf`
   has the shared resolver). Every upstream is a `set $upstream ...`, resolved
   per request, so a project's stack being down never stops the edge from
-  starting. Shared bits are in `edge/snippets/`. Check a change with
+  starting. Upstreams use the target's `container_name` (see "Running it"
+  above for why). Shared bits are in `edge/snippets/`. Check a change with
   `docker exec edge-nginx nginx -t`, then `docker exec edge-nginx nginx -s reload`.
+- **Unknown hostnames** hit `01-default.conf`: port 80 closes the connection
+  (ACME challenges still work), port 443 refuses the TLS handshake. Nothing
+  reaches a project unless its vhost names the host.
+- **TLS** settings (`snippets/tls.conf`) follow Mozilla's intermediate
+  profile and send HSTS with a 1-day `max-age`. Raise it once every hostname
+  is confirmed to work over https: browsers remember it, so a long value
+  can't be taken back quickly.
 - **Certificates** are state, not versioned (`edge/certbot/`). To issue one
   for new hostnames (their DNS has to already point at the host):
 
   ```bash
-  docker compose -f edge/compose.yml run --rm --entrypoint certbot certbot     certonly --webroot -w /var/www/certbot -d host1.example.com -d host2.example.com     --non-interactive --agree-tos -m you@example.com
+  docker compose -f edge/compose.yml run --rm --entrypoint certbot certbot \
+    certonly --webroot -w /var/www/certbot -d host1.example.com -d host2.example.com \
+    --non-interactive --agree-tos -m you@example.com
   ```
 
   Then add the `server` blocks pointing at
@@ -138,10 +247,10 @@ re-creates its piece on the next `up`, since both are idempotent.
 
 ```
 Infrastructure/
-├─ compose.yml            # postgres, zitadel, seaweedfs, rabbitmq, jaeger: the prod shape
+├─ compose.yml            # postgres, zitadel (+ login), seaweedfs, rabbitmq, jaeger: the prod shape
 ├─ edge/                  # shared public nginx + certbot (prod only): compose.yml, conf.d/, snippets/
 ├─ compose.dev.yml        # dev overlay: publishes ports, adds pgAdmin
 ├─ .env.example / .env.prod.example
-├─ postgres-init/         # creates the per-project application databases
+├─ provision/            # per-project credentials: Postgres, RabbitMQ, Zitadel, S3 identities
-└─ zitadel/.output/       # PAT written by Zitadel on init (not versioned)
+└─ zitadel/.output/       # admin PAT + one folder per project with its own PAT (not versioned)
 ```

compose.dev.yml

@@ -5,37 +5,49 @@ name: infra-dev
 # of joining infra-net. Adds pgAdmin.
 #
 #   docker compose -f compose.yml -f compose.dev.yml up -d
+#
+# Ports are bound to DEV_BIND_ADDRESS (127.0.0.1 by default), so the dev
+# credentials in .env are not reachable from the LAN. Other containers still
+# reach them through host.docker.internal. The browser on the host does too
+# as long as the hosts file maps host.docker.internal to 127.0.0.1; if yours
+# maps it to the LAN IP, set DEV_BIND_ADDRESS=0.0.0.0 (see .env.example).
 
 services:
   postgres:
     ports:
-      - "${POSTGRES_HOST_PORT}:5432"
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:${POSTGRES_HOST_PORT}:5432"
 
   zitadel:
     ports:
-      - "${ZITADEL_EXTERNAL_PORT}:8080"
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:${ZITADEL_EXTERNAL_PORT}:8080"
+
+  # No edge in dev to put the login under Zitadel's own port, so it gets its
+  # own (ZITADEL_LOGIN_BASE_URL in .env points the browser at it).
+  zitadel-login:
+    ports:
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:${ZITADEL_LOGIN_HOST_PORT}:3000"
 
   seaweedfs:
     ports:
-      - "9333:9333" # master / admin UI
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:9333:9333" # master / admin UI
-      - "8888:8888" # filer
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:8888:8888" # filer
-      - "8333:8333" # S3 API
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:8333:8333" # S3 API
 
   rabbitmq:
     ports:
-      - "5672:5672"   # AMQP
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:5672:5672"   # AMQP
-      - "15672:15672" # management UI
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:15672:15672" # management UI
 
   jaeger:
     ports:
-      - "16686:16686" # UI
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:16686:16686" # UI
-      - "4317:4317"   # OTLP gRPC
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:4317:4317"   # OTLP gRPC
-      - "4318:4318"   # OTLP HTTP
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:4318:4318"   # OTLP HTTP
 
   # Optional Postgres UI: docker compose ... --profile tools up -d
   pgadmin:
     profiles: [tools]
-    image: dpage/pgadmin4:latest
+    image: dpage/pgadmin4:9.18
     container_name: infra-pgadmin
     environment:
       PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
@@ -45,9 +57,15 @@ services:
       postgres:
         condition: service_healthy
     ports:
-      - "${PGADMIN_HOST_PORT}:80"
+      - "${DEV_BIND_ADDRESS:-127.0.0.1}:${PGADMIN_HOST_PORT}:80"
     volumes:
       - pgadmin_data:/var/lib/pgadmin
+    # Same rotation as compose.yml (YAML anchors don't cross files).
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
     restart: unless-stopped
 
 volumes:

compose.yml

@@ -7,22 +7,36 @@
 # of joining a network. See README.md for why.
 #
 #   dev   docker compose -f compose.yml -f compose.dev.yml up -d
-#   prod  docker compose --env-file .env.prod -f compose.yml -f compose.prod.yml up -d
+#   prod  docker compose --env-file .env.prod -f compose.yml up -d
 #
 # A consuming project's own deploy stack does NOT get its own copy of any of
 # these services. It only needs this stack to already be up, and its own
 # one-shot bootstrap jobs (zitadel-init, seaweedfs-init) to create its own
 # OIDC apps / bucket against the shared instances below.
 
+# Docker's default json-file driver never rotates, so logs would grow until
+# the disk fills up.
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "10m"
+    max-file: "3"
+
+# The consuming projects that get their own credentials (see provision/).
+# Adding one: add it here, give it the <PROJECT>_* settings in both
+# `provision` and `seaweedfs` below, and their values in the env files.
+x-projects: &projects musify ping dockermanager
+
 networks:
   infra-net:
     name: infra-net
 
 services:
   # --- Database ---------------------------------------------------------
-  # One instance, one database per consuming project (see postgres-init/).
+  # One instance, one database and one role per consuming project (created
-  # Zitadel provisions its own `zitadel` database itself, from the admin
+  # by `provision` below). Zitadel creates its own `zitadel` database and
-  # credentials below, so it does not need an init script.
+  # user itself, from the admin credentials. The superuser is only for this
+  # stack: no project gets it.
   postgres:
     image: postgres:17-alpine
     container_name: infra-postgres
@@ -31,13 +45,13 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
     volumes:
       - postgres_data:/var/lib/postgresql/data
-      - ./postgres-init:/docker-entrypoint-initdb.d:ro
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
       interval: 10s
       timeout: 5s
       retries: 10
     networks: [infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
   # --- Identity -----------------------------------------------------------
@@ -45,8 +59,13 @@ services:
   # setup (both idempotent) and then starts, so the same command works on a
   # cold and on a warm volume.
   zitadel:
-    image: ghcr.io/zitadel/zitadel:latest
+    # Pinned together with zitadel-login below: bump both at once.
+    image: ghcr.io/zitadel/zitadel:v4.19.1
     container_name: infra-zitadel
+    # The image runs as uid 1000, but Docker creates both the bind mount
+    # (on Linux) and the bootstrap volume as root, so Zitadel couldn't write
+    # the PATs into them. Same as Zitadel's own compose.
+    user: "0"
     # `disabled` (dev, plain http) makes Zitadel advertise http URLs; behind a
     # TLS-terminating proxy it must be `external` or the console calls
     # http://<domain> and the CSP blocks it.
@@ -69,47 +88,106 @@ services:
       ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: ${POSTGRES_USER}
       ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: ${POSTGRES_PASSWORD}
       ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable
-      # Initial console admin. Login: <username>@zitadel.<external domain>
+      # First organization and its console admin. Only read when the instance
+      # is created; changing them later doesn't rename anything that exists.
+      # A username without "@" gets `@<org domain>` appended; one with "@" is
+      # kept as is (e.g. admin@example.com).
+      ZITADEL_FIRSTINSTANCE_ORG_NAME: ${ZITADEL_ORG_NAME:-ZITADEL}
       ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME: ${ZITADEL_ADMIN_USERNAME}
+      ZITADEL_FIRSTINSTANCE_ORG_HUMAN_FIRSTNAME: ${ZITADEL_ADMIN_FIRST_NAME:-ZITADEL}
+      ZITADEL_FIRSTINSTANCE_ORG_HUMAN_LASTNAME: ${ZITADEL_ADMIN_LAST_NAME:-Admin}
+      ZITADEL_FIRSTINSTANCE_ORG_HUMAN_DISPLAYNAME: ${ZITADEL_ADMIN_DISPLAY_NAME:-ZITADEL Admin}
       ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD: ${ZITADEL_ADMIN_PASSWORD}
       ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED: "false"
-      # Service account + PAT written on init; each consuming project's own
+      # Instance-admin service account + PAT written on init. Only `provision`
-      # zitadel-init one-shot uses it to create its own project/apps.
+      # uses it, to give each project its own, project-scoped account.
       ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME: zitadel-admin-sa
       ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME: Admin Service Account
       ZITADEL_FIRSTINSTANCE_ORG_MACHINE_PAT_EXPIRATIONDATE: "2099-01-01T00:00:00Z"
       ZITADEL_FIRSTINSTANCE_PATPATH: /output/admin-sa.pat
+      # Service account the login UI (zitadel-login) calls the API with. Its
+      # PAT goes to a volume only that container mounts, not to /output.
+      ZITADEL_FIRSTINSTANCE_ORG_LOGINCLIENT_MACHINE_USERNAME: login-client
+      ZITADEL_FIRSTINSTANCE_ORG_LOGINCLIENT_MACHINE_NAME: Automatically Initialized IAM_LOGIN_CLIENT
+      ZITADEL_FIRSTINSTANCE_ORG_LOGINCLIENT_PAT_EXPIRATIONDATE: "2099-01-01T00:00:00Z"
+      ZITADEL_FIRSTINSTANCE_LOGINCLIENTPATPATH: /bootstrap/login-client.pat
+      # Login V2: sign-in pages are served by zitadel-login, not by Zitadel
+      # itself. Like the FIRSTINSTANCE values, these are only read when the
+      # instance is created (see README for an existing instance).
+      ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_REQUIRED: "true"
+      ZITADEL_DEFAULTINSTANCE_FEATURES_LOGINV2_BASEURI: ${ZITADEL_LOGIN_BASE_URL}/
+      ZITADEL_OIDC_DEFAULTLOGINURLV2: ${ZITADEL_LOGIN_BASE_URL}/login?authRequest=
+      ZITADEL_OIDC_DEFAULTLOGOUTURLV2: ${ZITADEL_LOGIN_BASE_URL}/logout?post_logout_redirect=
+      ZITADEL_SAML_DEFAULTLOGINURLV2: ${ZITADEL_LOGIN_BASE_URL}/login?samlRequest=
     depends_on:
       postgres:
         condition: service_healthy
     volumes:
       - ./zitadel/.output:/output
+      - zitadel_bootstrap:/bootstrap
+    healthcheck:
+      test: ["CMD", "/app/zitadel", "ready"]
+      interval: 10s
+      timeout: 30s
+      retries: 12
+      start_period: 20s
     networks: [infra-net]
+    logging: *default-logging
+    restart: unless-stopped
+
+  # Login V2 UI (Next.js), served under /ui/v2/login. It calls the API
+  # directly on infra-net, so it has to present the public Host/scheme for
+  # Zitadel to find the instance.
+  zitadel-login:
+    image: ghcr.io/zitadel/zitadel-login:v4.19.1
+    container_name: infra-zitadel-login
+    environment:
+      ZITADEL_API_URL: http://zitadel:8080
+      NEXT_PUBLIC_BASE_PATH: /ui/v2/login
+      ZITADEL_SERVICE_USER_TOKEN_FILE: /bootstrap/login-client.pat
+      CUSTOM_REQUEST_HEADERS: Host:${ZITADEL_EXTERNAL_DOMAIN},X-Forwarded-Proto:${ZITADEL_PUBLIC_SCHEME}
+    healthcheck:
+      test: ["CMD", "/bin/sh", "-c", "node /app/healthcheck.mjs http://localhost:3000/ui/v2/login/healthy"]
+      interval: 10s
+      timeout: 30s
+      retries: 12
+      start_period: 20s
+    depends_on:
+      zitadel:
+        condition: service_healthy
+    volumes:
+      - zitadel_bootstrap:/bootstrap:ro
+    networks: [infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
   # --- Storage --------------------------------------------------------------
   # S3-compatible object store shared by every project. All-in-one: master +
-  # volume + filer + S3 gateway. The S3 identity file is rendered from the
+  # volume + filer + S3 gateway. The S3 identities are rendered from the
-  # environment on start so the credentials can never drift from the ones
+  # environment on start (provision/seaweedfs-s3-config.sh): an admin one
-  # handed to consuming projects. No bucket is created here; each
+  # for this stack, and one per project limited to that project's bucket.
-  # consuming project's own seaweedfs-init one-shot creates its own.
+  # No bucket is created here; each consuming project's own seaweedfs-init
+  # one-shot creates its own.
   seaweedfs:
-    image: chrislusf/seaweedfs:latest
+    image: chrislusf/seaweedfs:4.47
     container_name: infra-seaweedfs
+    environment:
+      PROJECTS: *projects
+      S3_ACCESS_KEY: ${S3_ACCESS_KEY}
+      S3_SECRET_KEY: ${S3_SECRET_KEY}
+      MUSIFY_S3_ACCESS_KEY: ${MUSIFY_S3_ACCESS_KEY:-}
+      MUSIFY_S3_SECRET_KEY: ${MUSIFY_S3_SECRET_KEY:-}
+      MUSIFY_S3_BUCKET: webapi-storage
+      PING_S3_ACCESS_KEY: ${PING_S3_ACCESS_KEY:-}
+      PING_S3_SECRET_KEY: ${PING_S3_SECRET_KEY:-}
+      PING_S3_BUCKET: ping-storage
+      DOCKERMANAGER_S3_ACCESS_KEY: ${DOCKERMANAGER_S3_ACCESS_KEY:-}
+      DOCKERMANAGER_S3_SECRET_KEY: ${DOCKERMANAGER_S3_SECRET_KEY:-}
+      DOCKERMANAGER_S3_BUCKET: dockermanager
     entrypoint: ["/bin/sh", "-c"]
     command:
       - |
-        cat > /tmp/s3.conf <<'JSON'
+        sh /provision/seaweedfs-s3-config.sh >/tmp/s3.conf || exit 1
-        {
-          "identities": [
-            {
-              "name": "infra",
-              "credentials": [{ "accessKey": "${S3_ACCESS_KEY}", "secretKey": "${S3_SECRET_KEY}" }],
-              "actions": ["Admin", "Read", "Write"]
-            }
-          ]
-        }
-        JSON
         exec weed server \
           -ip.bind=0.0.0.0 \
           -dir=/data \
@@ -120,12 +198,14 @@ services:
           -s3.config=/tmp/s3.conf
     volumes:
       - seaweedfs_data:/data
+      - ./provision:/provision:ro
     healthcheck:
       test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9333/cluster/healthz"]
       interval: 10s
       timeout: 5s
       retries: 10
     networks: [infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
   # --- Messaging --------------------------------------------------------
@@ -134,6 +214,10 @@ services:
   rabbitmq:
     image: rabbitmq:4-management-alpine
     container_name: infra-rabbitmq
+    # The node is named rabbit@<hostname> and its data lives under that name.
+    # Without a fixed hostname every recreated container gets a new one and
+    # starts on an empty data directory (users, vhosts, queues all gone).
+    hostname: rabbitmq
     environment:
       RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
       RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
@@ -145,19 +229,72 @@ services:
       timeout: 5s
       retries: 10
     networks: [infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
   # --- Tracing --------------------------------------------------------------
   # OTLP collector + trace UI, shared by any project's OpenTelemetry exporter.
+  # Last Jaeger v1 release (v1 is deprecated; v2 is jaegertracing/jaeger).
   jaeger:
-    image: jaegertracing/all-in-one:latest
+    image: jaegertracing/all-in-one:1.76.0
     container_name: infra-jaeger
     environment:
       COLLECTOR_OTLP_ENABLED: "true"
+      # Traces are kept in memory; without a cap they grow until the
+      # container runs out of RAM. The oldest are dropped first.
+      MEMORY_MAX_TRACES: "100000"
+    healthcheck:
+      # Admin port: answers 200 once the collector and query service are up.
+      test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:14269/"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
     networks: [infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
+  # --- Per-project credentials ----------------------------------------------
+  # One-shot, runs on every `up` (idempotent): each project's own Postgres
+  # role/database, RabbitMQ user/vhost and Zitadel service account, from the
+  # <PROJECT>_* settings (see provision/provision.sh). An empty setting skips
+  # that service for that project.
+  provision:
+    image: postgres:17-alpine
+    container_name: infra-provision
+    entrypoint: ["/bin/sh", "/provision/provision.sh"]
+    environment:
+      PROJECTS: *projects
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
+      ZITADEL_EXTERNAL_DOMAIN: ${ZITADEL_EXTERNAL_DOMAIN}
+      ZITADEL_PUBLIC_SCHEME: ${ZITADEL_PUBLIC_SCHEME}
+      MUSIFY_DB_PASSWORD: ${MUSIFY_DB_PASSWORD:-}
+      MUSIFY_RABBITMQ_PASSWORD: ${MUSIFY_RABBITMQ_PASSWORD:-}
+      MUSIFY_ZITADEL_PROJECT: Musify
+      PING_DB_PASSWORD: ${PING_DB_PASSWORD:-}
+      PING_RABBITMQ_PASSWORD: ${PING_RABBITMQ_PASSWORD:-}
+      PING_ZITADEL_PROJECT: Ping
+      DOCKERMANAGER_DB_PASSWORD: ${DOCKERMANAGER_DB_PASSWORD:-}
+      DOCKERMANAGER_RABBITMQ_PASSWORD: ${DOCKERMANAGER_RABBITMQ_PASSWORD:-}
+      DOCKERMANAGER_ZITADEL_PROJECT: Docker Manager
+    volumes:
+      - ./provision:/provision:ro
+      - ./zitadel/.output:/output
+    depends_on:
+      postgres:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+      zitadel:
+        condition: service_healthy
+    networks: [infra-net]
+    logging: *default-logging
+    restart: "no"
+
 volumes:
   postgres_data:
+  zitadel_bootstrap:
   seaweedfs_data:
   rabbitmq_data:

edge/compose.yml

@@ -10,6 +10,14 @@
 
 name: edge
 
+# Docker's default json-file driver never rotates, so logs would grow until
+# the disk fills up.
+x-logging: &default-logging
+  driver: json-file
+  options:
+    max-size: "10m"
+    max-file: "3"
+
 networks:
   infra-net:
     name: infra-net
@@ -36,10 +44,11 @@ services:
       - ./certbot/www:/var/www/certbot:ro
       - ./certbot/conf:/etc/letsencrypt:ro
     networks: [gitea, infra-net]
+    logging: *default-logging
     restart: unless-stopped
 
   certbot:
-    image: certbot/certbot
+    image: certbot/certbot:v5.8.0
     container_name: edge-certbot
     entrypoint:
       - /bin/sh
@@ -48,4 +57,5 @@ services:
     volumes:
       - ./certbot/conf:/etc/letsencrypt
       - ./certbot/www:/var/www/certbot
+    logging: *default-logging
     restart: unless-stopped

edge/conf.d/01-default.conf

@@ -0,0 +1,25 @@
+# Catch-all for Host headers no vhost claims (scanners, direct IP access,
+# stale DNS). Without it nginx hands them to the first vhost it loaded,
+# i.e. Gitea. ACME stays reachable here so a certificate for a new hostname
+# can be issued before its vhost exists.
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+    server_name _;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 444;
+    }
+}
+
+# No certificate to present for an unknown name, so refuse the handshake.
+server {
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+    server_name _;
+    ssl_reject_handshake on;
+}

edge/conf.d/musify.conf

@@ -15,12 +15,19 @@ server {
     server_name musify.ikerdemo.cat;
     client_max_body_size 1024m;
 
+    # The OIDC callback answers with the session cookies (encrypted tokens),
+    # bigger than the 4k default response-header buffer.
+    proxy_buffer_size 32k;
+    proxy_buffers 8 32k;
+    proxy_busy_buffers_size 64k;
+    large_client_header_buffers 4 16k;
+
     ssl_certificate     /etc/letsencrypt/live/musify.ikerdemo.cat/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem;
     include /etc/nginx/snippets/tls.conf;
 
     location / {
-        set $upstream http://web-player:3000;
+        set $upstream http://musify-web-player:3000;
         proxy_pass $upstream;
         include /etc/nginx/snippets/proxy.conf;
     }
@@ -37,7 +44,7 @@ server {
     include /etc/nginx/snippets/tls.conf;
 
     location / {
-        set $upstream http://api:5111;
+        set $upstream http://musify-api:5111;
         proxy_pass $upstream;
         include /etc/nginx/snippets/proxy.conf;
     }
@@ -55,7 +62,7 @@ server {
     include /etc/nginx/snippets/tls.conf;
 
     location / {
-        set $upstream http://gateway:8081;
+        set $upstream http://musify-gateway:8081;
         proxy_pass $upstream;
         include /etc/nginx/snippets/proxy.conf;
         proxy_buffering off;
@@ -73,11 +80,17 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/musify.ikerdemo.cat/privkey.pem;
     include /etc/nginx/snippets/tls.conf;
 
+    # Login V2 UI, a separate container (see ../compose.yml).
+    location /ui/v2/login {
+        set $upstream http://infra-zitadel-login:3000;
+        proxy_pass $upstream;
+        include /etc/nginx/snippets/proxy.conf;
+    }
+
     location / {
-        set $upstream http://zitadel:8080;
+        set $upstream http://infra-zitadel:8080;
         proxy_pass $upstream;
         include /etc/nginx/snippets/proxy.conf;
-        grpc_set_header Host $host;
     }
 }
 
@@ -94,7 +107,7 @@ server {
     include /etc/nginx/snippets/tls.conf;
 
     location / {
-        set $upstream http://seaweedfs:8333;
+        set $upstream http://infra-seaweedfs:8333;
         proxy_pass $upstream;
         include /etc/nginx/snippets/proxy.conf;
         proxy_request_buffering off;

edge/snippets/proxy.conf

@@ -1,7 +1,9 @@
 proxy_http_version 1.1;
 proxy_set_header Host $host;
 proxy_set_header X-Real-IP $remote_addr;
-proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+# The edge is the first hop, so whatever X-Forwarded-For the client sent is
+# made up: replace it instead of appending to it.
+proxy_set_header X-Forwarded-For $remote_addr;
 proxy_set_header X-Forwarded-Proto $scheme;
 proxy_set_header Upgrade $http_upgrade;
 proxy_set_header Connection $connection_upgrade;

edge/snippets/tls.conf

@@ -1,2 +1,13 @@
+# Mozilla "intermediate" profile, without the DHE suites (they'd need a
+# dhparam file): forward secrecy on every connection, no CBC.
 ssl_protocols TLSv1.2 TLSv1.3;
-ssl_ciphers HIGH:!aNULL:!MD5;
+ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305;
+ssl_prefer_server_ciphers off;
+ssl_session_cache shared:SSL:10m;
+ssl_session_timeout 1d;
+ssl_session_tickets off;
+
+# Short max-age for now: raise it (e.g. 63072000) once every hostname is
+# confirmed working over https. A location that sets its own add_header
+# stops inheriting this one.
+add_header Strict-Transport-Security "max-age=86400" always;

postgres-init/010-create-app-databases.sql

@@ -1,9 +0,0 @@
--- Runs once, only on first init of the postgres_data volume.
--- Zitadel provisions its own "zitadel" database itself, from the
--- ZITADEL_DATABASE_POSTGRES_ADMIN_* credentials in compose.yml, so it does
--- not need to be created here. This script only creates the plain
--- application databases each project's own connection string points at.
-
-CREATE DATABASE musify_db;
-CREATE DATABASE ping_db;
-CREATE DATABASE dockermanager_db;

provision/provision.sh

@@ -0,0 +1,227 @@
+#!/bin/sh
+# Gives every consuming project its own credentials on the shared services,
+# so no project holds another's data or the admin accounts:
+#
+#   Postgres  role <project>, owner of database <project>_db, which no
+#             other role can connect to
+#   RabbitMQ  user <project> with permissions only on vhost <project>
+#   Zitadel   service account <project>-provisioner, PROJECT_OWNER of the
+#             project's own Zitadel project only; its PAT is written to
+#             /output/<project>/zitadel.pat for that project's zitadel-init
+#
+# (S3 identities are rendered into SeaweedFS's own config when it starts, see
+# seaweedfs-s3-config.sh.)
+#
+# Idempotent: runs on every `up`. A service is only provisioned for a project
+# when its setting is non-empty (no MUSIFY_RABBITMQ_PASSWORD, no RabbitMQ
+# user for Musify). Passwords are re-applied on every run, so changing one
+# in the env file and running `up` again rotates it.
+set -eu -o pipefail
+
+apk add --no-cache curl jq >/dev/null
+
+# Value of the <PROJECT>_<suffix> setting, empty when unset.
+setting() { eval "printf '%s' \"\${$(echo "$1" | tr '[:lower:]' '[:upper:]')_$2:-}\""; }
+
+# --- Postgres ----------------------------------------------------------------
+export PGHOST=postgres PGUSER="$POSTGRES_USER" PGPASSWORD="$POSTGRES_PASSWORD"
+
+# By default every role may connect to every database (PUBLIC has CONNECT),
+# including `zitadel` and `postgres`. Only the owner, roles granted it
+# explicitly (Zitadel's own user on `zitadel`) and the superuser keep it.
+lock_down_postgres() {
+  psql -q -v ON_ERROR_STOP=1 -d postgres <<'SQL'
+SELECT format('REVOKE CONNECT, TEMPORARY ON DATABASE %I FROM PUBLIC', datname)
+FROM pg_database WHERE NOT datistemplate \gexec
+SQL
+  echo "  postgres: no database open to PUBLIC"
+}
+
+provision_postgres() {
+  local project=$1 password=$2 db="$1_db"
+  psql -q -v ON_ERROR_STOP=1 -d postgres -v role="$project" -v pw="$password" -v db="$db" <<'SQL'
+SELECT format('CREATE ROLE %I LOGIN', :'role')
+WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'role') \gexec
+ALTER ROLE :"role" WITH LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE PASSWORD :'pw';
+SELECT format('CREATE DATABASE %I OWNER %I', :'db', :'role')
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = :'db') \gexec
+ALTER DATABASE :"db" OWNER TO :"role";
+REVOKE CONNECT, TEMPORARY ON DATABASE :"db" FROM PUBLIC;
+SQL
+
+  # A database from before this (created by the superuser, as were the
+  # tables its migrations made) is handed over: the project's migrations
+  # have to own what they alter.
+  psql -q -v ON_ERROR_STOP=1 -d "$db" -v role="$project" <<'SQL' >/dev/null
+SELECT set_config('provision.role', :'role', false);
+DO $$
+DECLARE
+  target text := current_setting('provision.role');
+  r record;
+BEGIN
+  FOR r IN
+    SELECT n.nspname FROM pg_namespace n
+    WHERE n.nspname NOT LIKE 'pg\_%' AND n.nspname <> 'information_schema'
+      AND pg_get_userbyid(n.nspowner) NOT IN (target, 'pg_database_owner')
+      AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_namespace'::regclass AND d.objid = n.oid AND d.deptype = 'e')
+  LOOP
+    EXECUTE format('ALTER SCHEMA %I OWNER TO %I', r.nspname, target);
+  END LOOP;
+
+  -- Indexes and the sequences behind serial/identity columns follow their
+  -- table; extension members stay with the extension.
+  FOR r IN
+    SELECT c.oid::regclass AS obj, c.relkind FROM pg_class c
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    WHERE n.nspname NOT LIKE 'pg\_%' AND n.nspname <> 'information_schema'
+      AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
+      AND pg_get_userbyid(c.relowner) <> target
+      AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid AND d.deptype IN ('a', 'i', 'e'))
+  LOOP
+    EXECUTE format('ALTER %s %s OWNER TO %I',
+      CASE r.relkind WHEN 'v' THEN 'VIEW' WHEN 'm' THEN 'MATERIALIZED VIEW'
+                     WHEN 'S' THEN 'SEQUENCE' WHEN 'f' THEN 'FOREIGN TABLE' ELSE 'TABLE' END,
+      r.obj, target);
+  END LOOP;
+
+  -- Enums, domains, composite and range types (not a table's row type nor
+  -- array types, which follow their element type).
+  FOR r IN
+    SELECT t.oid::regtype AS obj, t.typtype FROM pg_type t
+    JOIN pg_namespace n ON n.oid = t.typnamespace
+    WHERE n.nspname NOT LIKE 'pg\_%' AND n.nspname <> 'information_schema'
+      AND t.typtype IN ('e', 'd', 'c', 'r', 'm') AND t.typelem = 0
+      AND (t.typrelid = 0 OR (SELECT relkind FROM pg_class WHERE oid = t.typrelid) = 'c')
+      AND pg_get_userbyid(t.typowner) <> target
+      AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_type'::regclass AND d.objid = t.oid AND d.deptype IN ('i', 'e'))
+  LOOP
+    EXECUTE format('ALTER %s %s OWNER TO %I', CASE r.typtype WHEN 'd' THEN 'DOMAIN' ELSE 'TYPE' END, r.obj, target);
+  END LOOP;
+
+  FOR r IN
+    SELECT p.oid::regprocedure AS obj FROM pg_proc p
+    JOIN pg_namespace n ON n.oid = p.pronamespace
+    WHERE n.nspname NOT LIKE 'pg\_%' AND n.nspname <> 'information_schema'
+      AND pg_get_userbyid(p.proowner) <> target
+      AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_proc'::regclass AND d.objid = p.oid AND d.deptype = 'e')
+  LOOP
+    EXECUTE format('ALTER ROUTINE %s OWNER TO %I', r.obj, target);
+  END LOOP;
+END $$;
+SQL
+  echo "  postgres: role $project owns database $db"
+}
+
+# --- RabbitMQ ----------------------------------------------------------------
+rabbitmq_put() {
+  curl -fsS -o /dev/null -X PUT "http://rabbitmq:15672/api$1" \
+    -u "$RABBITMQ_DEFAULT_USER:$RABBITMQ_DEFAULT_PASS" \
+    -H "Content-Type: application/json" --data-raw "$2"
+}
+
+provision_rabbitmq() {
+  local project=$1 password=$2
+  rabbitmq_put "/vhosts/$project" '{}'
+  rabbitmq_put "/users/$project" "$(jq -n --arg pw "$password" '{password: $pw, tags: ""}')"
+  rabbitmq_put "/permissions/$project/$project" '{"configure": ".*", "write": ".*", "read": ".*"}'
+  echo "  rabbitmq: user $project, only on vhost $project"
+}
+
+# --- Zitadel -----------------------------------------------------------------
+# zitadel <token> <method> <path> [body]: prints the response body; on a
+# non-2xx answer also reports it on stderr and returns non-zero. It calls
+# Zitadel on infra-net, so it presents the public Host/scheme for Zitadel to
+# find the instance (same as zitadel-login).
+zitadel() {
+  local response code
+  response=$(curl -sS -w '\n%{http_code}' -X "$2" "http://zitadel:8080$3" \
+    -H "Authorization: Bearer $1" -H "Content-Type: application/json" \
+    -H "Host: $ZITADEL_EXTERNAL_DOMAIN" -H "X-Forwarded-Proto: $ZITADEL_PUBLIC_SCHEME" \
+    ${4+--data-raw "$4"})
+  code=$(printf '%s\n' "$response" | tail -n1)
+  response=$(printf '%s\n' "$response" | sed '$d')
+  printf '%s\n' "$response"
+  case "$code" in
+    2??) return 0 ;;
+    *) printf 'zitadel: %s %s -> %s %s\n' "$2" "$3" "$code" "$response" >&2; return 1 ;;
+  esac
+}
+
+# Stops the run when a lookup or creation didn't produce a value.
+need() { [ -n "$2" ] && [ "$2" != null ] || { echo "zitadel: got no $1" >&2; exit 1; }; }
+
+provision_zitadel() {
+  local project=$1 zitadel_project=$2 username="$1-provisioner"
+  local project_id user_id member_roles pat_file token state
+
+  project_id=$(zitadel "$ADMIN_PAT" POST /management/v1/projects/_search \
+    "$(jq -n --arg n "$zitadel_project" '{queries: [{nameQuery: {name: $n, method: "TEXT_QUERY_METHOD_EQUALS"}}]}')" \
+    | jq -r '.result // [] | first | .id // empty')
+  if [ -z "$project_id" ]; then
+    project_id=$(zitadel "$ADMIN_PAT" POST /management/v1/projects "$(jq -n --arg n "$zitadel_project" '{name: $n}')" | jq -r .id)
+  fi
+  need "project id for $zitadel_project" "$project_id"
+
+  user_id=$(zitadel "$ADMIN_PAT" POST /management/v1/users/_search \
+    "$(jq -n --arg u "$username" '{queries: [{userNameQuery: {userName: $u, method: "TEXT_QUERY_METHOD_EQUALS"}}]}')" \
+    | jq -r '.result // [] | first | .id // empty')
+  if [ -z "$user_id" ]; then
+    user_id=$(zitadel "$ADMIN_PAT" POST /management/v1/users/machine \
+      "$(jq -n --arg u "$username" --arg p "$zitadel_project" \
+        '{userName: $u, name: ($p + " provisioner"), description: ("zitadel-init of " + $p + ": manages that project only"), accessTokenType: "ACCESS_TOKEN_TYPE_BEARER"}')" \
+      | jq -r .userId)
+  fi
+  need "user id for $username" "$user_id"
+
+  # Owner of this one project (its apps, roles and grants) and nothing else.
+  member_roles=$(zitadel "$ADMIN_PAT" POST "/management/v1/projects/$project_id/members/_search" \
+    "$(jq -n --arg u "$user_id" '{queries: [{userIdQuery: {userId: $u}}]}')" \
+    | jq -c '.result // [] | first | .roles // empty')
+  if [ -z "$member_roles" ]; then
+    zitadel "$ADMIN_PAT" POST "/management/v1/projects/$project_id/members" \
+      "$(jq -n --arg u "$user_id" '{userId: $u, roles: ["PROJECT_OWNER"]}')" >/dev/null
+  elif [ "$member_roles" != '["PROJECT_OWNER"]' ]; then
+    zitadel "$ADMIN_PAT" PUT "/management/v1/projects/$project_id/members/$user_id" '{"roles": ["PROJECT_OWNER"]}' >/dev/null
+  fi
+
+  # Keep a PAT that still works. A new one (they can't be read back) is only
+  # issued when there is none, or it was revoked or lost with a volume reset.
+  pat_file="/output/$project/zitadel.pat"
+  mkdir -p "/output/$project"
+  chmod 700 "/output/$project"
+  if [ -s "$pat_file" ] \
+    && zitadel "$(tr -d '\r\n' <"$pat_file")" GET /auth/v1/users/me 2>/dev/null \
+      | jq -e --arg u "$username" '.user.userName == $u' >/dev/null 2>&1; then
+    state="kept"
+  else
+    token=$(zitadel "$ADMIN_PAT" POST "/management/v1/users/$user_id/pats" '{"expirationDate": "2099-01-01T00:00:00Z"}' | jq -r .token)
+    need "PAT for $username" "$token"
+    (umask 077 && printf '%s\n' "$token" >"$pat_file.tmp")
+    mv "$pat_file.tmp" "$pat_file"
+    state="issued"
+  fi
+  echo "  zitadel: $username owns project \"$zitadel_project\", PAT $state"
+}
+
+# -----------------------------------------------------------------------------
+echo "==> waiting for Zitadel's admin PAT"
+deadline=$(( $(date +%s) + 180 ))
+until [ -s /output/admin-sa.pat ]; do
+  [ "$(date +%s)" -lt "$deadline" ] || { echo "no /output/admin-sa.pat; check 'docker logs infra-zitadel'" >&2; exit 1; }
+  sleep 3
+done
+ADMIN_PAT=$(tr -d '\r\n' </output/admin-sa.pat)
+
+echo "==> shared"
+lock_down_postgres
+
+for project in $PROJECTS; do
+  echo "==> $project"
+  value=$(setting "$project" DB_PASSWORD)
+  [ -z "$value" ] || provision_postgres "$project" "$value"
+  value=$(setting "$project" RABBITMQ_PASSWORD)
+  [ -z "$value" ] || provision_rabbitmq "$project" "$value"
+  value=$(setting "$project" ZITADEL_PROJECT)
+  [ -z "$value" ] || provision_zitadel "$project" "$value"
+done
+echo "==> done"

provision/seaweedfs-s3-config.sh

@@ -0,0 +1,60 @@
+#!/bin/sh
+# Prints SeaweedFS's S3 identity config (JSON) from the environment:
+#
+#   infra      S3_ACCESS_KEY / S3_SECRET_KEY, admin on every bucket (for
+#              operating the stack, not for projects)
+#   <project>  <PROJECT>_S3_ACCESS_KEY / _SECRET_KEY, limited to the bucket
+#              <PROJECT>_S3_BUCKET (it may create it, nothing else)
+#
+# A project without keys gets no identity. Runs in the SeaweedFS container on
+# every start, so the credentials can't drift from the env file.
+set -eu
+
+upper() { echo "$1" | tr '[:lower:]' '[:upper:]'; }
+setting() { eval "printf '%s' \"\${$(upper "$1")_$2:-}\""; }
+
+# The values go into JSON as they are: keep them to characters that need no
+# escaping (what `openssl rand -hex` / -base64 produce).
+check() {
+  case "$2" in
+    '' | *[!A-Za-z0-9_+/=.-]*) echo "seaweedfs-s3-config: $1 must be non-empty and only use A-Z a-z 0-9 _ + / = . -" >&2; exit 1 ;;
+  esac
+}
+
+identity() {
+  printf '    {"name": "%s", "credentials": [{"accessKey": "%s", "secretKey": "%s"}], "actions": [%s]}' "$1" "$2" "$3" "$4"
+}
+
+# SeaweedFS refuses to start (and keeps restarting) on a repeated access key,
+# e.g. two values left at CHANGE_ME: say which ones instead.
+seen=" "
+unique() {
+  case "$seen" in
+    *" $2 "*) echo "seaweedfs-s3-config: $1 repeats an access key already used above; every identity needs its own" >&2; exit 1 ;;
+  esac
+  seen="$seen$2 "
+}
+
+check S3_ACCESS_KEY "$S3_ACCESS_KEY"
+unique S3_ACCESS_KEY "$S3_ACCESS_KEY"
+check S3_SECRET_KEY "$S3_SECRET_KEY"
+printf '{\n  "identities": [\n'
+identity infra "$S3_ACCESS_KEY" "$S3_SECRET_KEY" '"Admin", "Read", "Write"'
+
+for project in $PROJECTS; do
+  access=$(setting "$project" S3_ACCESS_KEY)
+  [ -n "$access" ] || continue
+  secret=$(setting "$project" S3_SECRET_KEY)
+  bucket=$(setting "$project" S3_BUCKET)
+  prefix=$(upper "$project")
+  check "${prefix}_S3_ACCESS_KEY" "$access"
+  check "${prefix}_S3_SECRET_KEY" "$secret"
+  check "${prefix}_S3_BUCKET" "$bucket"
+  unique "${prefix}_S3_ACCESS_KEY" "$access"
+  # Admin:<bucket> is what lets it create (and configure) that one bucket.
+  printf ',\n'
+  identity "$project" "$access" "$secret" \
+    "\"Admin:$bucket\", \"Read:$bucket\", \"Write:$bucket\", \"List:$bucket\", \"Tagging:$bucket\""
+  echo "seaweedfs-s3-config: $project -> bucket $bucket" >&2
+done
+printf '\n  ]\n}\n'