From 1d40abcab7e9bde1ebd907b43b5dd3f097a3c6cc Mon Sep 17 00:00:00 2001 From: XShine555 Date: Sun, 5 Jul 2026 20:32:40 +0200 Subject: [PATCH] feat(deploy): consolidated dev stack with auto-provisioned Zitadel Single docker-compose stack for local development (Postgres, Zitadel, RabbitMQ, SeaweedFS + bucket init, Jaeger) plus an optional overlay to run the .NET apps in Docker. up.ps1 orchestrates the full bootstrap: stack up, stream-ticket keys, EF Core migrations and Zitadel provisioning. - Zitadel is provisioned automatically (service-account PAT -> project + OIDC app with JWT access token); the generated ClientId is wired into Musify.Api user-secrets and .env. No manual console steps. - ZITADEL_EXTERNAL_DOMAIN=host.docker.internal so token validation works both with local apps and full-docker. - App Dockerfiles copy AppSettings*.json to lowercase so config loads on case-sensitive Linux; Worker image ships ffmpeg. - Scripts split by responsibility (common/stream-keys/migrate/provision); apps overlay de-duped via YAML anchors. - Remove the superseded standalone seaweedfs compose; point docs/development at the unified stack. Co-Authored-By: Claude Opus 4.8 --- backend/.dockerignore | 10 ++ backend/Musify.Api.Dockerfile | 25 +++ backend/Musify.StreamingGateway.Dockerfile | 17 ++ backend/Musify.Worker.Dockerfile | 25 +++ deploy/.env.example | 35 ++++ deploy/README.md | 192 +++++++++++++++++++++ deploy/docker-compose.apps.yml | 96 +++++++++++ deploy/docker-compose.yml | 146 ++++++++++++++++ deploy/scripts/common.ps1 | 15 ++ deploy/scripts/migrate.ps1 | 15 ++ deploy/scripts/stream-keys.ps1 | 28 +++ deploy/seaweedfs/docker-compose.yml | 32 ---- deploy/up.ps1 | 121 +++++++++++++ deploy/zitadel/.gitignore | 2 + deploy/zitadel/provision.ps1 | 121 +++++++++++++ docs/development.md | 33 ++-- 16 files changed, 865 insertions(+), 48 deletions(-) create mode 100644 backend/.dockerignore create mode 100644 backend/Musify.Api.Dockerfile create mode 100644 backend/Musify.StreamingGateway.Dockerfile create mode 100644 backend/Musify.Worker.Dockerfile create mode 100644 deploy/.env.example create mode 100644 deploy/README.md create mode 100644 deploy/docker-compose.apps.yml create mode 100644 deploy/docker-compose.yml create mode 100644 deploy/scripts/common.ps1 create mode 100644 deploy/scripts/migrate.ps1 create mode 100644 deploy/scripts/stream-keys.ps1 delete mode 100644 deploy/seaweedfs/docker-compose.yml create mode 100644 deploy/up.ps1 create mode 100644 deploy/zitadel/.gitignore create mode 100644 deploy/zitadel/provision.ps1 diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..32afcf6 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,10 @@ +**/bin/ +**/obj/ +**/.vs/ +**/.idea/ +.git/ +.github/ +.agents/ +.claude/ +**/DesignSettings.json +*.user diff --git a/backend/Musify.Api.Dockerfile b/backend/Musify.Api.Dockerfile new file mode 100644 index 0000000..1afac7f --- /dev/null +++ b/backend/Musify.Api.Dockerfile @@ -0,0 +1,25 @@ +# Build context: backend/ (docker compose -f deploy/docker-compose.apps.yml build) +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src + +# Restore: copy central config + csproj files first to cache the layer. +COPY Directory.Build.props Directory.Packages.props nuget.config ./ +COPY Musify.Domain/Musify.Domain.csproj Musify.Domain/ +COPY Musify.Application/Musify.Application.csproj Musify.Application/ +COPY Musify.Infrastructure/Musify.Infrastructure.csproj Musify.Infrastructure/ +COPY Musify.Api/Musify.Api.csproj Musify.Api/ +RUN dotnet restore Musify.Api/Musify.Api.csproj + +COPY . . +RUN dotnet publish Musify.Api/Musify.Api.csproj -c Release -o /app --no-restore +# Config files are AppSettings*.json (PascalCase); on case-sensitive Linux the host +# looks for appsettings*.json. Add lowercase copies so they get loaded. +RUN cp /app/AppSettings.json /app/appsettings.json \ + && cp /app/AppSettings.Development.json /app/appsettings.Development.json + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime +WORKDIR /app +COPY --from=build /app ./ +ENV ASPNETCORE_URLS=http://+:5111 +EXPOSE 5111 +ENTRYPOINT ["dotnet", "Musify.Api.dll"] diff --git a/backend/Musify.StreamingGateway.Dockerfile b/backend/Musify.StreamingGateway.Dockerfile new file mode 100644 index 0000000..1e20496 --- /dev/null +++ b/backend/Musify.StreamingGateway.Dockerfile @@ -0,0 +1,17 @@ +# Build context: backend/ +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src + +COPY Directory.Build.props Directory.Packages.props nuget.config ./ +COPY Musify.StreamingGateway/Musify.StreamingGateway.csproj Musify.StreamingGateway/ +RUN dotnet restore Musify.StreamingGateway/Musify.StreamingGateway.csproj + +COPY . . +RUN dotnet publish Musify.StreamingGateway/Musify.StreamingGateway.csproj -c Release -o /app --no-restore + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime +WORKDIR /app +COPY --from=build /app ./ +ENV ASPNETCORE_URLS=http://+:8081 +EXPOSE 8081 +ENTRYPOINT ["dotnet", "Musify.StreamingGateway.dll"] diff --git a/backend/Musify.Worker.Dockerfile b/backend/Musify.Worker.Dockerfile new file mode 100644 index 0000000..b65f627 --- /dev/null +++ b/backend/Musify.Worker.Dockerfile @@ -0,0 +1,25 @@ +# Build context: backend/ +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src + +COPY Directory.Build.props Directory.Packages.props nuget.config ./ +COPY Musify.Domain/Musify.Domain.csproj Musify.Domain/ +COPY Musify.Application/Musify.Application.csproj Musify.Application/ +COPY Musify.Infrastructure/Musify.Infrastructure.csproj Musify.Infrastructure/ +COPY Musify.Worker/Musify.Worker.csproj Musify.Worker/ +RUN dotnet restore Musify.Worker/Musify.Worker.csproj + +COPY . . +RUN dotnet publish Musify.Worker/Musify.Worker.csproj -c Release -o /app --no-restore +# AppSettings*.json (PascalCase) -> lowercase copies for case-sensitive Linux. +RUN cp /app/AppSettings.json /app/appsettings.json \ + && cp /app/AppSettings.Development.json /app/appsettings.Development.json + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime +WORKDIR /app +# ffmpeg: the Worker transcodes audio to .m4a. +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /app ./ +ENTRYPOINT ["dotnet", "Musify.Worker.dll"] diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..7f1d7e9 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,35 @@ +# Dev stack config (deploy/docker-compose.yml). +# Copy to deploy/.env and adjust. .env is not versioned (.gitignore ignores *.env). + +# PostgreSQL (shared by Musify and Zitadel). Local apps use localhost:59000. +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=musify_db +POSTGRES_HOST_PORT=59000 + +# RabbitMQ (MassTransit) +RABBITMQ_DEFAULT_USER=guest +RABBITMQ_DEFAULT_PASS=guest + +# Zitadel (OIDC). The masterkey MUST be exactly 32 characters. +ZITADEL_MASTERKEY=MasterkeyNeedsToHave32Characters +# The external domain/port forms the token issuer and must match everywhere. +# host.docker.internal resolves the same from browser, local apps and containers, +# so auth works both locally and full-docker. Changing it needs a re-init +# (up.ps1 -Destroy; up.ps1). Admin login: @zitadel.. +ZITADEL_EXTERNAL_DOMAIN=host.docker.internal +ZITADEL_EXTERNAL_PORT=8080 +ZITADEL_ADMIN_USERNAME=admin +ZITADEL_ADMIN_PASSWORD=Password1! + +# SeaweedFS (S3). Must match deploy/seaweedfs/s3.conf. +S3_ACCESS_KEY=admin_access_key +S3_SECRET_KEY=admin_secret_key +S3_BUCKET=webapi-storage + +# Host paths used by up.ps1 and the apps overlay. +STREAM_KEYS_DIR=D:/weed +WORKER_TEMP_DIR=D:/tempsFilesDev + +# OIDC ClientId, filled in by zitadel/provision.ps1 (used by the docker path). +AUTH_CLIENT_ID= diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..77b84d8 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,192 @@ +# Development deploy + +Everything the backend needs for development, in one place. A single +`docker-compose.yml` brings up **PostgreSQL, Zitadel, RabbitMQ, SeaweedFS and +Jaeger**; the .NET apps run from the IDE (or, optionally, in Docker too). + +## Quick start + +```powershell +# From the repo root (Musify/) +./deploy/up.ps1 +``` + +`up.ps1` does the full bootstrap: + +1. Creates `deploy/.env` from `.env.example` if missing. +2. Starts the stack (`docker compose up -d`) and waits for Postgres to be healthy. +3. Creates the `webapi-storage` bucket in SeaweedFS (`seaweedfs-init` service). +4. Generates the RS256 stream-ticket keys in `D:\weed` if missing. +5. Creates the Worker temp directory (`D:\tempsFilesDev`). +6. Applies the EF Core migrations to `musify_db`. +7. **Provisions Zitadel automatically** (project + OIDC app with JWT token) and + wires the ClientId into `Musify.Api` user-secrets and `.env`. + +When it finishes it prints a summary with URLs and credentials. **No manual steps.** + +### Without the script (infra only) + +```powershell +docker compose -f deploy/docker-compose.yml up -d +``` + +The compose file still creates the bucket on its own; run migrations with +`dotnet ef database update` (see [../docs/development.md](../docs/development.md)). + +## Services and ports + +| Service | Port(s) | Used for | +|---|---|---| +| PostgreSQL | `59000` → 5432 | API + Worker (EF Core) and the Zitadel database | +| Zitadel | `8080` | OIDC/OAuth2 (console at `/ui/console`) | +| RabbitMQ | `5672` / `15672` | MassTransit / management UI | +| SeaweedFS S3 | `8333` | S3 (presigned PUT/GET) | +| SeaweedFS filer | `8888` | serves files → StreamingGateway | +| SeaweedFS master | `9333` | admin/UI | +| Jaeger | `16686` / `4317` / `4318` | OTLP traces (UI / gRPC / HTTP) | + +Ports are published to the host, so apps run locally connect without changing +`AppSettings.Development.json` (Postgres, RabbitMQ, S3, OTLP). + +## Authentication (Zitadel) — automatic + +There are **no manual steps**. `up.ps1` runs `zitadel/provision.ps1`, which +idempotently: + +1. Uses a **service account** with a **PAT** that Zitadel writes on init + (`zitadel/.output/admin-sa.pat`). +2. Creates (or reuses) the **project** `Musify` and an **OIDC app** (PKCE, + `devMode`, **access token = JWT** — required for `JwtBearer` to validate). +3. Captures the generated **ClientId** and writes it to: + - **`Musify.Api` user-secrets** (for running the apps locally), and + - `deploy/.env` as `AUTH_CLIENT_ID` (for the full-docker path). + +The values stay stable while the Postgres volume persists. After `up.ps1 -Destroy` +the next `up.ps1` re-provisions (new ClientId, re-wired). + +- Console: `http://host.docker.internal:8080/ui/console` +- Admin login: `admin@zitadel.host.docker.internal` / `Password1!` (see `.env`). +- Test login: open Scalar (`http://localhost:5111/scalar/v1`) and authorize. + +### Why `host.docker.internal` + +`ZITADEL_EXTERNAL_DOMAIN=host.docker.internal` forms the token *issuer* and +resolves the **same** from the browser, local apps and containers, so token +validation works both ways (local and full-docker). Zitadel routes by the `Host` +header, so the domain must be identical everywhere. If you only run the apps +locally you can use `localhost`. Changing the domain requires a re-init: +`up.ps1 -Destroy; up.ps1`. + +## Running the apps + +### Recommended: from the IDE (best for debugging) + +```powershell +dotnet run --project backend/Musify.Api # :5111 +dotnet run --project backend/Musify.StreamingGateway # :8081 +dotnet run --project backend/Musify.Worker # background +``` + +Host requirements: **.NET 10 SDK**, **ffmpeg** on PATH (Worker audio transcode). + +### Optional: everything in Docker + +```powershell +docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.apps.yml up -d --build +``` + +Builds and runs `api`, `worker` and `gateway` as containers (the Worker already +ships ffmpeg). The stream-ticket keys are mounted from `D:\weed` (configurable via +`STREAM_KEYS_DIR` in `.env`). Auth works in this mode too (the API downloads the +Zitadel JWKS via `host.docker.internal`). + +> Note: on Linux the apps look for `appsettings*.json` in lowercase. The `Api`/ +> `Worker` projects use `AppSettings*.json` (PascalCase), so their Dockerfiles +> create lowercase copies at publish time. On Windows it did not matter +> (case-insensitive). + +## Useful commands + +```powershell +./deploy/up.ps1 -SkipMigrations # bring up without touching the DB +./deploy/up.ps1 -Down # stop and remove containers (keep data) +./deploy/up.ps1 -Destroy # stop and remove containers + volumes (wipe data) + +docker compose -f deploy/docker-compose.yml logs -f zitadel +docker compose -f deploy/docker-compose.yml ps +``` + +## Project layout — what each file does + +``` +deploy/ +├─ docker-compose.yml # infrastructure stack (single source of truth) +├─ docker-compose.apps.yml # optional overlay: the 3 .NET apps in Docker +├─ .env # local config (secrets/ports); not versioned +├─ .env.example # config template (versioned) +├─ up.ps1 # orchestrator: up / down / destroy +├─ README.md # this file +├─ scripts/ +│ ├─ common.ps1 # shared helpers (Read-DotEnv, Write-Step) +│ ├─ stream-keys.ps1 # generates the RS256 stream-ticket key pair +│ └─ migrate.ps1 # applies EF Core migrations +├─ seaweedfs/ +│ └─ s3.conf # SeaweedFS S3 identities/credentials +└─ zitadel/ + ├─ provision.ps1 # creates project + OIDC app, wires the ClientId + └─ .output/ # PAT + init secrets written by Zitadel (not versioned) + +backend/ +├─ Musify.Api.Dockerfile # API image +├─ Musify.Worker.Dockerfile # Worker image (includes ffmpeg) +├─ Musify.StreamingGateway.Dockerfile # Gateway image +└─ .dockerignore +``` + +### Compose + +- **`docker-compose.yml`** — the infra stack (`postgres`, `zitadel`, `rabbitmq`, + `seaweedfs`, the one-shot `seaweedfs-init` bucket creator, and `jaeger`). This is + the single source of truth for services, ports and volumes. Values come from `.env`. +- **`docker-compose.apps.yml`** — an optional overlay that adds the `api`, `worker` + and `gateway` services. It merges with the infra file (same `musify-dev` project + and network) and shares the backend env via YAML anchors. Used to run the whole + stack in Docker; for active development prefer running the apps from the IDE. + +### Config + +- **`.env`** — the actual values used by compose (Postgres/RabbitMQ/Zitadel/S3 + credentials, ports, host paths, `AUTH_CLIENT_ID`). Ignored by git (`*.env`). +- **`.env.example`** — the versioned template. Copy it to `.env` (or let `up.ps1` + do it) and adjust. + +### Scripts (PowerShell) + +- **`up.ps1`** — the orchestrator. Parses `.env`, starts the stack, waits for + Postgres, then calls the sub-scripts in order (keys → migrations → provisioning) + and prints the summary. Also handles `-Down` / `-Destroy` / `-SkipMigrations`. +- **`scripts/common.ps1`** — shared helpers dot-sourced by the others: `Read-DotEnv` + (parse `.env` into a hashtable) and `Write-Step` (step logging). +- **`scripts/stream-keys.ps1`** — generates the RS256 key pair that the API uses to + sign stream tickets and the Gateway uses to verify them. No-op if the keys exist. +- **`scripts/migrate.ps1`** — sets the connection string in user-secrets and runs + `dotnet ef database update` (startup project = `Musify.Infrastructure`). +- **`zitadel/provision.ps1`** — reads the service-account PAT, creates/reuses the + Zitadel project and OIDC app (JWT token), and wires the ClientId into + `Musify.Api` user-secrets and `.env`. Idempotent; can be run standalone. + +### Service configs + +- **`seaweedfs/s3.conf`** — the S3 identities (access/secret keys) SeaweedFS loads. + Must match `InfrastructureStorage:*` in the app settings and `S3_*` in `.env`. +- **`zitadel/.output/`** — created at runtime; holds the PAT Zitadel writes on init + (`admin-sa.pat`). Not versioned; wiped by `up.ps1 -Destroy`. + +### Dockerfiles (in `backend/`) + +- **`Musify.Api.Dockerfile`** / **`Musify.Worker.Dockerfile`** / + **`Musify.StreamingGateway.Dockerfile`** — multi-stage builds for each host. The + Worker image also installs ffmpeg. Built by the apps overlay; build context is + `backend/` so restore can copy the central props and all csproj files. +- **`.dockerignore`** — keeps `bin/`, `obj/`, IDE folders and `DesignSettings.json` + out of the build context. diff --git a/deploy/docker-compose.apps.yml b/deploy/docker-compose.apps.yml new file mode 100644 index 0000000..78a6c5f --- /dev/null +++ b/deploy/docker-compose.apps.yml @@ -0,0 +1,96 @@ +name: musify-dev + +# Optional overlay: runs the .NET apps in Docker. Merges with docker-compose.yml. +# +# docker compose -f deploy/docker-compose.yml -f deploy/docker-compose.apps.yml up -d --build +# +# For active development prefer infra in Docker + apps from the IDE (easier to +# debug). Auth works in both modes: ZITADEL_EXTERNAL_DOMAIN=host.docker.internal +# resolves the same from the browser, local apps and containers, so the token +# issuer matches everywhere (containers reach the host via extra_hosts). + +x-backend-depends: &backend-depends + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + seaweedfs: + condition: service_healthy + +x-backend-env: &backend-env + Database__ConnectionString: "Host=postgres;Port=5432;Database=${POSTGRES_DB};Username=${POSTGRES_USER};Password=${POSTGRES_PASSWORD}" + MassTransit__Address: "rabbitmq://rabbitmq:5672" + MassTransit__Username: ${RABBITMQ_DEFAULT_USER} + MassTransit__Password: ${RABBITMQ_DEFAULT_PASS} + InfrastructureStorage__Address: "http://seaweedfs:8333" + InfrastructureStorage__AccessKey: ${S3_ACCESS_KEY} + InfrastructureStorage__SecretAccessKey: ${S3_SECRET_KEY} + InfrastructureStorage__ForcePathStyle: "true" + InfrastructureStorage__UseHttp: "true" + ApplicationStorage__Bucket: ${S3_BUCKET} + OpenTelemetry__OtlpEndpoint: "http://jaeger:4317" + +services: + api: + build: + context: ../backend + dockerfile: Musify.Api.Dockerfile + container_name: musify-api + depends_on: *backend-depends + environment: + <<: *backend-env + ASPNETCORE_ENVIRONMENT: Development + ASPNETCORE_URLS: http://+:5111 + StreamGateway__PublicBaseUrl: "http://localhost:8081" + StreamTicket__PrivateKeyPath: "/keys/stream_private.pem" + # Auth values provisioned by zitadel/provision.ps1 (AUTH_CLIENT_ID in .env). + Authentication__MetadataAddress: "http://${ZITADEL_EXTERNAL_DOMAIN}:${ZITADEL_EXTERNAL_PORT}/.well-known/openid-configuration" + Authentication__IssuerAddress: "http://${ZITADEL_EXTERNAL_DOMAIN}:${ZITADEL_EXTERNAL_PORT}" + Authentication__ClientId: "${AUTH_CLIENT_ID}" + Authentication__AudienceAddress: "${AUTH_CLIENT_ID}" + Authentication__AuthorizationEndpoint: "http://${ZITADEL_EXTERNAL_DOMAIN}:${ZITADEL_EXTERNAL_PORT}/oauth/v2/authorize?prompt=login" + Authentication__TokenEndpoint: "http://${ZITADEL_EXTERNAL_DOMAIN}:${ZITADEL_EXTERNAL_PORT}/oauth/v2/token" + Authentication__ScalarRedirectUri: "http://localhost:5111/scalar/" + Authentication__Scopes__0: "openid" + Authentication__Scopes__1: "profile" + Authentication__Scopes__2: "email" + Authentication__RequireHttpsMetadata: "false" + extra_hosts: + - "${ZITADEL_EXTERNAL_DOMAIN}:host-gateway" + volumes: + - "${STREAM_KEYS_DIR:-D:/weed}:/keys:ro" + ports: + - "5111:5111" + restart: unless-stopped + + worker: + build: + context: ../backend + dockerfile: Musify.Worker.Dockerfile + container_name: musify-worker + depends_on: *backend-depends + environment: + <<: *backend-env + DOTNET_ENVIRONMENT: Development + AudioTranscoder__Routes__WorkingDirectory: "/tmp/musify" + Workers__Routes__TemporaryFilesDirectory: "/tmp/musify" + restart: unless-stopped + + gateway: + build: + context: ../backend + dockerfile: Musify.StreamingGateway.Dockerfile + container_name: musify-gateway + depends_on: + seaweedfs: + condition: service_healthy + environment: + ASPNETCORE_ENVIRONMENT: Development + ASPNETCORE_URLS: http://+:8081 + StreamTicket__PublicKeyPath: "/keys/stream_public.pem" + ReverseProxy__Clusters__seaweed-filer__Destinations__filer__Address: "http://seaweedfs:8888/" + volumes: + - "${STREAM_KEYS_DIR:-D:/weed}:/keys:ro" + ports: + - "8081:8081" + restart: unless-stopped diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..a427dd9 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,146 @@ +name: musify-dev + +# Infrastructure stack for Musify development: everything the backend needs +# (Postgres, Zitadel, RabbitMQ, SeaweedFS + bucket, Jaeger/OTLP). The .NET apps +# run from the IDE and connect over the published ports. Config in deploy/.env +# +# docker compose -f deploy/docker-compose.yml up -d + +services: + postgres: + image: postgres:17-alpine + container_name: musify-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - "${POSTGRES_HOST_PORT}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + zitadel: + image: ghcr.io/zitadel/zitadel:latest + container_name: musify-zitadel + command: 'start-from-init --masterkeyFromEnv --tlsMode disabled' + environment: + ZITADEL_MASTERKEY: ${ZITADEL_MASTERKEY} + ZITADEL_EXTERNALSECURE: "false" + ZITADEL_EXTERNALDOMAIN: ${ZITADEL_EXTERNAL_DOMAIN} + ZITADEL_EXTERNALPORT: ${ZITADEL_EXTERNAL_PORT} + ZITADEL_TLS_ENABLED: "false" + # Reuses the same Postgres: the admin user creates the 'zitadel' database + # and the 'zitadel' user on first start. + ZITADEL_DATABASE_POSTGRES_HOST: postgres + ZITADEL_DATABASE_POSTGRES_PORT: 5432 + ZITADEL_DATABASE_POSTGRES_DATABASE: zitadel + ZITADEL_DATABASE_POSTGRES_USER_USERNAME: zitadel + ZITADEL_DATABASE_POSTGRES_USER_PASSWORD: zitadel + ZITADEL_DATABASE_POSTGRES_USER_SSL_MODE: disable + ZITADEL_DATABASE_POSTGRES_ADMIN_USERNAME: ${POSTGRES_USER} + ZITADEL_DATABASE_POSTGRES_ADMIN_PASSWORD: ${POSTGRES_PASSWORD} + ZITADEL_DATABASE_POSTGRES_ADMIN_SSL_MODE: disable + # Initial console admin. Login: @zitadel. + ZITADEL_FIRSTINSTANCE_ORG_HUMAN_USERNAME: ${ZITADEL_ADMIN_USERNAME} + ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORD: ${ZITADEL_ADMIN_PASSWORD} + ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED: "false" + # Service account + PAT for automatic provisioning. Zitadel writes the PAT + # to PATPATH on init; provision.ps1 uses it to create the project and app. + ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_USERNAME: zitadel-admin-sa + ZITADEL_FIRSTINSTANCE_ORG_MACHINE_MACHINE_NAME: Admin Service Account + ZITADEL_FIRSTINSTANCE_ORG_MACHINE_PAT_EXPIRATIONDATE: "2099-01-01T00:00:00Z" + ZITADEL_FIRSTINSTANCE_PATPATH: /output/admin-sa.pat + depends_on: + postgres: + condition: service_healthy + ports: + - "${ZITADEL_EXTERNAL_PORT}:8080" + volumes: + - ./zitadel/.output:/output + restart: unless-stopped + + rabbitmq: + image: rabbitmq:4-management-alpine + container_name: musify-rabbitmq + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS} + ports: + - "5672:5672" # AMQP -> MassTransit (API + Worker) + - "15672:15672" # management UI + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + seaweedfs: + image: chrislusf/seaweedfs:latest + container_name: musify-seaweedfs + # All-in-one dev server: master + volume + filer + S3 gateway. + command: > + server + -ip.bind=0.0.0.0 + -dir=/data + -master.volumeSizeLimitMB=1024 + -volume.max=0 + -filer + -s3 + -s3.config=/etc/seaweedfs/s3.conf + ports: + - "9333:9333" # master (admin/UI) + - "8888:8888" # filer HTTP -> StreamingGateway + - "8333:8333" # S3 API -> WebApi/Worker (InfrastructureStorage:Address) + volumes: + - seaweedfs_data:/data + - ./seaweedfs/s3.conf:/etc/seaweedfs/s3.conf:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9333/cluster/healthz"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + # One-shot: creates the app bucket in SeaweedFS (it is not auto-created). + seaweedfs-init: + image: amazon/aws-cli:latest + container_name: musify-seaweedfs-init + depends_on: + seaweedfs: + condition: service_healthy + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + AWS_DEFAULT_REGION: us-east-1 + entrypoint: ["/bin/sh", "-c"] + command: + - | + aws --endpoint-url http://seaweedfs:8333 s3 mb s3://${S3_BUCKET} 2>/dev/null || true + aws --endpoint-url http://seaweedfs:8333 s3 ls + restart: "no" + + # OTLP collector + trace UI (MassTransit/EF/HTTP spans). OtlpEndpoint = :4317 + jaeger: + image: jaegertracing/all-in-one:latest + container_name: musify-jaeger + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - "16686:16686" # UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + restart: unless-stopped + +volumes: + postgres_data: + rabbitmq_data: + seaweedfs_data: diff --git a/deploy/scripts/common.ps1 b/deploy/scripts/common.ps1 new file mode 100644 index 0000000..5cf2123 --- /dev/null +++ b/deploy/scripts/common.ps1 @@ -0,0 +1,15 @@ +#requires -Version 5.1 +# Shared helpers for the deploy scripts. + +function Write-Step([string]$Message) { Write-Host "`n==> $Message" -ForegroundColor Cyan } + +# Parses a .env file into a hashtable (ignores comments and blank lines). +function Read-DotEnv([string]$Path) { + $values = @{} + if (Test-Path $Path) { + Get-Content $Path | ForEach-Object { + if ($_ -match '^\s*([^#=\s][^=]*)=(.*)$') { $values[$matches[1].Trim()] = $matches[2].Trim() } + } + } + return $values +} diff --git a/deploy/scripts/migrate.ps1 b/deploy/scripts/migrate.ps1 new file mode 100644 index 0000000..5b6449e --- /dev/null +++ b/deploy/scripts/migrate.ps1 @@ -0,0 +1,15 @@ +#requires -Version 5.1 +# Applies EF Core migrations to the Musify database. +[CmdletBinding()] +param([Parameter(Mandatory)][string]$ConnectionString) + +$ErrorActionPreference = "Stop" +$infraProject = Resolve-Path (Join-Path $PSScriptRoot "..\..\backend\Musify.Infrastructure") + +# The design-time factory reads the connection string from user-secrets. +dotnet user-secrets set "Database:ConnectionString" $ConnectionString --project $infraProject | Out-Null + +# Infrastructure is also the startup project: it owns the IDesignTimeDbContextFactory +# and the EFCore.Design package (Musify.Api does not reference Design). +dotnet ef database update --project $infraProject --startup-project $infraProject --context Database +if ($LASTEXITCODE -ne 0) { throw "EF Core migration failed" } diff --git a/deploy/scripts/stream-keys.ps1 b/deploy/scripts/stream-keys.ps1 new file mode 100644 index 0000000..50d260a --- /dev/null +++ b/deploy/scripts/stream-keys.ps1 @@ -0,0 +1,28 @@ +#requires -Version 5.1 +# Generates the RS256 key pair used to sign (API) and verify (Gateway) stream tickets. +# No-op if both keys already exist. +[CmdletBinding()] +param([Parameter(Mandatory)][string]$KeysDir) + +$ErrorActionPreference = "Stop" +$privateKey = Join-Path $KeysDir "stream_private.pem" +$publicKey = Join-Path $KeysDir "stream_public.pem" + +if ((Test-Path $privateKey) -and (Test-Path $publicKey)) { + Write-Host "Stream-ticket keys already present in $KeysDir" + return +} + +if (-not (Test-Path $KeysDir)) { New-Item -ItemType Directory -Path $KeysDir | Out-Null } + +if (Get-Command openssl -ErrorAction SilentlyContinue) { + & openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out $privateKey + & openssl rsa -in $privateKey -pubout -out $publicKey +} else { + $rsa = [System.Security.Cryptography.RSA]::Create(2048) + [System.IO.File]::WriteAllText($privateKey, $rsa.ExportPkcs8PrivateKeyPem()) + [System.IO.File]::WriteAllText($publicKey, $rsa.ExportSubjectPublicKeyInfoPem()) + $rsa.Dispose() +} + +Write-Host "Generated stream-ticket keys in $KeysDir" diff --git a/deploy/seaweedfs/docker-compose.yml b/deploy/seaweedfs/docker-compose.yml deleted file mode 100644 index 1d48e3e..0000000 --- a/deploy/seaweedfs/docker-compose.yml +++ /dev/null @@ -1,32 +0,0 @@ -services: - seaweedfs: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs - # All-in-one dev server: master + volume + filer + S3 gateway. - command: > - server - -ip.bind=0.0.0.0 - -dir=/data - -master.volumeSizeLimitMB=1024 - -volume.max=0 - -filer - -s3 - -s3.config=/etc/seaweedfs/s3.conf - ports: - - "9333:9333" # master (admin/UI) - - "8888:8888" # filer HTTP -> used by the Media Gateway - - "8333:8333" # S3 API -> used by the WebApi (InfrastructureStorage:Address) - # Volume server (8080) is left unpublished on purpose: it only needs to be - # reachable internally, and 8080 clashes with Zitadel on the host. - volumes: - - seaweedfs_data:/data - - ./s3.conf:/etc/seaweedfs/s3.conf:ro - restart: unless-stopped - healthcheck: - test: ["CMD", "wget", "-q", "-O", "-", "http://localhost:9333/cluster/healthz"] - interval: 10s - timeout: 5s - retries: 10 - -volumes: - seaweedfs_data: diff --git a/deploy/up.ps1 b/deploy/up.ps1 new file mode 100644 index 0000000..f87f187 --- /dev/null +++ b/deploy/up.ps1 @@ -0,0 +1,121 @@ +#requires -Version 5.1 +<# +.SYNOPSIS + Brings up the Musify development environment. + +.DESCRIPTION + Starts the infrastructure stack, generates the stream-ticket keys, applies EF + Core migrations and provisions Zitadel (project + OIDC app + ClientId). + +.EXAMPLE + ./up.ps1 # Bring everything up + ./up.ps1 -SkipMigrations # Skip the database migration step + ./up.ps1 -Down # Stop and remove containers (keep volumes) + ./up.ps1 -Destroy # Stop and remove containers + volumes (wipe data) +#> +[CmdletBinding()] +param( + [switch]$Down, + [switch]$Destroy, + [switch]$SkipMigrations +) + +$ErrorActionPreference = "Stop" +$deployDir = $PSScriptRoot +$composeFile = Join-Path $deployDir "docker-compose.yml" + +. (Join-Path $deployDir "scripts\common.ps1") + +# Teardown ------------------------------------------------------------------- +if ($Down -or $Destroy) { + if ($Destroy) { + Write-Step "Stopping the stack and REMOVING volumes (data is wiped)" + docker compose -f $composeFile down -v --remove-orphans + # Drop the Zitadel init secrets (PAT) so the next start re-initializes cleanly. + Get-ChildItem (Join-Path $deployDir "zitadel\.output") -File -ErrorAction SilentlyContinue | Remove-Item -Force + } else { + Write-Step "Stopping the stack (volumes are kept)" + docker compose -f $composeFile down --remove-orphans + } + return +} + +# Config --------------------------------------------------------------------- +$envFile = Join-Path $deployDir ".env" +if (-not (Test-Path $envFile)) { + Write-Step "Creating .env from .env.example" + Copy-Item (Join-Path $deployDir ".env.example") $envFile +} +$cfg = Read-DotEnv $envFile +$streamKeysDir = if ($cfg['STREAM_KEYS_DIR']) { $cfg['STREAM_KEYS_DIR'] } else { "D:/weed" } +$workerTempDir = if ($cfg['WORKER_TEMP_DIR']) { $cfg['WORKER_TEMP_DIR'] } else { "D:/tempsFilesDev" } + +# Infrastructure ------------------------------------------------------------- +Write-Step "Starting the infrastructure stack" +docker compose -f $composeFile up -d +if ($LASTEXITCODE -ne 0) { throw "docker compose up failed" } + +Write-Step "Waiting for Postgres to become healthy" +$deadline = (Get-Date).AddMinutes(2) +do { + Start-Sleep -Seconds 2 + $status = docker inspect --format '{{.State.Health.Status}}' musify-postgres 2>$null +} until ($status -eq "healthy" -or (Get-Date) -gt $deadline) +if ($status -ne "healthy") { throw "Postgres did not become healthy in time" } + +# Stream-ticket keys --------------------------------------------------------- +Write-Step "Ensuring stream-ticket keys" +& (Join-Path $deployDir "scripts\stream-keys.ps1") -KeysDir $streamKeysDir + +# Worker temp directory ------------------------------------------------------ +if (-not (Test-Path $workerTempDir)) { + Write-Step "Creating the Worker temp directory: $workerTempDir" + New-Item -ItemType Directory -Path $workerTempDir | Out-Null +} + +# Database migrations -------------------------------------------------------- +if ($SkipMigrations) { + Write-Host "`nSkipping migrations (-SkipMigrations)." +} else { + Write-Step "Applying EF Core migrations" + $conn = "Host=localhost;Port=$($cfg['POSTGRES_HOST_PORT']);Database=$($cfg['POSTGRES_DB']);Username=$($cfg['POSTGRES_USER']);Password=$($cfg['POSTGRES_PASSWORD'])" + & (Join-Path $deployDir "scripts\migrate.ps1") -ConnectionString $conn +} + +# Zitadel provisioning ------------------------------------------------------- +Write-Step "Provisioning Zitadel (project + OIDC app + ClientId)" +try { + & (Join-Path $deployDir "zitadel\provision.ps1") +} catch { + Write-Host "Warning: Zitadel provisioning failed: $($_.Exception.Message)" -ForegroundColor Yellow +} + +# Summary -------------------------------------------------------------------- +$zDomain = $cfg['ZITADEL_EXTERNAL_DOMAIN'] +$zPort = $cfg['ZITADEL_EXTERNAL_PORT'] +Write-Host @" + +============================================================ + Musify development environment READY +============================================================ + + Service URL / port + ------------------ -------------------------------------- + PostgreSQL localhost:$($cfg['POSTGRES_HOST_PORT']) (db: $($cfg['POSTGRES_DB'])) + Zitadel console http://$zDomain`:$zPort/ui/console + RabbitMQ mgmt http://localhost:15672 ($($cfg['RABBITMQ_DEFAULT_USER'])/$($cfg['RABBITMQ_DEFAULT_PASS'])) + SeaweedFS S3 http://localhost:8333 (bucket: $($cfg['S3_BUCKET'])) + SeaweedFS filer http://localhost:8888 + SeaweedFS master http://localhost:9333 + Jaeger traces http://localhost:16686 (OTLP: localhost:4317) + + Zitadel admin: $($cfg['ZITADEL_ADMIN_USERNAME'])@zitadel.$zDomain / $($cfg['ZITADEL_ADMIN_PASSWORD']) + Auth is provisioned automatically; ClientId is in Musify.Api user-secrets and .env. + Log in via Scalar: http://localhost:5111/scalar/v1 + + Run the apps from the IDE or: + dotnet run --project backend/Musify.Api # :5111 + dotnet run --project backend/Musify.StreamingGateway # :8081 + dotnet run --project backend/Musify.Worker # background +============================================================ +"@ -ForegroundColor Green diff --git a/deploy/zitadel/.gitignore b/deploy/zitadel/.gitignore new file mode 100644 index 0000000..cb93576 --- /dev/null +++ b/deploy/zitadel/.gitignore @@ -0,0 +1,2 @@ +# Secrets Zitadel writes on init (do not version) +.output/ diff --git a/deploy/zitadel/provision.ps1 b/deploy/zitadel/provision.ps1 new file mode 100644 index 0000000..fdddf5a --- /dev/null +++ b/deploy/zitadel/provision.ps1 @@ -0,0 +1,121 @@ +#requires -Version 5.1 +<# +.SYNOPSIS + Provisions Zitadel for Musify (idempotent). + +.DESCRIPTION + Creates the project and the OIDC app (PKCE, JWT access token) and wires the + generated ClientId into Musify.Api user-secrets and deploy/.env (AUTH_CLIENT_ID). + Uses the service-account PAT that Zitadel writes on init (zitadel/.output/admin-sa.pat). + Re-running reuses the existing project/app and refreshes the wired values. +#> +[CmdletBinding()] +param( + [string]$ProjectName = "Musify", + [string]$AppName = "Musify API (Scalar)" +) + +$ErrorActionPreference = "Stop" +$deployDir = Split-Path $PSScriptRoot -Parent +$apiProject = Resolve-Path (Join-Path $deployDir "..\backend\Musify.Api") +$patFile = Join-Path $PSScriptRoot ".output\admin-sa.pat" + +. (Join-Path $deployDir "scripts\common.ps1") + +$cfg = Read-DotEnv (Join-Path $deployDir ".env") +$issuer = "http://$($cfg['ZITADEL_EXTERNAL_DOMAIN']):$($cfg['ZITADEL_EXTERNAL_PORT'])" +$apiBase = "http://localhost:5111" +$scalarRedirect = "$apiBase/scalar/" + +# Zitadel writes the PAT during init; on a cold start it may take a moment. +$deadline = (Get-Date).AddMinutes(3) +while (-not (Test-Path $patFile) -and (Get-Date) -lt $deadline) { Start-Sleep -Seconds 3 } +if (-not (Test-Path $patFile)) { throw "PAT not found ($patFile). Check 'docker logs musify-zitadel'." } + +$headers = @{ Authorization = "Bearer $((Get-Content $patFile -Raw).Trim())" } +$mgmt = "$issuer/management/v1" + +function Invoke-Zitadel($method, $path, $body) { + $params = @{ Method = $method; Uri = "$mgmt$path"; Headers = $headers; ContentType = "application/json"; TimeoutSec = 20 } + if ($body) { $params.Body = ($body | ConvertTo-Json -Depth 10) } + Invoke-RestMethod @params +} + +Write-Step "Waiting for Zitadel at $issuer" +$deadline = (Get-Date).AddMinutes(2) +do { + try { Invoke-RestMethod "$issuer/.well-known/openid-configuration" -TimeoutSec 5 | Out-Null; $ready = $true } + catch { $ready = $false; Start-Sleep 3 } +} until ($ready -or (Get-Date) -gt $deadline) +if (-not $ready) { throw "Zitadel did not respond in time." } + +# Project (idempotent) ------------------------------------------------------- +Write-Step "Project '$ProjectName'" +$project = (Invoke-Zitadel POST "/projects/_search" @{ + queries = @(@{ nameQuery = @{ name = $ProjectName; method = "TEXT_QUERY_METHOD_EQUALS" } }) +}).result | Where-Object { $_.name -eq $ProjectName } | Select-Object -First 1 + +if ($project) { + $projectId = $project.id + Write-Host " reused (id $projectId)" +} else { + $projectId = (Invoke-Zitadel POST "/projects" @{ name = $ProjectName }).id + Write-Host " created (id $projectId)" +} + +# OIDC app (idempotent) ------------------------------------------------------ +Write-Step "OIDC app '$AppName'" +$app = (Invoke-Zitadel POST "/projects/$projectId/apps/_search" @{ + queries = @(@{ nameQuery = @{ name = $AppName; method = "TEXT_QUERY_METHOD_EQUALS" } }) +}).result | Where-Object { $_.name -eq $AppName } | Select-Object -First 1 + +if ($app) { + $clientId = (Invoke-Zitadel GET "/projects/$projectId/apps/$($app.id)" $null).app.oidcConfig.clientId + Write-Host " reused (clientId $clientId)" +} else { + $clientId = (Invoke-Zitadel POST "/projects/$projectId/apps/oidc" @{ + name = $AppName + redirectUris = @($scalarRedirect, "$apiBase/scalar/oauth2-redirect.html") + postLogoutRedirectUris = @($scalarRedirect) + responseTypes = @("OIDC_RESPONSE_TYPE_CODE") + grantTypes = @("OIDC_GRANT_TYPE_AUTHORIZATION_CODE") + appType = "OIDC_APP_TYPE_USER_AGENT" + authMethodType = "OIDC_AUTH_METHOD_TYPE_NONE" + version = "OIDC_VERSION_1_0" + devMode = $true # allow http redirect URIs + accessTokenType = "OIDC_TOKEN_TYPE_JWT" # required so JwtBearer can validate + accessTokenRoleAssertion = $true + idTokenRoleAssertion = $true + idTokenUserinfoAssertion = $true + }).clientId + Write-Host " created (clientId $clientId)" +} + +# Wire the config: user-secrets (local runs) + .env (docker runs) ------------- +Write-Step "Wiring Authentication into Musify.Api user-secrets" +@{ + "Authentication:ClientId" = $clientId + "Authentication:AudienceAddress" = $clientId + "Authentication:IssuerAddress" = $issuer + "Authentication:MetadataAddress" = "$issuer/.well-known/openid-configuration" + "Authentication:AuthorizationEndpoint" = "$issuer/oauth/v2/authorize?prompt=login" + "Authentication:TokenEndpoint" = "$issuer/oauth/v2/token" + "Authentication:ScalarRedirectUri" = $scalarRedirect + "Authentication:RequireHttpsMetadata" = "false" +}.GetEnumerator() | ForEach-Object { + dotnet user-secrets set $_.Key $_.Value --project $apiProject | Out-Null +} + +$envPath = Join-Path $deployDir ".env" +$envLines = Get-Content $envPath +if ($envLines -match '^\s*AUTH_CLIENT_ID=') { + $envLines = $envLines -replace '^\s*AUTH_CLIENT_ID=.*', "AUTH_CLIENT_ID=$clientId" +} else { + $envLines += "AUTH_CLIENT_ID=$clientId" +} +Set-Content -Path $envPath -Value $envLines -Encoding utf8 + +Write-Host "`nZitadel provisioned:" -ForegroundColor Green +Write-Host " Project: $ProjectName ($projectId)" +Write-Host " ClientId: $clientId" +Write-Host " Issuer: $issuer" diff --git a/docs/development.md b/docs/development.md index 401d83a..93d0c9a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,21 +4,21 @@ Cómo levantar todo en local y probarlo. ## Dependencias (Docker) +Todo (PostgreSQL, Zitadel, RabbitMQ, SeaweedFS, Jaeger) vive en un solo stack en +[`deploy/`](../deploy/README.md). Bootstrap completo (levanta + bucket + claves + migraciones): + ```powershell -# SeaweedFS (almacenamiento) -docker compose -f deploy\seaweedfs\docker-compose.yml up -d - -# PostgreSQL + pgAdmin -docker compose -f "C:\Users\XShin\OneDrive\Desktop\postgresql\docker-compose.yml" up -d - -# Zitadel (auth) -docker compose -f "C:\Users\XShin\OneDrive\Pictures\zitadel-compose\docker-compose.yml" up -d - -# RabbitMQ (contenedor suelto) -docker start rabbitmq +./deploy/up.ps1 ``` -> Nota: usa `docker compose up` (no `docker start`) para Postgres/Zitadel: si sus redes se borraron, compose las recrea. +O solo la infraestructura: + +```powershell +docker compose -f deploy\docker-compose.yml up -d +``` + +Config (usuarios/claves/puertos) en `deploy/.env`. Ver [deploy/README.md](../deploy/README.md) +para el detalle y el paso de configuración de Zitadel (crear la app OIDC y copiar el ClientId). ## Aplicaciones (.NET) @@ -44,14 +44,15 @@ DB: `musify_db`, usuario `postgres`/`postgres`. Bucket S3: `webapi-storage` (cre ## Migraciones -No hay migración automática al arrancar. Para aplicar cambios de esquema: +No hay migración automática al arrancar. `deploy/up.ps1` las aplica en el bootstrap; +para hacerlo a mano tras cambios de esquema: ```powershell -cd MusifyBackend\Infrastructure -dotnet ef database update --context Database +dotnet ef database update --project backend\Musify.Infrastructure --startup-project backend\Musify.Infrastructure --context Database ``` -La cadena de conexión para `dotnet ef` se lee de `DesignSettings.json` / user-secrets del proyecto Infrastructure. +La cadena de conexión para `dotnet ef` se lee de `DesignSettings.json` / user-secrets del proyecto Infrastructure +(`up.ps1` la deja en user-secrets automáticamente). ## Stream-tickets (claves RS256)