This commit is contained in:
2026-06-01 00:24:41 +02:00
parent 539899741d
commit 842b1ffd08
12 changed files with 22 additions and 44 deletions
@@ -1,4 +1,4 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace Musify.Application.Configuration namespace Musify.Application.Configuration
{ {
@@ -69,7 +69,6 @@ namespace Musify.Application.Configuration
public string BuildLargePicturePath(string pictureName) => CombineKey(ParentFolders, LargePicturesFolder, pictureName); public string BuildLargePicturePath(string pictureName) => CombineKey(ParentFolders, LargePicturesFolder, pictureName);
// Temp path used in Phase 2 for pre-signed upload keys (moved to final on consumption)
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) => public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
CombineKey(tempRootPrefix, userId.ToString(), ParentFolders, objectName); CombineKey(tempRootPrefix, userId.ToString(), ParentFolders, objectName);
@@ -1,4 +1,4 @@
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
namespace Musify.Application.Configuration namespace Musify.Application.Configuration
{ {
@@ -85,7 +85,6 @@ namespace Musify.Application.Configuration
public string BuildProcessedAudioPath(string audioName) => CombineKey(ProcessedAudiosPath, audioName); public string BuildProcessedAudioPath(string audioName) => CombineKey(ProcessedAudiosPath, audioName);
// Temp paths used in Phase 2 for pre-signed upload keys (moved to final on consumption)
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) => public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
CombineKey(tempRootPrefix, userId.ToString(), ParentFolder, objectName); CombineKey(tempRootPrefix, userId.ToString(), ParentFolder, objectName);
@@ -6,47 +6,37 @@ namespace Musify.Application.Configuration
{ {
public const string SectionName = "UploadIntent"; public const string SectionName = "UploadIntent";
// Pre-signed URL TTL in seconds
[Range(10, 3600)] [Range(10, 3600)]
public int UploadUrlExpiresInSeconds { get; set; } = 120; public int UploadUrlExpiresInSeconds { get; set; } = 120;
// Max concurrent Issued intents per user
[Range(1, 100)] [Range(1, 100)]
public int MaxActiveUploadIntentsPerUser { get; set; } = 5; public int MaxActiveUploadIntentsPerUser { get; set; } = 5;
// Max total bytes across all Issued intents for a user (default 200MB)
[Range(1, long.MaxValue)] [Range(1, long.MaxValue)]
public long MaxActiveUploadBytesPerUser { get; set; } = 200L * 1024 * 1024; public long MaxActiveUploadBytesPerUser { get; set; } = 200L * 1024 * 1024;
// Max size validated via HEAD before consuming an intent (default 100MB)
[Range(1, long.MaxValue)] [Range(1, long.MaxValue)]
public long MaxUploadBytes { get; set; } = 100L * 1024 * 1024; public long MaxUploadBytes { get; set; } = 100L * 1024 * 1024;
// Conservative fallback size used when ExpectedSizeBytes is not provided
[Range(1, long.MaxValue)] [Range(1, long.MaxValue)]
public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024; // 10MB public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024;
[Range(1, long.MaxValue)] [Range(1, long.MaxValue)]
public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024; // 100MB public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024;
// How often the expiration job runs
[Range(30, 86400)] [Range(30, 86400)]
public int ExpirationJobIntervalSeconds { get; set; } = 300; public int ExpirationJobIntervalSeconds { get; set; } = 300;
// How many days to retain Expired intents before deleting the DB row
[Range(0, 365)] [Range(0, 365)]
public int ExpiredIntentsRetentionDays { get; set; } = 7; public int ExpiredIntentsRetentionDays { get; set; } = 7;
// Root S3 prefix for temporary uploads (Phase 2)
[Required, MinLength(1)] [Required, MinLength(1)]
public string TempRootPrefix { get; set; } = "temp"; public string TemporalRootPrefix { get; set; } = "temporal";
// How many days to retain orphaned temp objects in S3 before the cleanup job deletes them
[Range(1, 30)] [Range(1, 30)]
public int TempUploadsRetentionDays { get; set; } = 3; public int TemporalUploadsRetentionDays { get; set; } = 3;
// How often the temp-upload cleanup job runs
[Range(60, 86400)] [Range(60, 86400)]
public int TempCleanupJobIntervalSeconds { get; set; } = 3600; public int TemporalCleanUpJobIntervalSeconds { get; set; } = 3600;
} }
} }
@@ -39,7 +39,7 @@ namespace Musify.Application.PlayLists.Handlers
var objectName = $"{Guid.NewGuid() }.{request.FileType.TrimStart('.').ToLowerInvariant() }"; var objectName = $"{Guid.NewGuid() }.{request.FileType.TrimStart('.').ToLowerInvariant() }";
var tempKey = playListConfiguration.Routes.BuildTempPicturePath( var tempKey = playListConfiguration.Routes.BuildTempPicturePath(
uploadIntentConfiguration.TempRootPrefix, request.UserId, objectName); uploadIntentConfiguration.TemporalRootPrefix, request.UserId, objectName);
try try
{ {
@@ -42,9 +42,9 @@ namespace Musify.Application.Tracks.Handlers
var audioObjectName = $"{Guid.NewGuid()}.{request.AudioFileType.TrimStart('.').ToLowerInvariant()}"; var audioObjectName = $"{Guid.NewGuid()}.{request.AudioFileType.TrimStart('.').ToLowerInvariant()}";
var tempPictureKey = trackConfiguration.Routes.BuildTempPicturePath( var tempPictureKey = trackConfiguration.Routes.BuildTempPicturePath(
uploadIntentConfiguration.TempRootPrefix, request.UserId, pictureObjectName); uploadIntentConfiguration.TemporalRootPrefix, request.UserId, pictureObjectName);
var tempAudioKey = trackConfiguration.Routes.BuildTempAudioPath( var tempAudioKey = trackConfiguration.Routes.BuildTempAudioPath(
uploadIntentConfiguration.TempRootPrefix, request.UserId, audioObjectName); uploadIntentConfiguration.TemporalRootPrefix, request.UserId, audioObjectName);
try try
{ {
-2
View File
@@ -17,11 +17,9 @@ namespace Musify.Domain.Entities
[Required, MaxLength(128)] [Required, MaxLength(128)]
public required string Bucket { get; set; } public required string Bucket { get; set; }
// Full S3 key (including prefix and object name)
[Required, MaxLength(512)] [Required, MaxLength(512)]
public required string Key { get; set; } public required string Key { get; set; }
// Just the filename portion stored in Track/PlayList entities (e.g. "{guid}.webp")
[Required, MaxLength(128)] [Required, MaxLength(128)]
public required string ObjectName { get; set; } public required string ObjectName { get; set; }
@@ -16,8 +16,8 @@ namespace Musify.Infrastructure.Jobs
protected override async Task ExecuteAsync(CancellationToken stoppingToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
logger.LogInformation("TemporalUploadsCleanUpJob started (interval: {Interval}s, prefix: {Prefix})", logger.LogInformation("TemporalUploadsCleanUpJob started (interval: {Interval}s, prefix: {Prefix})",
uploadIntentConfiguration.TempCleanupJobIntervalSeconds, uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds,
uploadIntentConfiguration.TempRootPrefix); uploadIntentConfiguration.TemporalRootPrefix);
while (!stoppingToken.IsCancellationRequested) while (!stoppingToken.IsCancellationRequested)
{ {
@@ -30,7 +30,7 @@ namespace Musify.Infrastructure.Jobs
logger.LogError(exception, "Error in TemporalUploadsCleanUpJob"); logger.LogError(exception, "Error in TemporalUploadsCleanUpJob");
} }
await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TempCleanupJobIntervalSeconds), stoppingToken); await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds), stoppingToken);
} }
} }
@@ -39,13 +39,13 @@ namespace Musify.Infrastructure.Jobs
await using var scope = scopeFactory.CreateAsyncScope(); await using var scope = scopeFactory.CreateAsyncScope();
var storageService = scope.ServiceProvider.GetRequiredService<IStorageService>(); var storageService = scope.ServiceProvider.GetRequiredService<IStorageService>();
var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TempUploadsRetentionDays); var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TemporalUploadsRetentionDays);
var deleted = 0; var deleted = 0;
var errors = 0; var errors = 0;
await foreach (var (key, lastModified) in storageService.ListObjectsAsync( await foreach (var (key, lastModified) in storageService.ListObjectsAsync(
storageConfiguration.Bucket, storageConfiguration.Bucket,
uploadIntentConfiguration.TempRootPrefix, uploadIntentConfiguration.TemporalRootPrefix,
cancellationToken)) cancellationToken))
{ {
if (lastModified >= cutOff) if (lastModified >= cutOff)
@@ -40,7 +40,6 @@ namespace Musify.Infrastructure.Jobs
var now = DateTime.UtcNow; var now = DateTime.UtcNow;
// Mark expired intents
var expiredIntents = await database.UploadIntents var expiredIntents = await database.UploadIntents
.Where(i => i.Status == UploadIntentStatus.Issued && i.ExpiresAt < now) .Where(i => i.Status == UploadIntentStatus.Issued && i.ExpiresAt < now)
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
@@ -54,7 +53,6 @@ namespace Musify.Infrastructure.Jobs
logger.LogInformation("Marked {Count} upload intents as Expired", expiredIntents.Count); logger.LogInformation("Marked {Count} upload intents as Expired", expiredIntents.Count);
} }
// Delete old Expired intents past retention window
if (uploadIntentConfiguration.ExpiredIntentsRetentionDays >= 0) if (uploadIntentConfiguration.ExpiredIntentsRetentionDays >= 0)
{ {
var retentionCutoff = now.AddDays(-uploadIntentConfiguration.ExpiredIntentsRetentionDays); var retentionCutoff = now.AddDays(-uploadIntentConfiguration.ExpiredIntentsRetentionDays);
@@ -1,4 +1,4 @@
// <auto-generated />
using System; using System;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Infrastructure;
@@ -15,8 +15,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
[Migration("20260530020027_InitialCreate")] [Migration("20260530020027_InitialCreate")]
partial class InitialCreate partial class InitialCreate
{ {
/// <inheritdoc /> protected override void BuildTargetModel(ModelBuilder modelBuilder)
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{ {
#pragma warning disable 612, 618 #pragma warning disable 612, 618
modelBuilder modelBuilder
@@ -1,15 +1,13 @@
using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Migrations;
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
#nullable disable #nullable disable
namespace Musify.Infrastructure.Persistence.Migrations namespace Musify.Infrastructure.Persistence.Migrations
{ {
/// <inheritdoc /> public partial class InitialCreate : Migration
public partial class InitialCreate : Migration
{ {
/// <inheritdoc /> protected override void Up(MigrationBuilder migrationBuilder)
protected override void Up(MigrationBuilder migrationBuilder)
{ {
migrationBuilder.CreateTable( migrationBuilder.CreateTable(
name: "InboxState", name: "InboxState",
@@ -321,8 +319,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
column: "UserId"); column: "UserId");
} }
/// <inheritdoc /> protected override void Down(MigrationBuilder migrationBuilder)
protected override void Down(MigrationBuilder migrationBuilder)
{ {
migrationBuilder.DropTable( migrationBuilder.DropTable(
name: "OutboxMessage"); name: "OutboxMessage");
@@ -1,4 +1,4 @@
// <auto-generated />
using System; using System;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Infrastructure;
@@ -44,7 +44,6 @@ namespace Musify.Infrastructure.Services
return serviceDescriptors; return serviceDescriptors;
} }
// Called by both the web API host and the worker
public static IServiceCollection AddUploadIntentConfiguration(this IServiceCollection serviceDescriptors, IConfiguration configuration) public static IServiceCollection AddUploadIntentConfiguration(this IServiceCollection serviceDescriptors, IConfiguration configuration)
{ {
serviceDescriptors serviceDescriptors
@@ -60,7 +59,6 @@ namespace Musify.Infrastructure.Services
return serviceDescriptors; return serviceDescriptors;
} }
// Called only by the worker host (registers background jobs)
public static IServiceCollection AddUploadIntentJobs(this IServiceCollection serviceDescriptors, IConfiguration configuration) public static IServiceCollection AddUploadIntentJobs(this IServiceCollection serviceDescriptors, IConfiguration configuration)
{ {
serviceDescriptors.AddUploadIntentConfiguration(configuration); serviceDescriptors.AddUploadIntentConfiguration(configuration);