Cleanup
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace Musify.Application.Configuration
|
||||
{
|
||||
@@ -69,7 +69,6 @@ namespace Musify.Application.Configuration
|
||||
|
||||
public string BuildLargePicturePath(string pictureName) => CombineKey(ParentFolders, LargePicturesFolder, pictureName);
|
||||
|
||||
// Temp path used in Phase 2 for pre-signed upload keys (moved to final on consumption)
|
||||
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
||||
CombineKey(tempRootPrefix, userId.ToString(), ParentFolders, objectName);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace Musify.Application.Configuration
|
||||
{
|
||||
@@ -85,7 +85,6 @@ namespace Musify.Application.Configuration
|
||||
|
||||
public string BuildProcessedAudioPath(string audioName) => CombineKey(ProcessedAudiosPath, audioName);
|
||||
|
||||
// Temp paths used in Phase 2 for pre-signed upload keys (moved to final on consumption)
|
||||
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
||||
CombineKey(tempRootPrefix, userId.ToString(), ParentFolder, objectName);
|
||||
|
||||
|
||||
@@ -6,47 +6,37 @@ namespace Musify.Application.Configuration
|
||||
{
|
||||
public const string SectionName = "UploadIntent";
|
||||
|
||||
// Pre-signed URL TTL in seconds
|
||||
[Range(10, 3600)]
|
||||
public int UploadUrlExpiresInSeconds { get; set; } = 120;
|
||||
|
||||
// Max concurrent Issued intents per user
|
||||
[Range(1, 100)]
|
||||
public int MaxActiveUploadIntentsPerUser { get; set; } = 5;
|
||||
|
||||
// Max total bytes across all Issued intents for a user (default 200MB)
|
||||
[Range(1, long.MaxValue)]
|
||||
public long MaxActiveUploadBytesPerUser { get; set; } = 200L * 1024 * 1024;
|
||||
|
||||
// Max size validated via HEAD before consuming an intent (default 100MB)
|
||||
[Range(1, long.MaxValue)]
|
||||
public long MaxUploadBytes { get; set; } = 100L * 1024 * 1024;
|
||||
|
||||
// Conservative fallback size used when ExpectedSizeBytes is not provided
|
||||
[Range(1, long.MaxValue)]
|
||||
public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024; // 10MB
|
||||
public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024;
|
||||
|
||||
[Range(1, long.MaxValue)]
|
||||
public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024; // 100MB
|
||||
public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024;
|
||||
|
||||
// How often the expiration job runs
|
||||
[Range(30, 86400)]
|
||||
public int ExpirationJobIntervalSeconds { get; set; } = 300;
|
||||
|
||||
// How many days to retain Expired intents before deleting the DB row
|
||||
[Range(0, 365)]
|
||||
public int ExpiredIntentsRetentionDays { get; set; } = 7;
|
||||
|
||||
// Root S3 prefix for temporary uploads (Phase 2)
|
||||
[Required, MinLength(1)]
|
||||
public string TempRootPrefix { get; set; } = "temp";
|
||||
public string TemporalRootPrefix { get; set; } = "temporal";
|
||||
|
||||
// How many days to retain orphaned temp objects in S3 before the cleanup job deletes them
|
||||
[Range(1, 30)]
|
||||
public int TempUploadsRetentionDays { get; set; } = 3;
|
||||
public int TemporalUploadsRetentionDays { get; set; } = 3;
|
||||
|
||||
// How often the temp-upload cleanup job runs
|
||||
[Range(60, 86400)]
|
||||
public int TempCleanupJobIntervalSeconds { get; set; } = 3600;
|
||||
public int TemporalCleanUpJobIntervalSeconds { get; set; } = 3600;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ namespace Musify.Application.PlayLists.Handlers
|
||||
var objectName = $"{Guid.NewGuid() }.{request.FileType.TrimStart('.').ToLowerInvariant() }";
|
||||
|
||||
var tempKey = playListConfiguration.Routes.BuildTempPicturePath(
|
||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, objectName);
|
||||
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, objectName);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@@ -42,9 +42,9 @@ namespace Musify.Application.Tracks.Handlers
|
||||
var audioObjectName = $"{Guid.NewGuid()}.{request.AudioFileType.TrimStart('.').ToLowerInvariant()}";
|
||||
|
||||
var tempPictureKey = trackConfiguration.Routes.BuildTempPicturePath(
|
||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, pictureObjectName);
|
||||
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, pictureObjectName);
|
||||
var tempAudioKey = trackConfiguration.Routes.BuildTempAudioPath(
|
||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, audioObjectName);
|
||||
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, audioObjectName);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@@ -17,11 +17,9 @@ namespace Musify.Domain.Entities
|
||||
[Required, MaxLength(128)]
|
||||
public required string Bucket { get; set; }
|
||||
|
||||
// Full S3 key (including prefix and object name)
|
||||
[Required, MaxLength(512)]
|
||||
public required string Key { get; set; }
|
||||
|
||||
// Just the filename portion stored in Track/PlayList entities (e.g. "{guid}.webp")
|
||||
[Required, MaxLength(128)]
|
||||
public required string ObjectName { get; set; }
|
||||
|
||||
|
||||
@@ -16,8 +16,8 @@ namespace Musify.Infrastructure.Jobs
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
logger.LogInformation("TemporalUploadsCleanUpJob started (interval: {Interval}s, prefix: {Prefix})",
|
||||
uploadIntentConfiguration.TempCleanupJobIntervalSeconds,
|
||||
uploadIntentConfiguration.TempRootPrefix);
|
||||
uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds,
|
||||
uploadIntentConfiguration.TemporalRootPrefix);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -30,7 +30,7 @@ namespace Musify.Infrastructure.Jobs
|
||||
logger.LogError(exception, "Error in TemporalUploadsCleanUpJob");
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TempCleanupJobIntervalSeconds), stoppingToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds), stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,13 +39,13 @@ namespace Musify.Infrastructure.Jobs
|
||||
await using var scope = scopeFactory.CreateAsyncScope();
|
||||
var storageService = scope.ServiceProvider.GetRequiredService<IStorageService>();
|
||||
|
||||
var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TempUploadsRetentionDays);
|
||||
var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TemporalUploadsRetentionDays);
|
||||
var deleted = 0;
|
||||
var errors = 0;
|
||||
|
||||
await foreach (var (key, lastModified) in storageService.ListObjectsAsync(
|
||||
storageConfiguration.Bucket,
|
||||
uploadIntentConfiguration.TempRootPrefix,
|
||||
uploadIntentConfiguration.TemporalRootPrefix,
|
||||
cancellationToken))
|
||||
{
|
||||
if (lastModified >= cutOff)
|
||||
|
||||
@@ -40,7 +40,6 @@ namespace Musify.Infrastructure.Jobs
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
// Mark expired intents
|
||||
var expiredIntents = await database.UploadIntents
|
||||
.Where(i => i.Status == UploadIntentStatus.Issued && i.ExpiresAt < now)
|
||||
.ToListAsync(cancellationToken);
|
||||
@@ -54,7 +53,6 @@ namespace Musify.Infrastructure.Jobs
|
||||
logger.LogInformation("Marked {Count} upload intents as Expired", expiredIntents.Count);
|
||||
}
|
||||
|
||||
// Delete old Expired intents past retention window
|
||||
if (uploadIntentConfiguration.ExpiredIntentsRetentionDays >= 0)
|
||||
{
|
||||
var retentionCutoff = now.AddDays(-uploadIntentConfiguration.ExpiredIntentsRetentionDays);
|
||||
|
||||
+2
-3
@@ -1,4 +1,4 @@
|
||||
// <auto-generated />
|
||||
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
@@ -15,8 +15,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
|
||||
[Migration("20260530020027_InitialCreate")]
|
||||
partial class InitialCreate
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Musify.Infrastructure.Persistence.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class InitialCreate : Migration
|
||||
public partial class InitialCreate : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "InboxState",
|
||||
@@ -321,8 +319,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
|
||||
column: "UserId");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropTable(
|
||||
name: "OutboxMessage");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// <auto-generated />
|
||||
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
|
||||
@@ -44,7 +44,6 @@ namespace Musify.Infrastructure.Services
|
||||
return serviceDescriptors;
|
||||
}
|
||||
|
||||
// Called by both the web API host and the worker
|
||||
public static IServiceCollection AddUploadIntentConfiguration(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
||||
{
|
||||
serviceDescriptors
|
||||
@@ -60,7 +59,6 @@ namespace Musify.Infrastructure.Services
|
||||
return serviceDescriptors;
|
||||
}
|
||||
|
||||
// Called only by the worker host (registers background jobs)
|
||||
public static IServiceCollection AddUploadIntentJobs(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
||||
{
|
||||
serviceDescriptors.AddUploadIntentConfiguration(configuration);
|
||||
|
||||
Reference in New Issue
Block a user