Cleanup
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
using System.ComponentModel.DataAnnotations;
|
using System.ComponentModel.DataAnnotations;
|
||||||
|
|
||||||
namespace Musify.Application.Configuration
|
namespace Musify.Application.Configuration
|
||||||
{
|
{
|
||||||
@@ -69,7 +69,6 @@ namespace Musify.Application.Configuration
|
|||||||
|
|
||||||
public string BuildLargePicturePath(string pictureName) => CombineKey(ParentFolders, LargePicturesFolder, pictureName);
|
public string BuildLargePicturePath(string pictureName) => CombineKey(ParentFolders, LargePicturesFolder, pictureName);
|
||||||
|
|
||||||
// Temp path used in Phase 2 for pre-signed upload keys (moved to final on consumption)
|
|
||||||
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
||||||
CombineKey(tempRootPrefix, userId.ToString(), ParentFolders, objectName);
|
CombineKey(tempRootPrefix, userId.ToString(), ParentFolders, objectName);
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using System.ComponentModel.DataAnnotations;
|
using System.ComponentModel.DataAnnotations;
|
||||||
|
|
||||||
namespace Musify.Application.Configuration
|
namespace Musify.Application.Configuration
|
||||||
{
|
{
|
||||||
@@ -85,7 +85,6 @@ namespace Musify.Application.Configuration
|
|||||||
|
|
||||||
public string BuildProcessedAudioPath(string audioName) => CombineKey(ProcessedAudiosPath, audioName);
|
public string BuildProcessedAudioPath(string audioName) => CombineKey(ProcessedAudiosPath, audioName);
|
||||||
|
|
||||||
// Temp paths used in Phase 2 for pre-signed upload keys (moved to final on consumption)
|
|
||||||
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
public string BuildTempPicturePath(string tempRootPrefix, long userId, string objectName) =>
|
||||||
CombineKey(tempRootPrefix, userId.ToString(), ParentFolder, objectName);
|
CombineKey(tempRootPrefix, userId.ToString(), ParentFolder, objectName);
|
||||||
|
|
||||||
|
|||||||
@@ -6,47 +6,37 @@ namespace Musify.Application.Configuration
|
|||||||
{
|
{
|
||||||
public const string SectionName = "UploadIntent";
|
public const string SectionName = "UploadIntent";
|
||||||
|
|
||||||
// Pre-signed URL TTL in seconds
|
|
||||||
[Range(10, 3600)]
|
[Range(10, 3600)]
|
||||||
public int UploadUrlExpiresInSeconds { get; set; } = 120;
|
public int UploadUrlExpiresInSeconds { get; set; } = 120;
|
||||||
|
|
||||||
// Max concurrent Issued intents per user
|
|
||||||
[Range(1, 100)]
|
[Range(1, 100)]
|
||||||
public int MaxActiveUploadIntentsPerUser { get; set; } = 5;
|
public int MaxActiveUploadIntentsPerUser { get; set; } = 5;
|
||||||
|
|
||||||
// Max total bytes across all Issued intents for a user (default 200MB)
|
|
||||||
[Range(1, long.MaxValue)]
|
[Range(1, long.MaxValue)]
|
||||||
public long MaxActiveUploadBytesPerUser { get; set; } = 200L * 1024 * 1024;
|
public long MaxActiveUploadBytesPerUser { get; set; } = 200L * 1024 * 1024;
|
||||||
|
|
||||||
// Max size validated via HEAD before consuming an intent (default 100MB)
|
|
||||||
[Range(1, long.MaxValue)]
|
[Range(1, long.MaxValue)]
|
||||||
public long MaxUploadBytes { get; set; } = 100L * 1024 * 1024;
|
public long MaxUploadBytes { get; set; } = 100L * 1024 * 1024;
|
||||||
|
|
||||||
// Conservative fallback size used when ExpectedSizeBytes is not provided
|
|
||||||
[Range(1, long.MaxValue)]
|
[Range(1, long.MaxValue)]
|
||||||
public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024; // 10MB
|
public long DefaultExpectedPictureSizeBytes { get; set; } = 10L * 1024 * 1024;
|
||||||
|
|
||||||
[Range(1, long.MaxValue)]
|
[Range(1, long.MaxValue)]
|
||||||
public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024; // 100MB
|
public long DefaultExpectedAudioSizeBytes { get; set; } = 100L * 1024 * 1024;
|
||||||
|
|
||||||
// How often the expiration job runs
|
|
||||||
[Range(30, 86400)]
|
[Range(30, 86400)]
|
||||||
public int ExpirationJobIntervalSeconds { get; set; } = 300;
|
public int ExpirationJobIntervalSeconds { get; set; } = 300;
|
||||||
|
|
||||||
// How many days to retain Expired intents before deleting the DB row
|
|
||||||
[Range(0, 365)]
|
[Range(0, 365)]
|
||||||
public int ExpiredIntentsRetentionDays { get; set; } = 7;
|
public int ExpiredIntentsRetentionDays { get; set; } = 7;
|
||||||
|
|
||||||
// Root S3 prefix for temporary uploads (Phase 2)
|
|
||||||
[Required, MinLength(1)]
|
[Required, MinLength(1)]
|
||||||
public string TempRootPrefix { get; set; } = "temp";
|
public string TemporalRootPrefix { get; set; } = "temporal";
|
||||||
|
|
||||||
// How many days to retain orphaned temp objects in S3 before the cleanup job deletes them
|
|
||||||
[Range(1, 30)]
|
[Range(1, 30)]
|
||||||
public int TempUploadsRetentionDays { get; set; } = 3;
|
public int TemporalUploadsRetentionDays { get; set; } = 3;
|
||||||
|
|
||||||
// How often the temp-upload cleanup job runs
|
|
||||||
[Range(60, 86400)]
|
[Range(60, 86400)]
|
||||||
public int TempCleanupJobIntervalSeconds { get; set; } = 3600;
|
public int TemporalCleanUpJobIntervalSeconds { get; set; } = 3600;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ namespace Musify.Application.PlayLists.Handlers
|
|||||||
var objectName = $"{Guid.NewGuid() }.{request.FileType.TrimStart('.').ToLowerInvariant() }";
|
var objectName = $"{Guid.NewGuid() }.{request.FileType.TrimStart('.').ToLowerInvariant() }";
|
||||||
|
|
||||||
var tempKey = playListConfiguration.Routes.BuildTempPicturePath(
|
var tempKey = playListConfiguration.Routes.BuildTempPicturePath(
|
||||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, objectName);
|
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, objectName);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -42,9 +42,9 @@ namespace Musify.Application.Tracks.Handlers
|
|||||||
var audioObjectName = $"{Guid.NewGuid()}.{request.AudioFileType.TrimStart('.').ToLowerInvariant()}";
|
var audioObjectName = $"{Guid.NewGuid()}.{request.AudioFileType.TrimStart('.').ToLowerInvariant()}";
|
||||||
|
|
||||||
var tempPictureKey = trackConfiguration.Routes.BuildTempPicturePath(
|
var tempPictureKey = trackConfiguration.Routes.BuildTempPicturePath(
|
||||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, pictureObjectName);
|
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, pictureObjectName);
|
||||||
var tempAudioKey = trackConfiguration.Routes.BuildTempAudioPath(
|
var tempAudioKey = trackConfiguration.Routes.BuildTempAudioPath(
|
||||||
uploadIntentConfiguration.TempRootPrefix, request.UserId, audioObjectName);
|
uploadIntentConfiguration.TemporalRootPrefix, request.UserId, audioObjectName);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -17,11 +17,9 @@ namespace Musify.Domain.Entities
|
|||||||
[Required, MaxLength(128)]
|
[Required, MaxLength(128)]
|
||||||
public required string Bucket { get; set; }
|
public required string Bucket { get; set; }
|
||||||
|
|
||||||
// Full S3 key (including prefix and object name)
|
|
||||||
[Required, MaxLength(512)]
|
[Required, MaxLength(512)]
|
||||||
public required string Key { get; set; }
|
public required string Key { get; set; }
|
||||||
|
|
||||||
// Just the filename portion stored in Track/PlayList entities (e.g. "{guid}.webp")
|
|
||||||
[Required, MaxLength(128)]
|
[Required, MaxLength(128)]
|
||||||
public required string ObjectName { get; set; }
|
public required string ObjectName { get; set; }
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ namespace Musify.Infrastructure.Jobs
|
|||||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
{
|
{
|
||||||
logger.LogInformation("TemporalUploadsCleanUpJob started (interval: {Interval}s, prefix: {Prefix})",
|
logger.LogInformation("TemporalUploadsCleanUpJob started (interval: {Interval}s, prefix: {Prefix})",
|
||||||
uploadIntentConfiguration.TempCleanupJobIntervalSeconds,
|
uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds,
|
||||||
uploadIntentConfiguration.TempRootPrefix);
|
uploadIntentConfiguration.TemporalRootPrefix);
|
||||||
|
|
||||||
while (!stoppingToken.IsCancellationRequested)
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
@@ -30,7 +30,7 @@ namespace Musify.Infrastructure.Jobs
|
|||||||
logger.LogError(exception, "Error in TemporalUploadsCleanUpJob");
|
logger.LogError(exception, "Error in TemporalUploadsCleanUpJob");
|
||||||
}
|
}
|
||||||
|
|
||||||
await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TempCleanupJobIntervalSeconds), stoppingToken);
|
await Task.Delay(TimeSpan.FromSeconds(uploadIntentConfiguration.TemporalCleanUpJobIntervalSeconds), stoppingToken);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -39,13 +39,13 @@ namespace Musify.Infrastructure.Jobs
|
|||||||
await using var scope = scopeFactory.CreateAsyncScope();
|
await using var scope = scopeFactory.CreateAsyncScope();
|
||||||
var storageService = scope.ServiceProvider.GetRequiredService<IStorageService>();
|
var storageService = scope.ServiceProvider.GetRequiredService<IStorageService>();
|
||||||
|
|
||||||
var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TempUploadsRetentionDays);
|
var cutOff = DateTime.UtcNow.AddDays(-uploadIntentConfiguration.TemporalUploadsRetentionDays);
|
||||||
var deleted = 0;
|
var deleted = 0;
|
||||||
var errors = 0;
|
var errors = 0;
|
||||||
|
|
||||||
await foreach (var (key, lastModified) in storageService.ListObjectsAsync(
|
await foreach (var (key, lastModified) in storageService.ListObjectsAsync(
|
||||||
storageConfiguration.Bucket,
|
storageConfiguration.Bucket,
|
||||||
uploadIntentConfiguration.TempRootPrefix,
|
uploadIntentConfiguration.TemporalRootPrefix,
|
||||||
cancellationToken))
|
cancellationToken))
|
||||||
{
|
{
|
||||||
if (lastModified >= cutOff)
|
if (lastModified >= cutOff)
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ namespace Musify.Infrastructure.Jobs
|
|||||||
|
|
||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
|
|
||||||
// Mark expired intents
|
|
||||||
var expiredIntents = await database.UploadIntents
|
var expiredIntents = await database.UploadIntents
|
||||||
.Where(i => i.Status == UploadIntentStatus.Issued && i.ExpiresAt < now)
|
.Where(i => i.Status == UploadIntentStatus.Issued && i.ExpiresAt < now)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
@@ -54,7 +53,6 @@ namespace Musify.Infrastructure.Jobs
|
|||||||
logger.LogInformation("Marked {Count} upload intents as Expired", expiredIntents.Count);
|
logger.LogInformation("Marked {Count} upload intents as Expired", expiredIntents.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete old Expired intents past retention window
|
|
||||||
if (uploadIntentConfiguration.ExpiredIntentsRetentionDays >= 0)
|
if (uploadIntentConfiguration.ExpiredIntentsRetentionDays >= 0)
|
||||||
{
|
{
|
||||||
var retentionCutoff = now.AddDays(-uploadIntentConfiguration.ExpiredIntentsRetentionDays);
|
var retentionCutoff = now.AddDays(-uploadIntentConfiguration.ExpiredIntentsRetentionDays);
|
||||||
|
|||||||
+2
-3
@@ -1,4 +1,4 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
@@ -15,8 +15,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
|
|||||||
[Migration("20260530020027_InitialCreate")]
|
[Migration("20260530020027_InitialCreate")]
|
||||||
partial class InitialCreate
|
partial class InitialCreate
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
|
||||||
{
|
{
|
||||||
#pragma warning disable 612, 618
|
#pragma warning disable 612, 618
|
||||||
modelBuilder
|
modelBuilder
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
namespace Musify.Infrastructure.Persistence.Migrations
|
namespace Musify.Infrastructure.Persistence.Migrations
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
public partial class InitialCreate : Migration
|
||||||
public partial class InitialCreate : Migration
|
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
protected override void Up(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
{
|
||||||
migrationBuilder.CreateTable(
|
migrationBuilder.CreateTable(
|
||||||
name: "InboxState",
|
name: "InboxState",
|
||||||
@@ -321,8 +319,7 @@ namespace Musify.Infrastructure.Persistence.Migrations
|
|||||||
column: "UserId");
|
column: "UserId");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
protected override void Down(MigrationBuilder migrationBuilder)
|
|
||||||
{
|
{
|
||||||
migrationBuilder.DropTable(
|
migrationBuilder.DropTable(
|
||||||
name: "OutboxMessage");
|
name: "OutboxMessage");
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
// <auto-generated />
|
|
||||||
using System;
|
using System;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
|||||||
@@ -44,7 +44,6 @@ namespace Musify.Infrastructure.Services
|
|||||||
return serviceDescriptors;
|
return serviceDescriptors;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called by both the web API host and the worker
|
|
||||||
public static IServiceCollection AddUploadIntentConfiguration(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
public static IServiceCollection AddUploadIntentConfiguration(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
||||||
{
|
{
|
||||||
serviceDescriptors
|
serviceDescriptors
|
||||||
@@ -60,7 +59,6 @@ namespace Musify.Infrastructure.Services
|
|||||||
return serviceDescriptors;
|
return serviceDescriptors;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called only by the worker host (registers background jobs)
|
|
||||||
public static IServiceCollection AddUploadIntentJobs(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
public static IServiceCollection AddUploadIntentJobs(this IServiceCollection serviceDescriptors, IConfiguration configuration)
|
||||||
{
|
{
|
||||||
serviceDescriptors.AddUploadIntentConfiguration(configuration);
|
serviceDescriptors.AddUploadIntentConfiguration(configuration);
|
||||||
|
|||||||
Reference in New Issue
Block a user