From d08d82a48381930bca822e6bcdec4e9a640eef73 Mon Sep 17 00:00:00 2001 From: XShine555 Date: Sun, 5 Jul 2026 04:58:27 +0200 Subject: [PATCH] fix(infra): repair TransferFiles compensation + tidy MassTransit consistency Fix (recoverability bug): - TransferFilesToBucketActivity returned Completed() with no log, so its compensation could never delete the transferred files (orphaned objects on fault). IStorageService.TransferFilesAsync now returns the real uploaded keys; the activity logs them so Compensate can remove them. Also drops the Path.Combine key recomputation (produced backslash keys on Windows). Consistency / DRY: - Normalize all courier activities to internal + folder-based namespaces (several were public and in the flat .Activities namespace) - Extract the duplicated exponential retry into UseStandardRetry - Unify endpoint-name logic into a single EndpointHelper (remove the parallel MassTransitEndpointNames helper) Build: 0 warnings, 0 errors. Co-Authored-By: Claude Opus 4.8 --- .../Contracts/IStorageService.cs | 2 +- .../GenerateAudioWorkflowPathsActivity.cs | 2 +- .../Audio/TranscodeAudioActivity.cs | 2 +- .../Files/TransferFilesToBucketActivity.cs | 17 +++---- .../Files/UploadFileToBucketActivity.cs | 2 +- .../GeneratePictureWorkflowPathsActivity.cs | 4 +- .../Pictures/ResizePictureActivity.cs | 4 +- .../Pictures/UpdatePlayListPictureActivity.cs | 4 +- .../Pictures/UpdateTrackPictureActivity.cs | 4 +- .../MassTransit/Consumers/EndpointHelper.cs | 10 ++-- ...assTransitDependencyInjection.Endpoints.cs | 50 ++++++------------- ...TransitDependencyInjection.Registration.cs | 2 +- .../MassTransitEndpointNames.cs | 9 ---- .../PictureWorkflowRoutingSlipBuilder.cs | 2 +- .../Services/StorageService.cs | 6 ++- 15 files changed, 49 insertions(+), 71 deletions(-) delete mode 100644 backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitEndpointNames.cs diff --git a/backend/Musify.Application/Contracts/IStorageService.cs b/backend/Musify.Application/Contracts/IStorageService.cs index 02fe1f1..e5f3c24 100644 --- a/backend/Musify.Application/Contracts/IStorageService.cs +++ b/backend/Musify.Application/Contracts/IStorageService.cs @@ -14,7 +14,7 @@ namespace Musify.Application.Contracts Task UploadFileAsync(Stream sourceStream, string contentType, string bucket, string key, CancellationToken cancellationToken); - Task TransferFilesAsync(string sourceDirectory, string bucket, string route, CancellationToken cancellationToken); + Task> TransferFilesAsync(string sourceDirectory, string bucket, string route, CancellationToken cancellationToken); Task RemoveFileAsync(string bucket, string key, CancellationToken cancellationToken); diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Audio/GenerateAudioWorkflowPathsActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/GenerateAudioWorkflowPathsActivity.cs index b404528..1db4ba5 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Audio/GenerateAudioWorkflowPathsActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/GenerateAudioWorkflowPathsActivity.cs @@ -6,7 +6,7 @@ using Musify.Domain.Entities; using Musify.Domain.ValueObjects; using Musify.Infrastructure.MassTransit.Arguments; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Audio { internal class GenerateAudioWorkflowPathsActivity( IDatabase database, diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Audio/TranscodeAudioActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/TranscodeAudioActivity.cs index 36f3f68..a0d2b33 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Audio/TranscodeAudioActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/TranscodeAudioActivity.cs @@ -7,7 +7,7 @@ using Musify.Domain.ValueObjects; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Logs; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Audio { internal class TranscodeAudioActivity( IDatabase database, diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Files/TransferFilesToBucketActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Files/TransferFilesToBucketActivity.cs index 8e1db84..c23193b 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Files/TransferFilesToBucketActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Files/TransferFilesToBucketActivity.cs @@ -4,9 +4,9 @@ using Musify.Application.Contracts; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Logs; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Files { - public class TransferFilesToBucketActivity( + internal class TransferFilesToBucketActivity( IStorageService storageHandler, ILogger logger) : IActivity @@ -22,16 +22,15 @@ namespace Musify.Infrastructure.MassTransit.Activities try { - var uploadedKeys = Directory.GetFiles(folderPath) - .Select(file => Path.Combine(destinationKey, Path.GetFileName(file))) - .ToArray(); - - await storageHandler.TransferFilesAsync( + var uploadedKeys = await storageHandler.TransferFilesAsync( folderPath, executeContext.Arguments.DestinationBucket, destinationKey, executeContext.CancellationToken); - return executeContext.Completed(); + + return executeContext.Completed(new TransferFilesToBucketLog( + executeContext.Arguments.DestinationBucket, + uploadedKeys.ToArray())); } catch (Exception exception) { @@ -63,4 +62,4 @@ namespace Musify.Infrastructure.MassTransit.Activities } } } -} \ No newline at end of file +} diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Files/UploadFileToBucketActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Files/UploadFileToBucketActivity.cs index 62961bc..78029e2 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Files/UploadFileToBucketActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Files/UploadFileToBucketActivity.cs @@ -6,7 +6,7 @@ using Musify.Infrastructure.MassTransit.Logs; namespace Musify.Infrastructure.MassTransit.Activities.Files { - public class UploadFileToBucketActivity(IStorageService storageHandler, + internal class UploadFileToBucketActivity(IStorageService storageHandler, ILogger logger) : IActivity { diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/GeneratePictureWorkflowPathsActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/GeneratePictureWorkflowPathsActivity.cs index f7a0c83..ce41d3f 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/GeneratePictureWorkflowPathsActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/GeneratePictureWorkflowPathsActivity.cs @@ -2,9 +2,9 @@ using MassTransit; using Microsoft.Extensions.Logging; using Musify.Infrastructure.MassTransit.Arguments; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Pictures { - public class GeneratePictureWorkflowPathsActivity( + internal class GeneratePictureWorkflowPathsActivity( ILogger logger) : IExecuteActivity { diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/ResizePictureActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/ResizePictureActivity.cs index 193b100..ddf088f 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/ResizePictureActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/ResizePictureActivity.cs @@ -4,9 +4,9 @@ using Musify.Application.Contracts; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Logs; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Pictures { - public class ResizePictureActivity( + internal class ResizePictureActivity( IPictureService pictureHandler, ILogger logger) : IActivity diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdatePlayListPictureActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdatePlayListPictureActivity.cs index 63dd568..7af0cb3 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdatePlayListPictureActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdatePlayListPictureActivity.cs @@ -4,9 +4,9 @@ using Musify.Application.Contracts; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Logs; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Pictures { - public class UpdatePlayListPictureActivity( + internal class UpdatePlayListPictureActivity( IDatabase database, ILogger logger) : IActivity diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdateTrackPictureActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdateTrackPictureActivity.cs index 7b16eaf..932d5a1 100644 --- a/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdateTrackPictureActivity.cs +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Pictures/UpdateTrackPictureActivity.cs @@ -5,9 +5,9 @@ using Musify.Domain.ValueObjects; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Logs; -namespace Musify.Infrastructure.MassTransit.Activities +namespace Musify.Infrastructure.MassTransit.Activities.Pictures { - public class UpdateTrackPictureActivity( + internal class UpdateTrackPictureActivity( IDatabase database, ILogger logger) : IActivity diff --git a/backend/Musify.Infrastructure/MassTransit/Consumers/EndpointHelper.cs b/backend/Musify.Infrastructure/MassTransit/Consumers/EndpointHelper.cs index 47c8efd..60677c0 100644 --- a/backend/Musify.Infrastructure/MassTransit/Consumers/EndpointHelper.cs +++ b/backend/Musify.Infrastructure/MassTransit/Consumers/EndpointHelper.cs @@ -2,10 +2,14 @@ namespace Musify.Infrastructure.MassTransit.Consumers { internal static class EndpointHelper { - internal static Uri BuildExecuteActivityUri(string endpointName) => new($"queue:{endpointName}_execute"); + internal static string ExecuteQueueName(string endpointName) => $"{endpointName}_execute"; - internal static Uri BuildCompensateActivityUri(string endpointName) => new($"queue:{endpointName}_compensate"); + internal static string CompensateQueueName(string endpointName) => $"{endpointName}_compensate"; + + internal static Uri BuildExecuteActivityUri(string endpointName) => new($"queue:{ExecuteQueueName(endpointName)}"); + + internal static Uri BuildCompensateActivityUri(string endpointName) => new($"queue:{CompensateQueueName(endpointName)}"); internal static Uri BuildConsumerUri(string endpointName) => new($"queue:{endpointName}"); } -} \ No newline at end of file +} diff --git a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Endpoints.cs b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Endpoints.cs index 82e8d35..be0cd47 100644 --- a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Endpoints.cs +++ b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Endpoints.cs @@ -6,6 +6,14 @@ namespace Musify.Infrastructure.MassTransit { public static partial class MassTransitDependencyInjection { + static void UseStandardRetry(IReceiveEndpointConfigurator endpointConfigurator) => + endpointConfigurator.UseMessageRetry(retryConfigurator => + retryConfigurator.Exponential( + retryLimit: 5, + minInterval: TimeSpan.FromSeconds(1), + maxInterval: TimeSpan.FromSeconds(30), + intervalDelta: TimeSpan.FromSeconds(2))); + static void ConfigureConsumerEndpoint( IRabbitMqBusFactoryConfigurator busFactoryConfigurator, IBusRegistrationContext busRegistrationContext, @@ -14,15 +22,8 @@ namespace Musify.Infrastructure.MassTransit { busFactoryConfigurator.ReceiveEndpoint(queueName, endpointConfigurator => { - endpointConfigurator.UseMessageRetry(retryConfigurator => - retryConfigurator.Exponential( - retryLimit: 5, - minInterval: TimeSpan.FromSeconds(1), - maxInterval: TimeSpan.FromSeconds(30), - intervalDelta: TimeSpan.FromSeconds(2))); - + UseStandardRetry(endpointConfigurator); endpointConfigurator.UseEntityFrameworkOutbox(busRegistrationContext); - endpointConfigurator.ConfigureConsumer(busRegistrationContext); }); } @@ -34,17 +35,10 @@ namespace Musify.Infrastructure.MassTransit where TActivity : class, IExecuteActivity where TArguments : class { - busFactoryConfigurator.ReceiveEndpoint(MassTransitEndpointNames.ExecuteQueue(endpointName), endpointConfigurator => + busFactoryConfigurator.ReceiveEndpoint(EndpointHelper.ExecuteQueueName(endpointName), endpointConfigurator => { - endpointConfigurator.UseMessageRetry(retryConfigurator => - retryConfigurator.Exponential( - retryLimit: 5, - minInterval: TimeSpan.FromSeconds(1), - maxInterval: TimeSpan.FromSeconds(30), - intervalDelta: TimeSpan.FromSeconds(2))); - + UseStandardRetry(endpointConfigurator); endpointConfigurator.UseEntityFrameworkOutbox(busRegistrationContext); - endpointConfigurator.ExecuteActivityHost(busRegistrationContext); }); } @@ -57,33 +51,19 @@ namespace Musify.Infrastructure.MassTransit where TArguments : class where TLog : class { - busFactoryConfigurator.ReceiveEndpoint(MassTransitEndpointNames.ExecuteQueue(endpointName), endpointConfigurator => + busFactoryConfigurator.ReceiveEndpoint(EndpointHelper.ExecuteQueueName(endpointName), endpointConfigurator => { - endpointConfigurator.UseMessageRetry(retryConfigurator => - retryConfigurator.Exponential( - retryLimit: 5, - minInterval: TimeSpan.FromSeconds(1), - maxInterval: TimeSpan.FromSeconds(30), - intervalDelta: TimeSpan.FromSeconds(2))); - + UseStandardRetry(endpointConfigurator); endpointConfigurator.UseEntityFrameworkOutbox(busRegistrationContext); - endpointConfigurator.ExecuteActivityHost( EndpointHelper.BuildCompensateActivityUri(endpointName), busRegistrationContext); }); - busFactoryConfigurator.ReceiveEndpoint(MassTransitEndpointNames.CompensateQueue(endpointName), endpointConfigurator => + busFactoryConfigurator.ReceiveEndpoint(EndpointHelper.CompensateQueueName(endpointName), endpointConfigurator => { - endpointConfigurator.UseMessageRetry(retryConfigurator => - retryConfigurator.Exponential( - retryLimit: 5, - minInterval: TimeSpan.FromSeconds(1), - maxInterval: TimeSpan.FromSeconds(30), - intervalDelta: TimeSpan.FromSeconds(2))); - + UseStandardRetry(endpointConfigurator); endpointConfigurator.UseEntityFrameworkOutbox(busRegistrationContext); - endpointConfigurator.CompensateActivityHost(busRegistrationContext); }); } diff --git a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs index 457f956..5e1d744 100644 --- a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs +++ b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs @@ -1,7 +1,7 @@ using MassTransit; using Microsoft.Extensions.DependencyInjection; using Musify.Infrastructure.Configuration; -using Musify.Infrastructure.MassTransit.Activities; +using Musify.Infrastructure.MassTransit.Activities.Pictures; using Musify.Infrastructure.MassTransit.Activities.Audio; using Musify.Infrastructure.MassTransit.Activities.Files; using Musify.Infrastructure.MassTransit.Activities.PlayLists; diff --git a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitEndpointNames.cs b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitEndpointNames.cs deleted file mode 100644 index cdcdffc..0000000 --- a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitEndpointNames.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Musify.Infrastructure.MassTransit -{ - internal static class MassTransitEndpointNames - { - internal static string ExecuteQueue(string endpointName) => $"{endpointName}_execute"; - - internal static string CompensateQueue(string endpointName) => $"{endpointName}_compensate"; - } -} diff --git a/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/PictureWorkflowRoutingSlipBuilder.cs b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/PictureWorkflowRoutingSlipBuilder.cs index 0fcbf34..71bc2ae 100644 --- a/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/PictureWorkflowRoutingSlipBuilder.cs +++ b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/PictureWorkflowRoutingSlipBuilder.cs @@ -3,7 +3,7 @@ using MassTransit.Courier.Contracts; using Musify.Application.Shared; using Musify.Application.Events; using Musify.Infrastructure.Configuration; -using Musify.Infrastructure.MassTransit.Activities; +using Musify.Infrastructure.MassTransit.Activities.Pictures; using Musify.Infrastructure.MassTransit.Activities.Files; using Musify.Infrastructure.MassTransit.Arguments; using Musify.Infrastructure.MassTransit.Consumers; diff --git a/backend/Musify.Infrastructure/Services/StorageService.cs b/backend/Musify.Infrastructure/Services/StorageService.cs index b07c765..ebbf220 100644 --- a/backend/Musify.Infrastructure/Services/StorageService.cs +++ b/backend/Musify.Infrastructure/Services/StorageService.cs @@ -105,12 +105,13 @@ namespace Musify.Infrastructure.Services await amazonS3.DeleteObjectAsync(request, cancellationToken); } - public async Task TransferFilesAsync(string sourceDirectory, string bucket, string route, CancellationToken cancellationToken) + public async Task> TransferFilesAsync(string sourceDirectory, string bucket, string route, CancellationToken cancellationToken) { var trasnsferUtility = new TransferUtility(amazonS3); var files = Directory.GetFiles(sourceDirectory, "*"); + var uploadedKeys = new List(); var successCount = 0; var failedCount = 0; @@ -144,6 +145,7 @@ namespace Musify.Infrastructure.Services }, cancellationToken); upload.State = UploadState.Successful; + uploadedKeys.Add(key); successCount++; logger.LogDebug("Transferred file to S3 {Bucket}/{Key}", bucket, key); } @@ -158,6 +160,8 @@ namespace Musify.Infrastructure.Services await database.SaveChangesAsync(cancellationToken); logger.LogInformation("File transfer completed to {Bucket}/{Route}. Success: {SuccessCount}, Failed: {FailedCount}", bucket, route, successCount, failedCount); + + return uploadedKeys; } public async Task UploadFileAsync(string filePath, string bucket, string key, CancellationToken cancellationToken)