feat(infra): track saga failure handling + cleanup (phase 2)

On definitive failure of the picture or audio slip, the saga now reaches a Failed state and recovers end-to-end: - Picture/audio track slips carry TrackId + ProcessKind variables and a subscription to a fault translator (TrackProcessingSlipFaultConsumer) that turns RoutingSlipFaulted into a typed Track{Picture,Audio}ProcessingFailed - The saga stores the final object keys from the start event and, on failure, transitions to Failed and publishes TrackProcessingFailed - TrackProcessingFailedConsumer runs a cleanup slip that removes the orphaned final picture/audio copies and marks the track LifeCycleStatus=Failed (MarkTrackAsFailedActivity) — the track stays as a visible, recoverable record - Migration for the new saga columns Verified: Worker boots against RabbitMQ + Postgres in Docker; the new endpoints (fault translator, failed consumer, mark-track-as-failed) configure and the bus starts clean. Build: 0 warnings, 0 errors. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-05 14:49:33 +02:00
parent 986941166c
commit 8664bfb800
15 changed files with 840 additions and 2 deletions
@@ -0,0 +1,32 @@
+using MassTransit;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Musify.Application.Contracts;
+using Musify.Domain.ValueObjects;
+using Musify.Infrastructure.MassTransit.Arguments;
+
+namespace Musify.Infrastructure.MassTransit.Activities.Tracks
+{
+    internal class MarkTrackAsFailedActivity(
+        IDatabase database,
+        ILogger<MarkTrackAsFailedActivity> logger)
+        : IExecuteActivity<MarkTrackAsFailedArguments>
+    {
+        public const string ExecuteEndpointName = "mark-track-as-failed";
+
+        public async Task<ExecutionResult> Execute(ExecuteContext<MarkTrackAsFailedArguments> executeContext)
+        {
+            var track = await database.Tracks
+                .SingleOrDefaultAsync(t => t.Id == executeContext.Arguments.TrackId, executeContext.CancellationToken);
+
+            if (track is not null)
+            {
+                track.LifeCycleStatus = LifeCycleStatus.Failed;
+                await database.SaveChangesAsync(executeContext.CancellationToken);
+                logger.LogInformation("Marked track {TrackId} as failed", executeContext.Arguments.TrackId);
+            }
+
+            return executeContext.Completed();
+        }
+    }
+}
@@ -0,0 +1,4 @@
+namespace Musify.Infrastructure.MassTransit.Arguments
+{
+    public record MarkTrackAsFailedArguments(Guid TrackId);
+}
@@ -0,0 +1,42 @@
+using MassTransit;
+using MassTransit.Courier.Contracts;
+using Musify.Application.Events;
+using Musify.Infrastructure.MassTransit.Activities.Files;
+using Musify.Infrastructure.MassTransit.Activities.Tracks;
+using Musify.Infrastructure.MassTransit.Arguments;
+using Musify.Infrastructure.MassTransit.RoutingSlip;
+
+namespace Musify.Infrastructure.MassTransit.Consumers
+{
+    public class TrackProcessingFailedConsumer(IBus bus) : IConsumer<TrackProcessingFailed>
+    {
+        public async Task Consume(ConsumeContext<TrackProcessingFailed> context)
+        {
+            var message = context.Message;
+
+            var builder = new RoutingSlipBuilder(NewId.NextGuid());
+            builder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName),
+                RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted);
+
+            if (!string.IsNullOrWhiteSpace(message.Bucket) && !string.IsNullOrWhiteSpace(message.PictureKey))
+                builder.AddActivity(
+                    ActivityNames.RemoveTrackOriginalPicture,
+                    EndpointHelper.BuildExecuteActivityUri(RemoveFileFromBucketActivity.ExecuteEndpointName),
+                    new RemoveFileFromBucketArguments(message.Bucket, message.PictureKey));
+
+            if (!string.IsNullOrWhiteSpace(message.Bucket) && !string.IsNullOrWhiteSpace(message.AudioKey))
+                builder.AddActivity(
+                    ActivityNames.RemoveTrackOriginalAudio,
+                    EndpointHelper.BuildExecuteActivityUri(RemoveFileFromBucketActivity.ExecuteEndpointName),
+                    new RemoveFileFromBucketArguments(message.Bucket, message.AudioKey));
+
+            builder.AddActivity(
+                ActivityNames.MarkTrackAsFailed,
+                EndpointHelper.BuildExecuteActivityUri(MarkTrackAsFailedActivity.ExecuteEndpointName),
+                new MarkTrackAsFailedArguments(message.TrackId));
+
+            await bus.Execute(builder.Build());
+        }
+    }
+}
@@ -0,0 +1,33 @@
+using MassTransit;
+using MassTransit.Courier.Contracts;
+using Musify.Application.Events;
+
+namespace Musify.Infrastructure.MassTransit.Consumers
+{
+    public class TrackProcessingSlipFaultConsumer : IConsumer<RoutingSlipFaulted>
+    {
+        public const string QueueName = "track-processing-slip-fault";
+
+        public async Task Consume(ConsumeContext<RoutingSlipFaulted> context)
+        {
+            var variables = context.Message.Variables;
+
+            if (!variables.TryGetValue(RoutingSlipVariableNames.Workflow.ProcessKind, out var kindObject))
+                return;
+
+            if (!variables.TryGetValue(RoutingSlipVariableNames.Workflow.TrackId, out var trackObject)
+                || !Guid.TryParse(trackObject?.ToString(), out var trackId))
+                return;
+
+            switch (kindObject?.ToString())
+            {
+                case RoutingSlipVariableNames.ProcessKinds.TrackPicture:
+                    await context.Publish(new TrackPictureProcessingFailed(trackId));
+                    break;
+                case RoutingSlipVariableNames.ProcessKinds.TrackAudio:
+                    await context.Publish(new TrackAudioProcessingFailed(trackId));
+                    break;
+            }
+        }
+    }
+}
@@ -36,6 +36,8 @@ namespace Musify.Infrastructure.MassTransit
            options.AddConsumer<CreateTrackConsumer>();
            options.AddConsumer<CreatePlayListConsumer>();
            options.AddConsumer<UpdatePlayListPictureSourceConsumer>();
+            options.AddConsumer<TrackProcessingSlipFaultConsumer>();
+            options.AddConsumer<TrackProcessingFailedConsumer>();

            options.AddExecuteActivity<RemoveFileFromBucketActivity, RemoveFileFromBucketArguments>();
            options.AddExecuteActivity<GenerateAudioWorkflowPathsActivity, GenerateAudioWorkflowPathsArguments>();
@@ -46,6 +48,7 @@ namespace Musify.Infrastructure.MassTransit
            options.AddExecuteActivity<DeletePlayListFromDbActivity, DeletePlayListFromDbArguments>();
            options.AddExecuteActivity<PublishTrackProcessingEventsActivity, PublishTrackProcessingEventsArguments>();
            options.AddExecuteActivity<PublishPlayListPictureProcessingEventActivity, PublishPlayListPictureProcessingEventArguments>();
+            options.AddExecuteActivity<MarkTrackAsFailedActivity, MarkTrackAsFailedArguments>();

            options.AddActivity<ResizePictureActivity, ResizePictureLocalArguments, ResizePictureLog>();
            options.AddActivity<UpdatePlayListPictureActivity, UpdatePlayListPictureArguments, UpdatePlayListPictureLog>();
@@ -3,6 +3,7 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip
    internal static class ActivityNames
    {
        internal const string MarkTrackAsRemoving = "MarkTrackAsRemoving";
+        internal const string MarkTrackAsFailed = "MarkTrackAsFailed";
        internal const string DeleteTrackFromDb = "DeleteTrackFromDb";
        internal const string MarkPlayListAsRemoving = "MarkPlayListAsRemoving";
        internal const string DeletePlayListFromDb = "DeletePlayListFromDb";
@@ -60,6 +60,12 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip.Builders
                    message.TrackId,
                    message.DestinationFolderKey));

+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.TrackId, message.TrackId);
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackAudio);
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(TrackProcessingSlipFaultConsumer.QueueName),
+                RoutingSlipEvents.Faulted);
+
            return routingSlipBuilder;
        }
    }
@@ -32,6 +32,12 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip.Builders
                    RoutingSlipVariableNames.Picture.MediumResizedFilePath,
                    RoutingSlipVariableNames.Picture.LargeResizedFilePath));

+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.TrackId, message.TrackId);
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackPicture);
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(TrackProcessingSlipFaultConsumer.QueueName),
+                RoutingSlipEvents.Faulted);
+
            return routingSlipBuilder;
        }

@@ -6,6 +6,14 @@ namespace Musify.Infrastructure.MassTransit
        {
            internal const string TemporalDirectory = "Workflow.TemporalDirectory";
            internal const string CorrelationId = "Workflow.CorrelationId";
+            internal const string TrackId = "Workflow.TrackId";
+            internal const string ProcessKind = "Workflow.ProcessKind";
+        }
+
+        internal static class ProcessKinds
+        {
+            internal const string TrackPicture = "TrackPicture";
+            internal const string TrackAudio = "TrackAudio";
        }

        internal static class Audio
@@ -12,6 +12,12 @@ namespace Musify.Infrastructure.MassTransit.Sagas

        public bool AudioProcessed { get; set; }

+        public string? Bucket { get; set; }
+
+        public string? PictureKey { get; set; }
+
+        public string? AudioKey { get; set; }
+
        public DateTime CreatedAt { get; set; }

        public DateTime UpdatedAt { get; set; }
@@ -30,6 +30,9 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                    {
                        context.Saga.CreatedAt = DateTime.UtcNow;
                        context.Saga.UpdatedAt = DateTime.UtcNow;
+                        context.Saga.Bucket = context.Message.Bucket;
+                        context.Saga.PictureKey = context.Message.PictureDestinationKey;
+                        context.Saga.AudioKey = context.Message.AudioDestinationKey;
                    })
                    .TransitionTo(Processing));

@@ -50,10 +53,20 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                    .If(context => context.Saga.PictureProcessed, binder => binder.Finalize()),
                When(PictureFailed)
                    .Then(context => context.Saga.UpdatedAt = DateTime.UtcNow)
-                    .TransitionTo(Failed),
+                    .TransitionTo(Failed)
+                    .Publish(context => new TrackProcessingFailed(
+                        context.Saga.CorrelationId,
+                        context.Saga.Bucket,
+                        context.Saga.PictureKey,
+                        context.Saga.AudioKey)),
                When(AudioFailed)
                    .Then(context => context.Saga.UpdatedAt = DateTime.UtcNow)
-                    .TransitionTo(Failed));
+                    .TransitionTo(Failed)
+                    .Publish(context => new TrackProcessingFailed(
+                        context.Saga.CorrelationId,
+                        context.Saga.Bucket,
+                        context.Saga.PictureKey,
+                        context.Saga.AudioKey)));

            During(Failed,
                Ignore(PictureProcessed),