feat(backend): auto-download YouTube tracks saved to playlists

2026-07-16 01:06:04 +02:00
parent 2bd5ef729e
commit 24595a17d6
18 changed files with 672 additions and 1 deletions
@@ -0,0 +1,94 @@
+using MassTransit;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Musify.Application.Contracts;
+using Musify.Domain.Entities;
+using Musify.Domain.ValueObjects;
+using Musify.Infrastructure.MassTransit.Arguments;
+using Musify.Infrastructure.MassTransit.Logs;
+
+namespace Musify.Infrastructure.MassTransit.Activities.Audio
+{
+    internal class DownloadYouTubeAudioActivity(
+        IDatabase database,
+        IYouTubeDownloaderService youTubeDownloader,
+        ILogger<DownloadYouTubeAudioActivity> logger)
+        : IActivity<DownloadYouTubeAudioArguments, DownloadYouTubeAudioLog>
+    {
+        public const string ExecuteEndpointName = "download-you-tube-audio";
+
+        public async Task<ExecutionResult> Execute(ExecuteContext<DownloadYouTubeAudioArguments> executeContext)
+        {
+            var destinationPath = executeContext.GetVariable<string>(executeContext.Arguments.DestinationFilePathVariable);
+            ArgumentNullException.ThrowIfNull(destinationPath, nameof(destinationPath));
+
+            Track track;
+            try
+            {
+                var getTrack = await database.Tracks.SingleOrDefaultAsync(t => t.Id == executeContext.Arguments.TrackId, executeContext.CancellationToken);
+                track = getTrack ?? throw new InvalidOperationException($"Track with Id {executeContext.Arguments.TrackId} not found");
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Failed to retrieve track with Id {TrackId} from database", executeContext.Arguments.TrackId);
+                throw;
+            }
+
+            try
+            {
+                var downloadResult = await youTubeDownloader.DownloadAudioAsync(
+                    executeContext.Arguments.VideoId,
+                    destinationPath,
+                    executeContext.CancellationToken);
+
+                if (downloadResult.ExitCode != 0)
+                    throw new InvalidOperationException(
+                        $"yt-dlp failed with exit code {downloadResult.ExitCode}: {downloadResult.StandardError}");
+
+                if (!File.Exists(destinationPath))
+                    throw new InvalidOperationException($"yt-dlp did not produce the expected file {destinationPath}.");
+
+                logger.LogInformation("Downloaded YouTube audio for video {VideoId} to {DestinationPath}",
+                    executeContext.Arguments.VideoId, destinationPath);
+
+                return executeContext.Completed(new DownloadYouTubeAudioLog(
+                    executeContext.Arguments.VideoId,
+                    destinationPath));
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Failed to download YouTube audio for video {VideoId}",
+                    executeContext.Arguments.VideoId);
+
+                try
+                {
+                    track.AudioTranscodeProcessingStatus = ProcessingStatus.Failed;
+                    await database.SaveChangesAsync(executeContext.CancellationToken);
+                }
+                catch (Exception dbException)
+                {
+                    logger.LogError(dbException, "Failed to update ProcessingStatus to Failed for track {TrackId}",
+                        executeContext.Arguments.TrackId);
+                }
+
+                throw;
+            }
+        }
+
+        public Task<CompensationResult> Compensate(CompensateContext<DownloadYouTubeAudioLog> compensateContext)
+        {
+            try
+            {
+                if (File.Exists(compensateContext.Log.DestinationFilePath))
+                    File.Delete(compensateContext.Log.DestinationFilePath);
+                return Task.FromResult(compensateContext.Compensated());
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Failed to compensate downloaded YouTube audio {DestinationFilePath}",
+                    compensateContext.Log.DestinationFilePath);
+                return Task.FromResult(compensateContext.Failed(exception));
+            }
+        }
+    }
+}
@@ -0,0 +1,69 @@
+using MassTransit;
+using Microsoft.Extensions.Logging;
+using Musify.Infrastructure.MassTransit.Arguments;
+using Musify.Infrastructure.MassTransit.Logs;
+
+namespace Musify.Infrastructure.MassTransit.Activities.Files
+{
+    internal class DownloadFileFromUrlActivity(
+        IHttpClientFactory httpClientFactory,
+        ILogger<DownloadFileFromUrlActivity> logger)
+        : IActivity<DownloadFileFromUrlArguments, DownloadFileFromUrlLog>
+    {
+        public const string ExecuteEndpointName = "download-file-from-url";
+
+        public async Task<ExecutionResult> Execute(ExecuteContext<DownloadFileFromUrlArguments> executeContext)
+        {
+            var destinationPath = executeContext.GetVariable<string>(executeContext.Arguments.DestinationFilePathVariable);
+            ArgumentNullException.ThrowIfNull(destinationPath, nameof(destinationPath));
+
+            try
+            {
+                var httpClient = httpClientFactory.CreateClient();
+
+                using var response = await httpClient.GetAsync(
+                    executeContext.Arguments.SourceUrl,
+                    HttpCompletionOption.ResponseHeadersRead,
+                    executeContext.CancellationToken);
+                response.EnsureSuccessStatusCode();
+
+                var destinationDirectory = Path.GetDirectoryName(destinationPath);
+                if (!string.IsNullOrWhiteSpace(destinationDirectory))
+                    Directory.CreateDirectory(destinationDirectory);
+
+                await using (var destinationStream = File.Create(destinationPath))
+                {
+                    await response.Content.CopyToAsync(destinationStream, executeContext.CancellationToken);
+                }
+
+                logger.LogInformation("Downloaded {SourceUrl} to {DestinationPath}",
+                    executeContext.Arguments.SourceUrl, destinationPath);
+
+                return executeContext.Completed(new DownloadFileFromUrlLog(
+                    executeContext.Arguments.SourceUrl,
+                    destinationPath));
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Failed to download file from {SourceUrl}", executeContext.Arguments.SourceUrl);
+                throw;
+            }
+        }
+
+        public Task<CompensationResult> Compensate(CompensateContext<DownloadFileFromUrlLog> compensateContext)
+        {
+            try
+            {
+                if (File.Exists(compensateContext.Log.DestinationFilePath))
+                    File.Delete(compensateContext.Log.DestinationFilePath);
+                return Task.FromResult(compensateContext.Compensated());
+            }
+            catch (Exception exception)
+            {
+                logger.LogError(exception, "Failed to compensate downloaded file {DestinationFilePath}",
+                    compensateContext.Log.DestinationFilePath);
+                return Task.FromResult(compensateContext.Failed(exception));
+            }
+        }
+    }
+}
@@ -0,0 +1,6 @@
+namespace Musify.Infrastructure.MassTransit.Arguments
+{
+    public record DownloadFileFromUrlArguments(
+        string SourceUrl,
+        string DestinationFilePathVariable);
+}
@@ -0,0 +1,7 @@
+namespace Musify.Infrastructure.MassTransit.Arguments
+{
+    public record DownloadYouTubeAudioArguments(
+        Guid TrackId,
+        string VideoId,
+        string DestinationFilePathVariable);
+}
@@ -0,0 +1,26 @@
+using MassTransit;
+using Musify.Application.Events;
+using Musify.Infrastructure.MassTransit.RoutingSlip.Builders;
+
+namespace Musify.Infrastructure.MassTransit.Consumers
+{
+    public class DownloadYouTubeTrackConsumer(
+        IBus bus,
+        YouTubeTrackRoutingSlipBuilder routingSlipBuilder)
+        : IConsumer<DownloadYouTubeTrackEvent>
+    {
+        public async Task Consume(ConsumeContext<DownloadYouTubeTrackEvent> consumeContext)
+        {
+            var audioRoutingSlip = routingSlipBuilder
+                .BuildAudioWorkflow(consumeContext.Message, consumeContext.CorrelationId)
+                .Build();
+
+            var pictureRoutingSlip = routingSlipBuilder
+                .BuildPictureWorkflow(consumeContext.Message, consumeContext.CorrelationId)
+                .Build();
+
+            await bus.Execute(audioRoutingSlip);
+            await bus.Execute(pictureRoutingSlip);
+        }
+    }
+}
@@ -23,6 +23,7 @@ namespace Musify.Infrastructure.MassTransit
            services.AddScoped<DeletePlayListRoutingSlipBuilder>();
            services.AddScoped<CreateTrackRoutingSlipBuilder>();
            services.AddScoped<PlayListPictureSourceRoutingSlipBuilder>();
+            services.AddScoped<YouTubeTrackRoutingSlipBuilder>();
        }

        static void RegisterConsumersAndActivities(IBusRegistrationConfigurator options)
@@ -39,6 +40,7 @@ namespace Musify.Infrastructure.MassTransit
            options.AddConsumer<ProcessingSlipFaultConsumer>();
            options.AddConsumer<TrackProcessingFailedConsumer>();
            options.AddConsumer<PlayListProcessingFailedConsumer>();
+            options.AddConsumer<DownloadYouTubeTrackConsumer>();

            options.AddExecuteActivity<RemoveFileFromBucketActivity, RemoveFileFromBucketArguments>();
            options.AddExecuteActivity<GenerateAudioWorkflowPathsActivity, GenerateAudioWorkflowPathsArguments>();
@@ -62,6 +64,8 @@ namespace Musify.Infrastructure.MassTransit
            options.AddActivity<UploadFileToBucketActivity, UploadFileToBucketArguments, UploadFileToBucketLog>();
            options.AddActivity<CopyFileInBucketActivity, CopyFileInBucketArguments, CopyFileInBucketLog>();
            options.AddActivity<ConsumeUploadIntentsActivity, ConsumeUploadIntentsArguments, ConsumeUploadIntentsLog>();
+            options.AddActivity<DownloadYouTubeAudioActivity, DownloadYouTubeAudioArguments, DownloadYouTubeAudioLog>();
+            options.AddActivity<DownloadFileFromUrlActivity, DownloadFileFromUrlArguments, DownloadFileFromUrlLog>();
        }
    }
 }
@@ -0,0 +1,6 @@
+namespace Musify.Infrastructure.MassTransit.Logs
+{
+    public record DownloadFileFromUrlLog(
+        string SourceUrl,
+        string DestinationFilePath);
+}
@@ -0,0 +1,6 @@
+namespace Musify.Infrastructure.MassTransit.Logs
+{
+    public record DownloadYouTubeAudioLog(
+        string VideoId,
+        string DestinationFilePath);
+}
@@ -13,6 +13,8 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip
        internal const string GeneratePictureWorkflowPaths = "GeneratePictureWorkflowPaths";

        internal const string DownloadFile = "DownloadFile";
+        internal const string DownloadYouTubeAudio = "DownloadYouTubeAudio";
+        internal const string DownloadThumbnail = "DownloadThumbnail";

        internal const string TranscodeAudio = "TranscodeAudio";

@@ -0,0 +1,166 @@
+using MassTransit;
+using MassTransit.Courier.Contracts;
+using Musify.Application.Events;
+using Musify.Application.Shared;
+using Musify.Infrastructure.Configuration;
+using Musify.Infrastructure.MassTransit.Activities.Audio;
+using Musify.Infrastructure.MassTransit.Activities.Files;
+using Musify.Infrastructure.MassTransit.Activities.Pictures;
+using Musify.Infrastructure.MassTransit.Arguments;
+using Musify.Infrastructure.MassTransit.Consumers;
+
+namespace Musify.Infrastructure.MassTransit.RoutingSlip.Builders
+{
+    public class YouTubeTrackRoutingSlipBuilder(WorkerConfiguration workerConfiguration)
+    {
+        public RoutingSlipBuilder BuildAudioWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId)
+        {
+            var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid());
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty);
+
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName),
+                RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted);
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.GenerateAudioWorkflowPaths,
+                EndpointHelper.BuildExecuteActivityUri(GenerateAudioWorkflowPathsActivity.ExecuteEndpointName),
+                new GenerateAudioWorkflowPathsArguments(
+                    message.TrackId,
+                    workerConfiguration.Routes.TemporaryFilesDirectory,
+                    $"{message.VideoId}.m4a"));
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.DownloadYouTubeAudio,
+                EndpointHelper.BuildExecuteActivityUri(DownloadYouTubeAudioActivity.ExecuteEndpointName),
+                new DownloadYouTubeAudioArguments(
+                    message.TrackId,
+                    message.VideoId,
+                    RoutingSlipVariableNames.Audio.SourceFilePath));
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.TranscodeAudio,
+                EndpointHelper.BuildExecuteActivityUri(TranscodeAudioActivity.ExecuteEndpointName),
+                new TranscodeAudioArguments(
+                    message.TrackId,
+                    RoutingSlipVariableNames.Audio.SourceFilePath,
+                    RoutingSlipVariableNames.Workflow.TemporalDirectory));
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.TransferFiles,
+                EndpointHelper.BuildExecuteActivityUri(TransferFilesToBucketActivity.ExecuteEndpointName),
+                new TransferFilesToBucketArguments(
+                    message.Bucket,
+                    RoutingSlipVariableNames.Audio.TranscodedDirectory,
+                    message.AudioProcessedFolderKey));
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.UpdateTrackAudio,
+                EndpointHelper.BuildExecuteActivityUri(UpdateTrackAudioActivity.ExecuteEndpointName),
+                new UpdateTrackAudioArguments(
+                    message.TrackId,
+                    message.AudioProcessedFolderKey,
+                    RoutingSlipVariableNames.Audio.DurationSeconds));
+
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId);
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackAudio);
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName),
+                RoutingSlipEvents.Faulted);
+
+            return routingSlipBuilder;
+        }
+
+        public RoutingSlipBuilder BuildPictureWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId)
+        {
+            var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid());
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty);
+
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName),
+                RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted);
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.GeneratePictureWorkflowPaths,
+                EndpointHelper.BuildExecuteActivityUri(GeneratePictureWorkflowPathsActivity.ExecuteEndpointName),
+                new GeneratePictureWorkflowPathsArguments(
+                    workerConfiguration.Routes.TemporaryFilesDirectory,
+                    $"{message.VideoId}.jpg"));
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.DownloadThumbnail,
+                EndpointHelper.BuildExecuteActivityUri(DownloadFileFromUrlActivity.ExecuteEndpointName),
+                new DownloadFileFromUrlArguments(
+                    message.ThumbnailUrl,
+                    RoutingSlipVariableNames.Picture.OriginalFilePath));
+
+            AddResizeActivity(
+                routingSlipBuilder,
+                ActivityNames.ResizeSmall,
+                ActivityNames.UploadSmall,
+                message.Bucket,
+                RoutingSlipVariableNames.Picture.SmallResizedFilePath,
+                message.Small);
+
+            AddResizeActivity(
+                routingSlipBuilder,
+                ActivityNames.ResizeMedium,
+                ActivityNames.UploadMedium,
+                message.Bucket,
+                RoutingSlipVariableNames.Picture.MediumResizedFilePath,
+                message.Medium);
+
+            AddResizeActivity(
+                routingSlipBuilder,
+                ActivityNames.ResizeLarge,
+                ActivityNames.UploadLarge,
+                message.Bucket,
+                RoutingSlipVariableNames.Picture.LargeResizedFilePath,
+                message.Large);
+
+            routingSlipBuilder.AddActivity(
+                ActivityNames.UpdateTrackPicture,
+                EndpointHelper.BuildExecuteActivityUri(UpdateTrackPictureActivity.ExecuteEndpointName),
+                new UpdateTrackPictureArguments(
+                    message.TrackId,
+                    string.Empty,
+                    RoutingSlipVariableNames.Picture.SmallResizedFilePath,
+                    RoutingSlipVariableNames.Picture.MediumResizedFilePath,
+                    RoutingSlipVariableNames.Picture.LargeResizedFilePath));
+
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId);
+            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackPicture);
+            routingSlipBuilder.AddSubscription(
+                EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName),
+                RoutingSlipEvents.Faulted);
+
+            return routingSlipBuilder;
+        }
+
+        static void AddResizeActivity(
+            RoutingSlipBuilder routingSlipBuilder,
+            string resizeActivityName,
+            string uploadActivityName,
+            string bucketName,
+            string destinationFilePathVariableName,
+            ImageSize imageSize)
+        {
+            routingSlipBuilder.AddActivity(
+                resizeActivityName,
+                EndpointHelper.BuildExecuteActivityUri(ResizePictureActivity.ExecuteEndpointName),
+                new ResizePictureLocalArguments(
+                    RoutingSlipVariableNames.Picture.OriginalFilePath,
+                    destinationFilePathVariableName,
+                    imageSize.Width,
+                    imageSize.Height));
+
+            routingSlipBuilder.AddActivity(
+                uploadActivityName,
+                EndpointHelper.BuildExecuteActivityUri(UploadFileToBucketActivity.ExecuteEndpointName),
+                new UploadFileToBucketArguments(
+                    destinationFilePathVariableName,
+                    bucketName,
+                    imageSize.SavePath));
+        }
+    }
+}
@@ -9,6 +9,7 @@ namespace Musify.Infrastructure.MassTransit.Sagas
        public State Failed { get; private set; } = null!;

        public Event<CreateTrackResourcesEvent> ProcessingStarted { get; private set; } = null!;
+        public Event<DownloadYouTubeTrackEvent> YouTubeDownloadStarted { get; private set; } = null!;
        public Event<TrackPictureProcessed> PictureProcessed { get; private set; } = null!;
        public Event<TrackAudioProcessed> AudioProcessed { get; private set; } = null!;
        public Event<TrackPictureProcessingFailed> PictureFailed { get; private set; } = null!;
@@ -19,6 +20,7 @@ namespace Musify.Infrastructure.MassTransit.Sagas
            InstanceState(state => state.CurrentState);

            Event(() => ProcessingStarted, config => config.CorrelateById(context => context.Message.TrackId));
+            Event(() => YouTubeDownloadStarted, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => PictureProcessed, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => AudioProcessed, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => PictureFailed, config => config.CorrelateById(context => context.Message.TrackId));
@@ -34,9 +36,20 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                        context.Saga.PictureKey = context.Message.PictureDestinationKey;
                        context.Saga.AudioKey = context.Message.AudioDestinationKey;
                    } )
+                    .TransitionTo(Processing),
+                When(YouTubeDownloadStarted)
+                    .Then(context =>
+                    {
+                        context.Saga.CreatedAt = DateTime.UtcNow;
+                        context.Saga.UpdatedAt = DateTime.UtcNow;
+                        context.Saga.Bucket = context.Message.Bucket;
+                        context.Saga.PictureKey = string.Empty;
+                        context.Saga.AudioKey = string.Empty;
+                    } )
                    .TransitionTo(Processing));

            During(Processing,
+                Ignore(YouTubeDownloadStarted),
                When(PictureProcessed)
                    .Then(context =>
                    {
@@ -72,7 +85,18 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                Ignore(PictureProcessed),
                Ignore(AudioProcessed),
                Ignore(PictureFailed),
-                Ignore(AudioFailed));
+                Ignore(AudioFailed),
+                When(YouTubeDownloadStarted)
+                    .Then(context =>
+                    {
+                        context.Saga.UpdatedAt = DateTime.UtcNow;
+                        context.Saga.PictureProcessed = false;
+                        context.Saga.AudioProcessed = false;
+                        context.Saga.Bucket = context.Message.Bucket;
+                        context.Saga.PictureKey = string.Empty;
+                        context.Saga.AudioKey = string.Empty;
+                    } )
+                    .TransitionTo(Processing));

            SetCompletedWhenFinalized();
        }