Remove YouTube integration from backend and deployment config

Drops the YouTube-backed second catalog source entirely: the Local/External split on Track/Album collapses into single concrete entities, Artist/TrackArtist/ExternalTrack/ExternalAlbum and the YouTube service/download pipeline (yt-dlp, MassTransit consumer, routing slip) are deleted, and mixes now source discovery only from local listening history. Includes the RemoveYouTubeIntegration EF Core migration and matching Worker/deploy/compose cleanup (no more yt-dlp/python/deno in the Worker image, no thumbnail-cache/YtDlp config sections).
2026-09-15 21:17:03 +02:00
parent 5696683859
commit 31032bd8e8
137 changed files with 1933 additions and 3969 deletions
@@ -1,94 +0,0 @@
-using MassTransit;
-using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.Logging;
-using Musify.Application.Contracts;
-using Musify.Domain.Entities;
-using Musify.Domain.ValueObjects;
-using Musify.Infrastructure.MassTransit.Arguments;
-using Musify.Infrastructure.MassTransit.Logs;
-
-namespace Musify.Infrastructure.MassTransit.Activities.Audio
-{
-    internal class DownloadYouTubeAudioActivity(
-        IDatabase database,
-        IYouTubeDownloaderService youTubeDownloader,
-        ILogger<DownloadYouTubeAudioActivity> logger)
-        : IActivity<DownloadYouTubeAudioArguments, DownloadYouTubeAudioLog>
-    {
-        public const string ExecuteEndpointName = "download-you-tube-audio";
-
-        public async Task<ExecutionResult> Execute(ExecuteContext<DownloadYouTubeAudioArguments> executeContext)
-        {
-            var destinationPath = executeContext.GetVariable<string>(executeContext.Arguments.DestinationFilePathVariable);
-            ArgumentNullException.ThrowIfNull(destinationPath, nameof(destinationPath));
-
-            Track track;
-            try
-            {
-                var getTrack = await database.Tracks.SingleOrDefaultAsync(t => t.Id == executeContext.Arguments.TrackId, executeContext.CancellationToken);
-                track = getTrack ?? throw new InvalidOperationException($"Track with Id {executeContext.Arguments.TrackId} not found");
-            }
-            catch (Exception exception)
-            {
-                logger.LogError(exception, "Failed to retrieve track with Id {TrackId} from database", executeContext.Arguments.TrackId);
-                throw;
-            }
-
-            try
-            {
-                var downloadResult = await youTubeDownloader.DownloadAudioAsync(
-                    executeContext.Arguments.VideoId,
-                    destinationPath,
-                    executeContext.CancellationToken);
-
-                if (downloadResult.ExitCode != 0)
-                    throw new InvalidOperationException(
-                        $"YouTube audio download failed: {downloadResult.StandardError}");
-
-                if (!File.Exists(destinationPath))
-                    throw new InvalidOperationException($"YouTube audio download did not produce the expected file {destinationPath}.");
-
-                logger.LogInformation("Downloaded YouTube audio for video {VideoId} to {DestinationPath}",
-                    executeContext.Arguments.VideoId, destinationPath);
-
-                return executeContext.Completed(new DownloadYouTubeAudioLog(
-                    executeContext.Arguments.VideoId,
-                    destinationPath));
-            }
-            catch (Exception exception)
-            {
-                logger.LogError(exception, "Failed to download YouTube audio for video {VideoId}",
-                    executeContext.Arguments.VideoId);
-
-                try
-                {
-                    track.Audio.TranscodeStatus = ProcessingStatus.Failed;
-                    await database.SaveChangesAsync(executeContext.CancellationToken);
-                }
-                catch (Exception dbException)
-                {
-                    logger.LogError(dbException, "Failed to update ProcessingStatus to Failed for track {TrackId}",
-                        executeContext.Arguments.TrackId);
-                }
-
-                throw;
-            }
-        }
-
-        public Task<CompensationResult> Compensate(CompensateContext<DownloadYouTubeAudioLog> compensateContext)
-        {
-            try
-            {
-                if (File.Exists(compensateContext.Log.DestinationFilePath))
-                    File.Delete(compensateContext.Log.DestinationFilePath);
-                return Task.FromResult(compensateContext.Compensated());
-            }
-            catch (Exception exception)
-            {
-                logger.LogError(exception, "Failed to compensate downloaded YouTube audio {DestinationFilePath}",
-                    compensateContext.Log.DestinationFilePath);
-                return Task.FromResult(compensateContext.Failed(exception));
-            }
-        }
-    }
-}
@@ -1,7 +0,0 @@
-namespace Musify.Infrastructure.MassTransit.Arguments
-{
-    public record DownloadYouTubeAudioArguments(
-        Guid TrackId,
-        string VideoId,
-        string DestinationFilePathVariable);
-}
@@ -1,26 +0,0 @@
-using MassTransit;
-using Musify.Application.Events;
-using Musify.Infrastructure.MassTransit.RoutingSlip.Builders;
-
-namespace Musify.Infrastructure.MassTransit.Consumers
-{
-    public class DownloadYouTubeTrackConsumer(
-        IBus bus,
-        YouTubeTrackRoutingSlipBuilder routingSlipBuilder)
-        : IConsumer<DownloadYouTubeTrackEvent>
-    {
-        public async Task Consume(ConsumeContext<DownloadYouTubeTrackEvent> consumeContext)
-        {
-            var audioRoutingSlip = routingSlipBuilder
-                .BuildAudioWorkflow(consumeContext.Message, consumeContext.CorrelationId)
-                .Build();
-
-            var pictureRoutingSlip = routingSlipBuilder
-                .BuildPictureWorkflow(consumeContext.Message, consumeContext.CorrelationId)
-                .Build();
-
-            await bus.Execute(audioRoutingSlip);
-            await bus.Execute(pictureRoutingSlip);
-        }
-    }
-}
@@ -25,7 +25,6 @@ namespace Musify.Infrastructure.MassTransit
            services.AddScoped<CreateTrackRoutingSlipBuilder>();
            services.AddScoped<PlayListPictureSourceRoutingSlipBuilder>();
            services.AddScoped<AlbumPictureSourceRoutingSlipBuilder>();
-            services.AddScoped<YouTubeTrackRoutingSlipBuilder>();
        }

        static void RegisterConsumersAndActivities(IBusRegistrationConfigurator options)
@@ -42,7 +41,6 @@ namespace Musify.Infrastructure.MassTransit
            options.AddConsumer<ProcessingSlipFaultConsumer>();
            options.AddConsumer<TrackProcessingFailedConsumer>();
            options.AddConsumer<PlayListProcessingFailedConsumer>();
-            options.AddConsumer<DownloadYouTubeTrackConsumer>();
            options.AddConsumer<CreateAlbumConsumer>();
            options.AddConsumer<UpdateAlbumPictureSourceConsumer>();
            options.AddConsumer<UpdateAlbumPictureConsumer>();
@@ -73,7 +71,6 @@ namespace Musify.Infrastructure.MassTransit
            options.AddActivity<UploadFileToBucketActivity, UploadFileToBucketArguments, UploadFileToBucketLog>();
            options.AddActivity<CopyFileInBucketActivity, CopyFileInBucketArguments, CopyFileInBucketLog>();
            options.AddActivity<ConsumeUploadIntentsActivity, ConsumeUploadIntentsArguments, ConsumeUploadIntentsLog>();
-            options.AddActivity<DownloadYouTubeAudioActivity, DownloadYouTubeAudioArguments, DownloadYouTubeAudioLog>();
            options.AddActivity<DownloadFileFromUrlActivity, DownloadFileFromUrlArguments, DownloadFileFromUrlLog>();
        }
    }
@@ -1,6 +0,0 @@
-namespace Musify.Infrastructure.MassTransit.Logs
-{
-    public record DownloadYouTubeAudioLog(
-        string VideoId,
-        string DestinationFilePath);
-}
@@ -14,7 +14,6 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip
        internal const string GeneratePictureWorkflowPaths = "GeneratePictureWorkflowPaths";

        internal const string DownloadFile = "DownloadFile";
-        internal const string DownloadYouTubeAudio = "DownloadYouTubeAudio";
        internal const string DownloadThumbnail = "DownloadThumbnail";

        internal const string TranscodeAudio = "TranscodeAudio";
@@ -1,166 +0,0 @@
-using MassTransit;
-using MassTransit.Courier.Contracts;
-using Musify.Application.Events;
-using Musify.Application.Shared;
-using Musify.Infrastructure.Configuration;
-using Musify.Infrastructure.MassTransit.Activities.Audio;
-using Musify.Infrastructure.MassTransit.Activities.Files;
-using Musify.Infrastructure.MassTransit.Activities.Pictures;
-using Musify.Infrastructure.MassTransit.Arguments;
-using Musify.Infrastructure.MassTransit.Consumers;
-
-namespace Musify.Infrastructure.MassTransit.RoutingSlip.Builders
-{
-    public class YouTubeTrackRoutingSlipBuilder(WorkerConfiguration workerConfiguration)
-    {
-        public RoutingSlipBuilder BuildAudioWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId)
-        {
-            var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid());
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty);
-
-            routingSlipBuilder.AddSubscription(
-                EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName),
-                RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted);
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.GenerateAudioWorkflowPaths,
-                EndpointHelper.BuildExecuteActivityUri(GenerateAudioWorkflowPathsActivity.ExecuteEndpointName),
-                new GenerateAudioWorkflowPathsArguments(
-                    message.TrackId,
-                    workerConfiguration.Routes.TemporaryFilesDirectory,
-                    $"{message.VideoId}.m4a"));
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.DownloadYouTubeAudio,
-                EndpointHelper.BuildExecuteActivityUri(DownloadYouTubeAudioActivity.ExecuteEndpointName),
-                new DownloadYouTubeAudioArguments(
-                    message.TrackId,
-                    message.VideoId,
-                    RoutingSlipVariableNames.Audio.SourceFilePath));
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.TranscodeAudio,
-                EndpointHelper.BuildExecuteActivityUri(TranscodeAudioActivity.ExecuteEndpointName),
-                new TranscodeAudioArguments(
-                    message.TrackId,
-                    RoutingSlipVariableNames.Audio.SourceFilePath,
-                    RoutingSlipVariableNames.Workflow.TemporalDirectory));
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.TransferFiles,
-                EndpointHelper.BuildExecuteActivityUri(TransferFilesToBucketActivity.ExecuteEndpointName),
-                new TransferFilesToBucketArguments(
-                    message.Bucket,
-                    RoutingSlipVariableNames.Audio.TranscodedDirectory,
-                    message.AudioProcessedFolderKey));
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.UpdateTrackAudio,
-                EndpointHelper.BuildExecuteActivityUri(UpdateTrackAudioActivity.ExecuteEndpointName),
-                new UpdateTrackAudioArguments(
-                    message.TrackId,
-                    message.AudioProcessedFolderKey,
-                    RoutingSlipVariableNames.Audio.DurationSeconds));
-
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId);
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackAudio);
-            routingSlipBuilder.AddSubscription(
-                EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName),
-                RoutingSlipEvents.Faulted);
-
-            return routingSlipBuilder;
-        }
-
-        public RoutingSlipBuilder BuildPictureWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId)
-        {
-            var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid());
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty);
-
-            routingSlipBuilder.AddSubscription(
-                EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName),
-                RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted);
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.GeneratePictureWorkflowPaths,
-                EndpointHelper.BuildExecuteActivityUri(GeneratePictureWorkflowPathsActivity.ExecuteEndpointName),
-                new GeneratePictureWorkflowPathsArguments(
-                    workerConfiguration.Routes.TemporaryFilesDirectory,
-                    $"{message.VideoId}.jpg"));
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.DownloadThumbnail,
-                EndpointHelper.BuildExecuteActivityUri(DownloadFileFromUrlActivity.ExecuteEndpointName),
-                new DownloadFileFromUrlArguments(
-                    message.ThumbnailUrl,
-                    RoutingSlipVariableNames.Picture.OriginalFilePath));
-
-            AddResizeActivity(
-                routingSlipBuilder,
-                ActivityNames.ResizeSmall,
-                ActivityNames.UploadSmall,
-                message.Bucket,
-                RoutingSlipVariableNames.Picture.SmallResizedFilePath,
-                message.Small);
-
-            AddResizeActivity(
-                routingSlipBuilder,
-                ActivityNames.ResizeMedium,
-                ActivityNames.UploadMedium,
-                message.Bucket,
-                RoutingSlipVariableNames.Picture.MediumResizedFilePath,
-                message.Medium);
-
-            AddResizeActivity(
-                routingSlipBuilder,
-                ActivityNames.ResizeLarge,
-                ActivityNames.UploadLarge,
-                message.Bucket,
-                RoutingSlipVariableNames.Picture.LargeResizedFilePath,
-                message.Large);
-
-            routingSlipBuilder.AddActivity(
-                ActivityNames.UpdateTrackPicture,
-                EndpointHelper.BuildExecuteActivityUri(UpdateTrackPictureActivity.ExecuteEndpointName),
-                new UpdateTrackPictureArguments(
-                    message.TrackId,
-                    string.Empty,
-                    RoutingSlipVariableNames.Picture.SmallResizedFilePath,
-                    RoutingSlipVariableNames.Picture.MediumResizedFilePath,
-                    RoutingSlipVariableNames.Picture.LargeResizedFilePath));
-
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId);
-            routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackPicture);
-            routingSlipBuilder.AddSubscription(
-                EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName),
-                RoutingSlipEvents.Faulted);
-
-            return routingSlipBuilder;
-        }
-
-        static void AddResizeActivity(
-            RoutingSlipBuilder routingSlipBuilder,
-            string resizeActivityName,
-            string uploadActivityName,
-            string bucketName,
-            string destinationFilePathVariableName,
-            ImageSize imageSize)
-        {
-            routingSlipBuilder.AddActivity(
-                resizeActivityName,
-                EndpointHelper.BuildExecuteActivityUri(ResizePictureActivity.ExecuteEndpointName),
-                new ResizePictureLocalArguments(
-                    RoutingSlipVariableNames.Picture.OriginalFilePath,
-                    destinationFilePathVariableName,
-                    imageSize.Width,
-                    imageSize.Height));
-
-            routingSlipBuilder.AddActivity(
-                uploadActivityName,
-                EndpointHelper.BuildExecuteActivityUri(UploadFileToBucketActivity.ExecuteEndpointName),
-                new UploadFileToBucketArguments(
-                    destinationFilePathVariableName,
-                    bucketName,
-                    imageSize.SavePath));
-        }
-    }
-}
@@ -9,7 +9,6 @@ namespace Musify.Infrastructure.MassTransit.Sagas
        public State Failed { get; private set; } = null!;

        public Event<CreateTrackResourcesEvent> ProcessingStarted { get; private set; } = null!;
-        public Event<DownloadYouTubeTrackEvent> YouTubeDownloadStarted { get; private set; } = null!;
        public Event<TrackPictureProcessed> PictureProcessed { get; private set; } = null!;
        public Event<TrackAudioProcessed> AudioProcessed { get; private set; } = null!;
        public Event<TrackPictureProcessingFailed> PictureFailed { get; private set; } = null!;
@@ -20,7 +19,6 @@ namespace Musify.Infrastructure.MassTransit.Sagas
            InstanceState(state => state.CurrentState);

            Event(() => ProcessingStarted, config => config.CorrelateById(context => context.Message.TrackId));
-            Event(() => YouTubeDownloadStarted, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => PictureProcessed, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => AudioProcessed, config => config.CorrelateById(context => context.Message.TrackId));
            Event(() => PictureFailed, config => config.CorrelateById(context => context.Message.TrackId));
@@ -36,20 +34,9 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                        context.Saga.PictureKey = context.Message.PictureDestinationKey;
                        context.Saga.AudioKey = context.Message.AudioDestinationKey;
                    } )
-                    .TransitionTo(Processing),
-                When(YouTubeDownloadStarted)
-                    .Then(context =>
-                    {
-                        context.Saga.CreatedAt = DateTime.UtcNow;
-                        context.Saga.UpdatedAt = DateTime.UtcNow;
-                        context.Saga.Bucket = context.Message.Bucket;
-                        context.Saga.PictureKey = string.Empty;
-                        context.Saga.AudioKey = string.Empty;
-                    } )
                    .TransitionTo(Processing));

            During(Processing,
-                Ignore(YouTubeDownloadStarted),
                When(PictureProcessed)
                    .Then(context =>
                    {
@@ -85,18 +72,7 @@ namespace Musify.Infrastructure.MassTransit.Sagas
                Ignore(PictureProcessed),
                Ignore(AudioProcessed),
                Ignore(PictureFailed),
-                Ignore(AudioFailed),
-                When(YouTubeDownloadStarted)
-                    .Then(context =>
-                    {
-                        context.Saga.UpdatedAt = DateTime.UtcNow;
-                        context.Saga.PictureProcessed = false;
-                        context.Saga.AudioProcessed = false;
-                        context.Saga.Bucket = context.Message.Bucket;
-                        context.Saga.PictureKey = string.Empty;
-                        context.Saga.AudioKey = string.Empty;
-                    } )
-                    .TransitionTo(Processing));
+                Ignore(AudioFailed));

            SetCompletedWhenFinalized();
        }