diff --git a/backend/Musify.Api/DataTransferObjects/PlayLists/AddYouTubeTrackRequest.cs b/backend/Musify.Api/DataTransferObjects/PlayLists/AddYouTubeTrackRequest.cs new file mode 100644 index 0000000..e69f190 --- /dev/null +++ b/backend/Musify.Api/DataTransferObjects/PlayLists/AddYouTubeTrackRequest.cs @@ -0,0 +1,8 @@ +namespace Musify.Api.DataTransferObjects.PlayLists; + +public record AddYouTubeTrackRequest( + string VideoId, + string Title, + string Artist, + int DurationSeconds, + string ThumbnailUrl); diff --git a/backend/Musify.Api/Endpoints/PlayListEndpoints.cs b/backend/Musify.Api/Endpoints/PlayListEndpoints.cs index 37abeb3..a8b8882 100644 --- a/backend/Musify.Api/Endpoints/PlayListEndpoints.cs +++ b/backend/Musify.Api/Endpoints/PlayListEndpoints.cs @@ -93,6 +93,17 @@ public static class PlayListEndpoints .Produces(StatusCodes.Status404NotFound) .Produces(StatusCodes.Status409Conflict); + group.MapPost("/{playlistId}/youtube-tracks", AddYouTubeTrackToPlayList) + .WithName("AddYouTubeTrackToPlayList") + .WithSummary("Add A YouTube Track To A PlayList, Downloading It In The Background If Needed.") + .AddEndpointFilter>() + .RequireAuthorization() + .Produces(StatusCodes.Status201Created) + .ProducesValidationProblem() + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound) + .Produces(StatusCodes.Status409Conflict); + group.MapDelete("/{playlistId}/tracks/{trackId}", RemoveTrackFromPlayList) .WithName("RemoveTrackFromPlayList") .WithSummary("Remove A Track From A PlayList.") @@ -232,6 +243,27 @@ public static class PlayListEndpoints return result.ToHttpResult(); } + private static async Task AddYouTubeTrackToPlayList( + IMediator mediator, + CurrentUser currentUser, + Guid playlistId, + AddYouTubeTrackRequest request, + CancellationToken cancellationToken) + { + var result = await mediator.Send( + new AddYouTubeTrackToPlayListCommand( + currentUser.RequiredId, + playlistId, + request.VideoId, + request.Title, + request.Artist, + request.DurationSeconds, + request.ThumbnailUrl), + cancellationToken); + + return result.ToCreatedResult(track => $"/tracks/{track.Id}"); + } + private static async Task RemoveTrackFromPlayList( IMediator mediator, CurrentUser currentUser, diff --git a/backend/Musify.Api/Validators/PlayLists/AddYouTubeTrackRequestValidator.cs b/backend/Musify.Api/Validators/PlayLists/AddYouTubeTrackRequestValidator.cs new file mode 100644 index 0000000..a9623f8 --- /dev/null +++ b/backend/Musify.Api/Validators/PlayLists/AddYouTubeTrackRequestValidator.cs @@ -0,0 +1,29 @@ +using FluentValidation; +using Musify.Api.DataTransferObjects.PlayLists; + +namespace Musify.Api.Validators.PlayLists; + +public sealed class AddYouTubeTrackRequestValidator : AbstractValidator +{ + public AddYouTubeTrackRequestValidator() + { + RuleFor(x => x.VideoId) + .NotEmpty() + .Matches("^[A-Za-z0-9_-]{11}$"); + + RuleFor(x => x.Title) + .NotEmpty() + .MaximumLength(200); + + RuleFor(x => x.Artist) + .MaximumLength(400); + + RuleFor(x => x.DurationSeconds) + .GreaterThan(0); + + RuleFor(x => x.ThumbnailUrl) + .NotEmpty() + .Must(url => Uri.TryCreate(url, UriKind.Absolute, out var uri) && uri.Scheme == Uri.UriSchemeHttps) + .WithMessage("ThumbnailUrl must be an absolute https URL."); + } +} diff --git a/backend/Musify.Application/Events/DownloadYouTubeTrackEvent.cs b/backend/Musify.Application/Events/DownloadYouTubeTrackEvent.cs new file mode 100644 index 0000000..dd880a6 --- /dev/null +++ b/backend/Musify.Application/Events/DownloadYouTubeTrackEvent.cs @@ -0,0 +1,14 @@ +using Musify.Application.Shared; + +namespace Musify.Application.Events +{ + public record DownloadYouTubeTrackEvent( + Guid TrackId, + string VideoId, + string ThumbnailUrl, + string Bucket, + string AudioProcessedFolderKey, + ImageSize Small, + ImageSize Medium, + ImageSize Large); +} diff --git a/backend/Musify.Application/PlayLists/AddYouTubeTrackToPlayList.cs b/backend/Musify.Application/PlayLists/AddYouTubeTrackToPlayList.cs new file mode 100644 index 0000000..695715f --- /dev/null +++ b/backend/Musify.Application/PlayLists/AddYouTubeTrackToPlayList.cs @@ -0,0 +1,170 @@ +using ErrorOr; +using Mediator; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Musify.Application.Configuration; +using Musify.Application.Contracts; +using Musify.Application.Events; +using Musify.Application.Shared; +using Musify.Application.Tracks.Responses; +using Musify.Domain.Entities; +using Musify.Domain.ValueObjects; + +namespace Musify.Application.PlayLists +{ + public record AddYouTubeTrackToPlayListCommand( + long UserId, + Guid PlayListId, + string VideoId, + string Title, + string Artist, + int DurationSeconds, + string ThumbnailUrl) + : ICommand>; + + public class AddYouTubeTrackToPlayListCommandHandler( + IDatabase database, + IEventBus eventBus, + ApplicationStorageConfiguration storageConfiguration, + TrackConfiguration trackConfiguration, + ILogger logger) + : ICommandHandler> + { + public async ValueTask> Handle(AddYouTubeTrackToPlayListCommand request, CancellationToken cancellationToken) + { + var playList = await database.PlayLists.AsNoTracking() + .SingleOrDefaultAsync(pl => pl.Id == request.PlayListId, cancellationToken); + if (playList is null) + { + logger.LogInformation("Playlist {PlayListId} not found", request.PlayListId); + return Error.NotFound(); + } + + if (playList.UserId != request.UserId) + { + logger.LogWarning("User {UserId} is not the owner of playlist {PlayListId}", request.UserId, request.PlayListId); + return Error.Unauthorized(); + } + + var trackResult = await GetOrCreateTrackAsync(request, cancellationToken); + if (trackResult.IsError) + return trackResult.Errors; + + var track = trackResult.Value; + + var alreadyAdded = await database.PlayListHasTracks.AsNoTracking() + .AnyAsync(plt => plt.PlayListId == request.PlayListId && plt.TrackId == track.Id, cancellationToken); + if (alreadyAdded) + { + logger.LogInformation("Track {TrackId} already in playlist {PlayListId}", track.Id, request.PlayListId); + return Error.Conflict(description: "Track is already in the playlist."); + } + + var nextPosition = await database.PlayListHasTracks + .Where(plt => plt.PlayListId == request.PlayListId) + .Select(plt => (int?)plt.Position) + .MaxAsync(cancellationToken) + 1 ?? 0; + + await database.PlayListHasTracks.AddAsync(new PlayListHasTrack + { + PlayListId = request.PlayListId, + TrackId = track.Id, + Position = nextPosition + }, cancellationToken); + + await database.SaveChangesAsync(cancellationToken); + + logger.LogInformation("Added YouTube track {VideoId} ({TrackId}) to playlist {PlayListId} at position {Position}", + request.VideoId, track.Id, request.PlayListId, nextPosition); + + var listensCount = await database.ListeningHistories.AsNoTracking() + .CountAsync(lh => lh.TrackId == track.Id, cancellationToken); + + return TrackApplicationResponse.FromEntity(track, listensCount); + } + + private async Task> GetOrCreateTrackAsync(AddYouTubeTrackToPlayListCommand request, CancellationToken cancellationToken) + { + var track = await database.Tracks + .SingleOrDefaultAsync(t => t.Source == TrackSource.YouTube && t.ExternalId == request.VideoId, cancellationToken); + + if (track is not null) + { + if (track.AudioTranscodeProcessingStatus == ProcessingStatus.Failed) + { + track.AudioTranscodeProcessingStatus = ProcessingStatus.Pending; + track.PicturesProcessingStatus = ProcessingStatus.Pending; + track.LifeCycleStatus = LifeCycleStatus.Active; + track.RetryCount += 1; + track.LastRetryAt = DateTime.UtcNow; + await PublishDownloadEventAsync(track.Id, request, cancellationToken); + await database.SaveChangesAsync(cancellationToken); + logger.LogInformation("Re-queued failed YouTube track {VideoId} for download", request.VideoId); + } + return track; + } + + var title = Truncate(request.Title, 50); + track = new Track + { + Title = title, + NormalizedTitle = title.ToUpperInvariant(), + Artist = Truncate(request.Artist, 200), + Source = TrackSource.YouTube, + ExternalId = request.VideoId, + Duration = request.DurationSeconds, + OriginalPictureName = string.Empty, + OriginalAudioName = string.Empty, + SmallPictureName = trackConfiguration.Routes.PresetSmallPicture, + MediumPictureName = trackConfiguration.Routes.PresetMediumPicture, + LargePictureName = trackConfiguration.Routes.PresetLargePicture, + PicturesProcessingStatus = ProcessingStatus.Pending, + AudioTranscodeProcessingStatus = ProcessingStatus.Pending + }; + + await database.Tracks.AddAsync(track, cancellationToken); + await PublishDownloadEventAsync(track.Id, request, cancellationToken); + + try + { + await database.SaveChangesAsync(cancellationToken); + logger.LogInformation("Created YouTube track {VideoId} ({TrackId}) and queued its download", request.VideoId, track.Id); + return track; + } + catch (DbUpdateException) + { + logger.LogInformation("YouTube track {VideoId} was created concurrently", request.VideoId); + return Error.Conflict(description: "The track was just created by another request. Retry the operation."); + } + } + + private async Task PublishDownloadEventAsync(Guid trackId, AddYouTubeTrackToPlayListCommand request, CancellationToken cancellationToken) + { + var audioProcessedFolderKey = trackConfiguration.Routes.BuildProcessedAudioPath(Guid.NewGuid().ToString()); + + await eventBus.PublishAsync( + new DownloadYouTubeTrackEvent( + trackId, + request.VideoId, + request.ThumbnailUrl, + storageConfiguration.Bucket, + audioProcessedFolderKey, + new ImageSize( + trackConfiguration.Routes.SmallPicturesPath, + trackConfiguration.PicturesSizes.SmallPictureWidth, + trackConfiguration.PicturesSizes.SmallPictureHeight), + new ImageSize( + trackConfiguration.Routes.MediumPicturesPath, + trackConfiguration.PicturesSizes.MediumPictureWidth, + trackConfiguration.PicturesSizes.MediumPictureHeight), + new ImageSize( + trackConfiguration.Routes.LargePicturesPath, + trackConfiguration.PicturesSizes.LargePictureWidth, + trackConfiguration.PicturesSizes.LargePictureHeight)), + cancellationToken); + } + + private static string Truncate(string value, int maxLength) => + value.Length <= maxLength ? value : value[..maxLength]; + } +} diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Audio/DownloadYouTubeAudioActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/DownloadYouTubeAudioActivity.cs new file mode 100644 index 0000000..c2b5738 --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Audio/DownloadYouTubeAudioActivity.cs @@ -0,0 +1,94 @@ +using MassTransit; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Musify.Application.Contracts; +using Musify.Domain.Entities; +using Musify.Domain.ValueObjects; +using Musify.Infrastructure.MassTransit.Arguments; +using Musify.Infrastructure.MassTransit.Logs; + +namespace Musify.Infrastructure.MassTransit.Activities.Audio +{ + internal class DownloadYouTubeAudioActivity( + IDatabase database, + IYouTubeDownloaderService youTubeDownloader, + ILogger logger) + : IActivity + { + public const string ExecuteEndpointName = "download-you-tube-audio"; + + public async Task Execute(ExecuteContext executeContext) + { + var destinationPath = executeContext.GetVariable(executeContext.Arguments.DestinationFilePathVariable); + ArgumentNullException.ThrowIfNull(destinationPath, nameof(destinationPath)); + + Track track; + try + { + var getTrack = await database.Tracks.SingleOrDefaultAsync(t => t.Id == executeContext.Arguments.TrackId, executeContext.CancellationToken); + track = getTrack ?? throw new InvalidOperationException($"Track with Id {executeContext.Arguments.TrackId} not found"); + } + catch (Exception exception) + { + logger.LogError(exception, "Failed to retrieve track with Id {TrackId} from database", executeContext.Arguments.TrackId); + throw; + } + + try + { + var downloadResult = await youTubeDownloader.DownloadAudioAsync( + executeContext.Arguments.VideoId, + destinationPath, + executeContext.CancellationToken); + + if (downloadResult.ExitCode != 0) + throw new InvalidOperationException( + $"yt-dlp failed with exit code {downloadResult.ExitCode}: {downloadResult.StandardError}"); + + if (!File.Exists(destinationPath)) + throw new InvalidOperationException($"yt-dlp did not produce the expected file {destinationPath}."); + + logger.LogInformation("Downloaded YouTube audio for video {VideoId} to {DestinationPath}", + executeContext.Arguments.VideoId, destinationPath); + + return executeContext.Completed(new DownloadYouTubeAudioLog( + executeContext.Arguments.VideoId, + destinationPath)); + } + catch (Exception exception) + { + logger.LogError(exception, "Failed to download YouTube audio for video {VideoId}", + executeContext.Arguments.VideoId); + + try + { + track.AudioTranscodeProcessingStatus = ProcessingStatus.Failed; + await database.SaveChangesAsync(executeContext.CancellationToken); + } + catch (Exception dbException) + { + logger.LogError(dbException, "Failed to update ProcessingStatus to Failed for track {TrackId}", + executeContext.Arguments.TrackId); + } + + throw; + } + } + + public Task Compensate(CompensateContext compensateContext) + { + try + { + if (File.Exists(compensateContext.Log.DestinationFilePath)) + File.Delete(compensateContext.Log.DestinationFilePath); + return Task.FromResult(compensateContext.Compensated()); + } + catch (Exception exception) + { + logger.LogError(exception, "Failed to compensate downloaded YouTube audio {DestinationFilePath}", + compensateContext.Log.DestinationFilePath); + return Task.FromResult(compensateContext.Failed(exception)); + } + } + } +} diff --git a/backend/Musify.Infrastructure/MassTransit/Activities/Files/DownloadFileFromUrlActivity.cs b/backend/Musify.Infrastructure/MassTransit/Activities/Files/DownloadFileFromUrlActivity.cs new file mode 100644 index 0000000..6565766 --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Activities/Files/DownloadFileFromUrlActivity.cs @@ -0,0 +1,69 @@ +using MassTransit; +using Microsoft.Extensions.Logging; +using Musify.Infrastructure.MassTransit.Arguments; +using Musify.Infrastructure.MassTransit.Logs; + +namespace Musify.Infrastructure.MassTransit.Activities.Files +{ + internal class DownloadFileFromUrlActivity( + IHttpClientFactory httpClientFactory, + ILogger logger) + : IActivity + { + public const string ExecuteEndpointName = "download-file-from-url"; + + public async Task Execute(ExecuteContext executeContext) + { + var destinationPath = executeContext.GetVariable(executeContext.Arguments.DestinationFilePathVariable); + ArgumentNullException.ThrowIfNull(destinationPath, nameof(destinationPath)); + + try + { + var httpClient = httpClientFactory.CreateClient(); + + using var response = await httpClient.GetAsync( + executeContext.Arguments.SourceUrl, + HttpCompletionOption.ResponseHeadersRead, + executeContext.CancellationToken); + response.EnsureSuccessStatusCode(); + + var destinationDirectory = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrWhiteSpace(destinationDirectory)) + Directory.CreateDirectory(destinationDirectory); + + await using (var destinationStream = File.Create(destinationPath)) + { + await response.Content.CopyToAsync(destinationStream, executeContext.CancellationToken); + } + + logger.LogInformation("Downloaded {SourceUrl} to {DestinationPath}", + executeContext.Arguments.SourceUrl, destinationPath); + + return executeContext.Completed(new DownloadFileFromUrlLog( + executeContext.Arguments.SourceUrl, + destinationPath)); + } + catch (Exception exception) + { + logger.LogError(exception, "Failed to download file from {SourceUrl}", executeContext.Arguments.SourceUrl); + throw; + } + } + + public Task Compensate(CompensateContext compensateContext) + { + try + { + if (File.Exists(compensateContext.Log.DestinationFilePath)) + File.Delete(compensateContext.Log.DestinationFilePath); + return Task.FromResult(compensateContext.Compensated()); + } + catch (Exception exception) + { + logger.LogError(exception, "Failed to compensate downloaded file {DestinationFilePath}", + compensateContext.Log.DestinationFilePath); + return Task.FromResult(compensateContext.Failed(exception)); + } + } + } +} diff --git a/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadFileFromUrlArguments.cs b/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadFileFromUrlArguments.cs new file mode 100644 index 0000000..5d7422b --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadFileFromUrlArguments.cs @@ -0,0 +1,6 @@ +namespace Musify.Infrastructure.MassTransit.Arguments +{ + public record DownloadFileFromUrlArguments( + string SourceUrl, + string DestinationFilePathVariable); +} diff --git a/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadYouTubeAudioArguments.cs b/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadYouTubeAudioArguments.cs new file mode 100644 index 0000000..abe62bb --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Arguments/DownloadYouTubeAudioArguments.cs @@ -0,0 +1,7 @@ +namespace Musify.Infrastructure.MassTransit.Arguments +{ + public record DownloadYouTubeAudioArguments( + Guid TrackId, + string VideoId, + string DestinationFilePathVariable); +} diff --git a/backend/Musify.Infrastructure/MassTransit/Consumers/DownloadYouTubeTrackConsumer.cs b/backend/Musify.Infrastructure/MassTransit/Consumers/DownloadYouTubeTrackConsumer.cs new file mode 100644 index 0000000..f724e7c --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Consumers/DownloadYouTubeTrackConsumer.cs @@ -0,0 +1,26 @@ +using MassTransit; +using Musify.Application.Events; +using Musify.Infrastructure.MassTransit.RoutingSlip.Builders; + +namespace Musify.Infrastructure.MassTransit.Consumers +{ + public class DownloadYouTubeTrackConsumer( + IBus bus, + YouTubeTrackRoutingSlipBuilder routingSlipBuilder) + : IConsumer + { + public async Task Consume(ConsumeContext consumeContext) + { + var audioRoutingSlip = routingSlipBuilder + .BuildAudioWorkflow(consumeContext.Message, consumeContext.CorrelationId) + .Build(); + + var pictureRoutingSlip = routingSlipBuilder + .BuildPictureWorkflow(consumeContext.Message, consumeContext.CorrelationId) + .Build(); + + await bus.Execute(audioRoutingSlip); + await bus.Execute(pictureRoutingSlip); + } + } +} diff --git a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs index ccfdb8b..c54e033 100644 --- a/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs +++ b/backend/Musify.Infrastructure/MassTransit/DependencyInjection/MassTransitDependencyInjection.Registration.cs @@ -23,6 +23,7 @@ namespace Musify.Infrastructure.MassTransit services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } static void RegisterConsumersAndActivities(IBusRegistrationConfigurator options) @@ -39,6 +40,7 @@ namespace Musify.Infrastructure.MassTransit options.AddConsumer(); options.AddConsumer(); options.AddConsumer(); + options.AddConsumer(); options.AddExecuteActivity(); options.AddExecuteActivity(); @@ -62,6 +64,8 @@ namespace Musify.Infrastructure.MassTransit options.AddActivity(); options.AddActivity(); options.AddActivity(); + options.AddActivity(); + options.AddActivity(); } } } diff --git a/backend/Musify.Infrastructure/MassTransit/Logs/DownloadFileFromUrlLog.cs b/backend/Musify.Infrastructure/MassTransit/Logs/DownloadFileFromUrlLog.cs new file mode 100644 index 0000000..5c80cc1 --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Logs/DownloadFileFromUrlLog.cs @@ -0,0 +1,6 @@ +namespace Musify.Infrastructure.MassTransit.Logs +{ + public record DownloadFileFromUrlLog( + string SourceUrl, + string DestinationFilePath); +} diff --git a/backend/Musify.Infrastructure/MassTransit/Logs/DownloadYouTubeAudioLog.cs b/backend/Musify.Infrastructure/MassTransit/Logs/DownloadYouTubeAudioLog.cs new file mode 100644 index 0000000..2623e8b --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/Logs/DownloadYouTubeAudioLog.cs @@ -0,0 +1,6 @@ +namespace Musify.Infrastructure.MassTransit.Logs +{ + public record DownloadYouTubeAudioLog( + string VideoId, + string DestinationFilePath); +} diff --git a/backend/Musify.Infrastructure/MassTransit/RoutingSlip/ActivityNames.cs b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/ActivityNames.cs index 902f107..0cf2953 100644 --- a/backend/Musify.Infrastructure/MassTransit/RoutingSlip/ActivityNames.cs +++ b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/ActivityNames.cs @@ -13,6 +13,8 @@ namespace Musify.Infrastructure.MassTransit.RoutingSlip internal const string GeneratePictureWorkflowPaths = "GeneratePictureWorkflowPaths"; internal const string DownloadFile = "DownloadFile"; + internal const string DownloadYouTubeAudio = "DownloadYouTubeAudio"; + internal const string DownloadThumbnail = "DownloadThumbnail"; internal const string TranscodeAudio = "TranscodeAudio"; diff --git a/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/YouTubeTrackRoutingSlipBuilder.cs b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/YouTubeTrackRoutingSlipBuilder.cs new file mode 100644 index 0000000..91c0f0c --- /dev/null +++ b/backend/Musify.Infrastructure/MassTransit/RoutingSlip/Builders/YouTubeTrackRoutingSlipBuilder.cs @@ -0,0 +1,166 @@ +using MassTransit; +using MassTransit.Courier.Contracts; +using Musify.Application.Events; +using Musify.Application.Shared; +using Musify.Infrastructure.Configuration; +using Musify.Infrastructure.MassTransit.Activities.Audio; +using Musify.Infrastructure.MassTransit.Activities.Files; +using Musify.Infrastructure.MassTransit.Activities.Pictures; +using Musify.Infrastructure.MassTransit.Arguments; +using Musify.Infrastructure.MassTransit.Consumers; + +namespace Musify.Infrastructure.MassTransit.RoutingSlip.Builders +{ + public class YouTubeTrackRoutingSlipBuilder(WorkerConfiguration workerConfiguration) + { + public RoutingSlipBuilder BuildAudioWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId) + { + var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid()); + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty); + + routingSlipBuilder.AddSubscription( + EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName), + RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted); + + routingSlipBuilder.AddActivity( + ActivityNames.GenerateAudioWorkflowPaths, + EndpointHelper.BuildExecuteActivityUri(GenerateAudioWorkflowPathsActivity.ExecuteEndpointName), + new GenerateAudioWorkflowPathsArguments( + message.TrackId, + workerConfiguration.Routes.TemporaryFilesDirectory, + $"{message.VideoId}.m4a")); + + routingSlipBuilder.AddActivity( + ActivityNames.DownloadYouTubeAudio, + EndpointHelper.BuildExecuteActivityUri(DownloadYouTubeAudioActivity.ExecuteEndpointName), + new DownloadYouTubeAudioArguments( + message.TrackId, + message.VideoId, + RoutingSlipVariableNames.Audio.SourceFilePath)); + + routingSlipBuilder.AddActivity( + ActivityNames.TranscodeAudio, + EndpointHelper.BuildExecuteActivityUri(TranscodeAudioActivity.ExecuteEndpointName), + new TranscodeAudioArguments( + message.TrackId, + RoutingSlipVariableNames.Audio.SourceFilePath, + RoutingSlipVariableNames.Workflow.TemporalDirectory)); + + routingSlipBuilder.AddActivity( + ActivityNames.TransferFiles, + EndpointHelper.BuildExecuteActivityUri(TransferFilesToBucketActivity.ExecuteEndpointName), + new TransferFilesToBucketArguments( + message.Bucket, + RoutingSlipVariableNames.Audio.TranscodedDirectory, + message.AudioProcessedFolderKey)); + + routingSlipBuilder.AddActivity( + ActivityNames.UpdateTrackAudio, + EndpointHelper.BuildExecuteActivityUri(UpdateTrackAudioActivity.ExecuteEndpointName), + new UpdateTrackAudioArguments( + message.TrackId, + message.AudioProcessedFolderKey, + RoutingSlipVariableNames.Audio.DurationSeconds)); + + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId); + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackAudio); + routingSlipBuilder.AddSubscription( + EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName), + RoutingSlipEvents.Faulted); + + return routingSlipBuilder; + } + + public RoutingSlipBuilder BuildPictureWorkflow(DownloadYouTubeTrackEvent message, Guid? correlationId) + { + var routingSlipBuilder = new RoutingSlipBuilder(NewId.NextGuid()); + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.CorrelationId, correlationId ?? Guid.Empty); + + routingSlipBuilder.AddSubscription( + EndpointHelper.BuildConsumerUri(RoutingSlipCleanUpConsumer.QueueName), + RoutingSlipEvents.Completed | RoutingSlipEvents.Faulted); + + routingSlipBuilder.AddActivity( + ActivityNames.GeneratePictureWorkflowPaths, + EndpointHelper.BuildExecuteActivityUri(GeneratePictureWorkflowPathsActivity.ExecuteEndpointName), + new GeneratePictureWorkflowPathsArguments( + workerConfiguration.Routes.TemporaryFilesDirectory, + $"{message.VideoId}.jpg")); + + routingSlipBuilder.AddActivity( + ActivityNames.DownloadThumbnail, + EndpointHelper.BuildExecuteActivityUri(DownloadFileFromUrlActivity.ExecuteEndpointName), + new DownloadFileFromUrlArguments( + message.ThumbnailUrl, + RoutingSlipVariableNames.Picture.OriginalFilePath)); + + AddResizeActivity( + routingSlipBuilder, + ActivityNames.ResizeSmall, + ActivityNames.UploadSmall, + message.Bucket, + RoutingSlipVariableNames.Picture.SmallResizedFilePath, + message.Small); + + AddResizeActivity( + routingSlipBuilder, + ActivityNames.ResizeMedium, + ActivityNames.UploadMedium, + message.Bucket, + RoutingSlipVariableNames.Picture.MediumResizedFilePath, + message.Medium); + + AddResizeActivity( + routingSlipBuilder, + ActivityNames.ResizeLarge, + ActivityNames.UploadLarge, + message.Bucket, + RoutingSlipVariableNames.Picture.LargeResizedFilePath, + message.Large); + + routingSlipBuilder.AddActivity( + ActivityNames.UpdateTrackPicture, + EndpointHelper.BuildExecuteActivityUri(UpdateTrackPictureActivity.ExecuteEndpointName), + new UpdateTrackPictureArguments( + message.TrackId, + string.Empty, + RoutingSlipVariableNames.Picture.SmallResizedFilePath, + RoutingSlipVariableNames.Picture.MediumResizedFilePath, + RoutingSlipVariableNames.Picture.LargeResizedFilePath)); + + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.SubjectId, message.TrackId); + routingSlipBuilder.AddVariable(RoutingSlipVariableNames.Workflow.ProcessKind, RoutingSlipVariableNames.ProcessKinds.TrackPicture); + routingSlipBuilder.AddSubscription( + EndpointHelper.BuildConsumerUri(ProcessingSlipFaultConsumer.QueueName), + RoutingSlipEvents.Faulted); + + return routingSlipBuilder; + } + + static void AddResizeActivity( + RoutingSlipBuilder routingSlipBuilder, + string resizeActivityName, + string uploadActivityName, + string bucketName, + string destinationFilePathVariableName, + ImageSize imageSize) + { + routingSlipBuilder.AddActivity( + resizeActivityName, + EndpointHelper.BuildExecuteActivityUri(ResizePictureActivity.ExecuteEndpointName), + new ResizePictureLocalArguments( + RoutingSlipVariableNames.Picture.OriginalFilePath, + destinationFilePathVariableName, + imageSize.Width, + imageSize.Height)); + + routingSlipBuilder.AddActivity( + uploadActivityName, + EndpointHelper.BuildExecuteActivityUri(UploadFileToBucketActivity.ExecuteEndpointName), + new UploadFileToBucketArguments( + destinationFilePathVariableName, + bucketName, + imageSize.SavePath)); + } + } +} diff --git a/backend/Musify.Infrastructure/MassTransit/Sagas/TrackProcessingStateMachine.cs b/backend/Musify.Infrastructure/MassTransit/Sagas/TrackProcessingStateMachine.cs index 9b157fb..7cd8c47 100644 --- a/backend/Musify.Infrastructure/MassTransit/Sagas/TrackProcessingStateMachine.cs +++ b/backend/Musify.Infrastructure/MassTransit/Sagas/TrackProcessingStateMachine.cs @@ -9,6 +9,7 @@ namespace Musify.Infrastructure.MassTransit.Sagas public State Failed { get; private set; } = null!; public Event ProcessingStarted { get; private set; } = null!; + public Event YouTubeDownloadStarted { get; private set; } = null!; public Event PictureProcessed { get; private set; } = null!; public Event AudioProcessed { get; private set; } = null!; public Event PictureFailed { get; private set; } = null!; @@ -19,6 +20,7 @@ namespace Musify.Infrastructure.MassTransit.Sagas InstanceState(state => state.CurrentState); Event(() => ProcessingStarted, config => config.CorrelateById(context => context.Message.TrackId)); + Event(() => YouTubeDownloadStarted, config => config.CorrelateById(context => context.Message.TrackId)); Event(() => PictureProcessed, config => config.CorrelateById(context => context.Message.TrackId)); Event(() => AudioProcessed, config => config.CorrelateById(context => context.Message.TrackId)); Event(() => PictureFailed, config => config.CorrelateById(context => context.Message.TrackId)); @@ -34,9 +36,20 @@ namespace Musify.Infrastructure.MassTransit.Sagas context.Saga.PictureKey = context.Message.PictureDestinationKey; context.Saga.AudioKey = context.Message.AudioDestinationKey; } ) + .TransitionTo(Processing), + When(YouTubeDownloadStarted) + .Then(context => + { + context.Saga.CreatedAt = DateTime.UtcNow; + context.Saga.UpdatedAt = DateTime.UtcNow; + context.Saga.Bucket = context.Message.Bucket; + context.Saga.PictureKey = string.Empty; + context.Saga.AudioKey = string.Empty; + } ) .TransitionTo(Processing)); During(Processing, + Ignore(YouTubeDownloadStarted), When(PictureProcessed) .Then(context => { @@ -72,7 +85,18 @@ namespace Musify.Infrastructure.MassTransit.Sagas Ignore(PictureProcessed), Ignore(AudioProcessed), Ignore(PictureFailed), - Ignore(AudioFailed)); + Ignore(AudioFailed), + When(YouTubeDownloadStarted) + .Then(context => + { + context.Saga.UpdatedAt = DateTime.UtcNow; + context.Saga.PictureProcessed = false; + context.Saga.AudioProcessed = false; + context.Saga.Bucket = context.Message.Bucket; + context.Saga.PictureKey = string.Empty; + context.Saga.AudioKey = string.Empty; + } ) + .TransitionTo(Processing)); SetCompletedWhenFinalized(); } diff --git a/backend/Musify.Worker/AppSettings.json b/backend/Musify.Worker/AppSettings.json index 7f58644..d0cd821 100644 --- a/backend/Musify.Worker/AppSettings.json +++ b/backend/Musify.Worker/AppSettings.json @@ -85,6 +85,12 @@ }, "TranscodingTimeout": "00:05:00" }, + "YtDlp": { + "ExecutableName": "yt-dlp", + "Format": "bestaudio", + "AdditionalArguments": "", + "DownloadTimeout": "00:05:00" + }, "Workers": { "Routes": { "TemporaryFilesDirectory": "D:\\tempsFilesDev" diff --git a/backend/Musify.Worker/Program.cs b/backend/Musify.Worker/Program.cs index 45e76c9..5178d26 100644 --- a/backend/Musify.Worker/Program.cs +++ b/backend/Musify.Worker/Program.cs @@ -14,6 +14,8 @@ services.AddValidatedOptions(configuration, PlayListConfi services.AddValidatedOptions(configuration, TrackConfiguration.SectionName); services.AddStorageService(configuration); services.AddAudioTranscoder(configuration); +services.AddYouTubeDownloader(configuration); +services.AddHttpClient(); services.AddPictureService(configuration); services.AddDatabase(configuration); services.AddMassTransitConsumers(configuration);