Remove YouTube integration from backend and deployment config

Drops the YouTube-backed second catalog source entirely: the
Local/External split on Track/Album collapses into single concrete
entities, Artist/TrackArtist/ExternalTrack/ExternalAlbum and the
YouTube service/download pipeline (yt-dlp, MassTransit consumer,
routing slip) are deleted, and mixes now source discovery only from
local listening history. Includes the RemoveYouTubeIntegration EF
Core migration and matching Worker/deploy/compose cleanup (no more
yt-dlp/python/deno in the Worker image, no thumbnail-cache/YtDlp
config sections).
This commit is contained in:
2026-09-15 21:17:03 +02:00
parent 5696683859
commit 31032bd8e8
137 changed files with 1933 additions and 3969 deletions
@@ -1,37 +0,0 @@
using ErrorOr;
using Musify.Application.Contracts;
namespace Musify.Infrastructure.Services
{
/// <summary>Stands in for <see cref="IYouTubeMusicService"/> when
/// <see cref="Musify.Infrastructure.Configuration.YouTubeConfiguration.Enabled"/> is <c>false</c>,
/// so the source can be turned off without a client ever being constructed. Every read fails
/// with the same <see cref="ErrorType.Forbidden"/> error; callers already treat a failed
/// YouTube call as "unavailable" (an HTTP error for direct requests, a skipped candidate for
/// mix generation), so nothing downstream needs to know the source is disabled specifically.</summary>
public sealed class DisabledYouTubeMusicService : IYouTubeMusicService
{
private static readonly Error DisabledError =
Error.Forbidden(description: "The YouTube Music source is disabled.");
public Task<ErrorOr<YouTubeSearchResult>> SearchSongsAsync(string query, string continuationToken, CancellationToken cancellationToken) =>
Task.FromResult<ErrorOr<YouTubeSearchResult>>(DisabledError);
public Task<ErrorOr<YouTubeAlbumSearchResult>> SearchAlbumsAsync(string query, string continuationToken, CancellationToken cancellationToken) =>
Task.FromResult<ErrorOr<YouTubeAlbumSearchResult>>(DisabledError);
public Task<ErrorOr<YouTubeAlbumDetail>> GetAlbumAsync(string albumId, CancellationToken cancellationToken) =>
Task.FromResult<ErrorOr<YouTubeAlbumDetail>>(DisabledError);
public Task<ErrorOr<YouTubeStreamInfo>> GetAudioStreamAsync(string videoId, CancellationToken cancellationToken) =>
Task.FromResult<ErrorOr<YouTubeStreamInfo>>(DisabledError);
public Task<ErrorOr<YouTubeSongResult>> GetSongAsync(string videoId, CancellationToken cancellationToken) =>
Task.FromResult<ErrorOr<YouTubeSongResult>>(DisabledError);
// Never reached in practice: every caller only reaches this after a successful search or
// song lookup above, neither of which can succeed while disabled. Passing the URL through
// unchanged is the safest thing to do if it's ever called anyway.
public string ResolveArtworkUrl(string thumbnailUrl) => thumbnailUrl;
}
}
@@ -1,7 +1,6 @@
using Amazon.S3;
using Hangfire;
using Hangfire.PostgreSql;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@@ -99,33 +98,6 @@ namespace Musify.Infrastructure.Services
return serviceDescriptors;
}
public static IServiceCollection AddYouTubeMusicService(this IServiceCollection serviceDescriptors, IConfiguration configuration)
{
serviceDescriptors.AddValidatedOptions<YouTubeConfiguration>(configuration, YouTubeConfiguration.SectionName);
serviceDescriptors.AddMemoryCache();
serviceDescriptors.AddSingleton<IYouTubeMusicService>(serviceProvider =>
{
var youTubeConfiguration = serviceProvider.GetRequiredService<YouTubeConfiguration>();
if (!youTubeConfiguration.Enabled)
return new DisabledYouTubeMusicService();
return new YouTubeMusicService(
youTubeConfiguration,
serviceProvider.GetRequiredService<IMemoryCache>(),
serviceProvider.GetRequiredService<ILogger<YouTubeMusicService>>());
} );
return serviceDescriptors;
}
public static IServiceCollection AddYouTubeDownloader(this IServiceCollection serviceDescriptors, IConfiguration configuration)
{
serviceDescriptors.AddValidatedOptions<YtDlpConfiguration>(configuration, YtDlpConfiguration.SectionName);
serviceDescriptors.AddScoped<IYouTubeDownloaderService, YouTubeDownloaderService>();
return serviceDescriptors;
}
public static IServiceCollection AddAudioTranscoder(this IServiceCollection serviceDescriptors, IConfiguration configuration)
{
serviceDescriptors
@@ -1,89 +0,0 @@
using Microsoft.Extensions.Logging;
using Musify.Application.Contracts;
using Musify.Infrastructure.Configuration;
using System.Diagnostics;
namespace Musify.Infrastructure.Services
{
public class YouTubeDownloaderService(
ILogger<YouTubeDownloaderService> logger,
YtDlpConfiguration configuration)
: IYouTubeDownloaderService
{
public async Task<YouTubeDownloadResult> DownloadAudioAsync(string videoId, string destinationFilePath, CancellationToken cancellationToken)
{
var destinationDirectory = Path.GetDirectoryName(destinationFilePath);
if (!string.IsNullOrWhiteSpace(destinationDirectory))
Directory.CreateDirectory(destinationDirectory);
var additionalArguments = string.IsNullOrWhiteSpace(configuration.AdditionalArguments)
? string.Empty
: $" {configuration.AdditionalArguments.Trim()}";
var arguments =
$"-f {configuration.Format} " +
"--no-playlist " +
$"-o \"{destinationFilePath}\"" +
additionalArguments +
$" -- {videoId}";
using var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = configuration.ExecutableName,
WorkingDirectory = destinationDirectory ?? Environment.CurrentDirectory,
Arguments = arguments,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
}
};
process.Start();
logger.LogInformation("yt-dlp started (PID: {ProcessId} ) for video {VideoId}", process.Id, videoId);
var outputTask = process.StandardOutput.ReadToEndAsync(cancellationToken);
var errorTask = process.StandardError.ReadToEndAsync(cancellationToken);
using var linkedCancellationTokens = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
linkedCancellationTokens.CancelAfter(configuration.DownloadTimeout);
try
{
await process.WaitForExitAsync(linkedCancellationTokens.Token);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
logger.LogError("yt-dlp timed out after {Timeout} for video {VideoId} (PID: {ProcessId} )",
configuration.DownloadTimeout, videoId, process.Id);
TryKillProcess(process);
throw new TimeoutException($"yt-dlp execution exceeded timeout {configuration.DownloadTimeout}.");
}
var standardError = await errorTask;
var standardOutput = await outputTask;
logger.LogDebug("yt-dlp finished (PID: {ProcessId} ). Exit code: {ExitCode}, Output: {OutputLength}B, Error: {ErrorLength}B",
process.Id, process.ExitCode, standardOutput.Length, standardError.Length);
return new YouTubeDownloadResult(process.ExitCode, standardError);
}
void TryKillProcess(Process process)
{
try
{
if (!process.HasExited)
process.Kill();
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to kill yt-dlp process (PID: {ProcessId} )", process.Id);
}
}
}
}
@@ -1,310 +0,0 @@
using ErrorOr;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Musify.Application.Contracts;
using Musify.Infrastructure.Configuration;
using Musify.Infrastructure.Helpers;
using YouTubeMusicAPI.Client;
using YouTubeMusicAPI.Models.Search;
using YouTubeMusicAPI.Models.Streaming;
using YouTubeMusicAPI.Pagination;
namespace Musify.Infrastructure.Services
{
public class YouTubeMusicService(YouTubeConfiguration configuration, IMemoryCache cache, ILogger<YouTubeMusicService> logger) : IYouTubeMusicService
{
private readonly YouTubeMusicClient client = new YouTubeMusicClient(geographicalLocation: configuration.GeographicalLocation);
private readonly SingleFlightCache singleFlight = new SingleFlightCache(cache);
private sealed record CachedStreamInfo(string Url, DateTime ExpiresAtUtc);
public async Task<ErrorOr<YouTubeSearchResult>> SearchSongsAsync(string query, string continuationToken, CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(continuationToken))
return await FetchSearchPageAsync(query, continuationToken, cancellationToken);
return await singleFlight.GetOrCreateAsync(
SearchResultsCacheKey(query),
TimeSpan.FromSeconds(configuration.SearchResultsCacheSeconds),
() => FetchSearchPageAsync(query, continuationToken, cancellationToken));
}
private sealed record SearchPage(IReadOnlyList<SearchResult> Results, string NextToken);
private async Task<ErrorOr<SearchPage>> FetchPageAsync(
string query,
SearchCategory category,
string continuationToken,
CancellationToken cancellationToken)
{
PaginatedAsyncEnumerable<SearchResult>? paginator;
if (string.IsNullOrEmpty(continuationToken))
{
paginator = client.SearchAsync(query, category);
}
else if (!cache.TryGetValue(SearchCacheKey(continuationToken), out paginator) || paginator is null)
{
return Error.NotFound(description: "The search continuation has expired.");
}
IReadOnlyList<SearchResult> results;
try
{
results = await paginator.FetchNextPageAsync(cancellationToken);
}
catch (Exception exception)
{
logger.LogError(exception, "YouTube Music {Category} search failed for query '{Query}'", category, query);
return Error.Failure(description: $"YouTube Music search failed: {exception.Message}");
}
var nextToken = string.Empty;
if (paginator.HasMore)
{
nextToken = Guid.NewGuid().ToString("N");
cache.Set(
SearchCacheKey(nextToken),
paginator,
TimeSpan.FromSeconds(configuration.SearchContinuationCacheSeconds));
}
return new SearchPage(results, nextToken);
}
private async Task<ErrorOr<YouTubeSearchResult>> FetchSearchPageAsync(string query, string continuationToken, CancellationToken cancellationToken)
{
var page = await FetchPageAsync(query, SearchCategory.Songs, continuationToken, cancellationToken);
if (page.IsError)
return page.Errors;
var items = page.Value.Results
.OfType<SongSearchResult>()
.Select(song => new YouTubeSongResult(
song.Id,
song.Name,
string.Join(", ", song.Artists.Select(artist => artist.Name)),
song.Album?.Name ?? string.Empty,
(int)song.Duration.TotalSeconds,
PickThumbnail(song.Thumbnails),
song.IsExplicit,
song.Artists
.Select(artist => new YouTubeArtistRef(
string.IsNullOrEmpty(artist.Id)? null: artist.Id,
artist.Name))
.ToList()))
.ToList();
return new YouTubeSearchResult(items, page.Value.NextToken);
}
public async Task<ErrorOr<YouTubeAlbumSearchResult>> SearchAlbumsAsync(string query, string continuationToken, CancellationToken cancellationToken)
{
if (!string.IsNullOrEmpty(continuationToken))
return await FetchAlbumSearchPageAsync(query, continuationToken, cancellationToken);
return await singleFlight.GetOrCreateAsync(
AlbumSearchResultsCacheKey(query),
TimeSpan.FromSeconds(configuration.SearchResultsCacheSeconds),
() => FetchAlbumSearchPageAsync(query, continuationToken, cancellationToken));
}
private async Task<ErrorOr<YouTubeAlbumSearchResult>> FetchAlbumSearchPageAsync(string query, string continuationToken, CancellationToken cancellationToken)
{
var page = await FetchPageAsync(query, SearchCategory.Albums, continuationToken, cancellationToken);
if (page.IsError)
return page.Errors;
var items = page.Value.Results
.OfType<AlbumSearchResult>()
.Select(ToAlbumResult)
.ToList();
return new YouTubeAlbumSearchResult(items, page.Value.NextToken);
}
public async Task<ErrorOr<YouTubeAlbumDetail>> GetAlbumAsync(string albumId, CancellationToken cancellationToken)
{
return await singleFlight.GetOrCreateAsync(
AlbumCacheKey(albumId),
TimeSpan.FromSeconds(configuration.SongInfoCacheSeconds),
() => FetchAlbumAsync(albumId, cancellationToken));
}
private async Task<ErrorOr<YouTubeAlbumDetail>> FetchAlbumAsync(string albumId, CancellationToken cancellationToken)
{
try
{
var browseId = await client.GetAlbumBrowseIdAsync(albumId, cancellationToken);
var info = await client.GetAlbumInfoAsync(browseId, cancellationToken);
var album = new YouTubeAlbumResult(
albumId,
info.Name,
string.Join(", ", info.Artists.Select(artist => artist.Name)),
PickThumbnail(info.Thumbnails),
info.ReleaseYear,
info.IsSingle,
info.IsEp,
info.Artists
.Select(artist => new YouTubeArtistRef(
string.IsNullOrEmpty(artist.Id)? null: artist.Id,
artist.Name))
.ToList());
var tracks = info.Songs
.Select((song, index) => new YouTubeAlbumTrack(
song.Id ?? string.Empty,
song.Name,
(int)song.Duration.TotalSeconds,
song.SongNumber ?? index + 1,
song.IsExplicit))
.Where(track => !string.IsNullOrEmpty(track.VideoId))
.ToList();
return new YouTubeAlbumDetail(
album,
StripWikipediaAttribution(info.Description),
(int)info.Duration.TotalSeconds,
tracks);
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to load YouTube album '{AlbumId}'", albumId);
return Error.Failure(description: $"Failed to load YouTube album '{albumId}': {exception.Message}");
}
}
private static string StripWikipediaAttribution(string? description)
{
if (string.IsNullOrEmpty(description))
return string.Empty;
var markerIndex = description.IndexOf("From Wikipedia", StringComparison.OrdinalIgnoreCase);
var cleaned = markerIndex >= 0 ? description[..markerIndex] : description;
return cleaned.TrimEnd();
}
private YouTubeAlbumResult ToAlbumResult(AlbumSearchResult album) =>
new(
album.Id,
album.Name,
string.Join(", ", album.Artists.Select(artist => artist.Name)),
PickThumbnail(album.Thumbnails),
album.ReleaseYear,
album.IsSingle,
album.IsEp,
album.Artists
.Select(artist => new YouTubeArtistRef(
string.IsNullOrEmpty(artist.Id) ? null : artist.Id,
artist.Name))
.ToList());
public async Task<ErrorOr<YouTubeStreamInfo>> GetAudioStreamAsync(string videoId, CancellationToken cancellationToken)
{
var now = DateTime.UtcNow;
if (cache.TryGetValue(StreamCacheKey(videoId), out CachedStreamInfo? cached)
&& cached != null
&& cached.ExpiresAtUtc > now)
{
return new YouTubeStreamInfo(cached.Url, (int)(cached.ExpiresAtUtc - now).TotalSeconds);
}
StreamingData streamingData;
try
{
streamingData = await client.GetStreamingDataAsync(videoId, cancellationToken);
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to resolve YouTube stream for '{VideoId}'", videoId);
return Error.Failure(description: $"Failed to resolve YouTube stream for '{videoId}': {exception.Message}");
}
var audioStream = streamingData.StreamInfo
.OfType<AudioStreamInfo>()
.OrderByDescending(stream => stream.Bitrate)
.FirstOrDefault();
if (audioStream is null || string.IsNullOrEmpty(audioStream.Url))
return Error.NotFound(description: $"No audio stream available for '{videoId}'.");
var expiresInSeconds = Math.Min(
configuration.StreamUrlCacheSeconds,
(int)streamingData.ExpiresIn.TotalSeconds);
if (expiresInSeconds > 0)
{
cache.Set(
StreamCacheKey(videoId),
new CachedStreamInfo(audioStream.Url, now.AddSeconds(expiresInSeconds)),
TimeSpan.FromSeconds(expiresInSeconds));
}
return new YouTubeStreamInfo(audioStream.Url, expiresInSeconds);
}
public async Task<ErrorOr<YouTubeSongResult>> GetSongAsync(string videoId, CancellationToken cancellationToken)
{
return await singleFlight.GetOrCreateAsync(
SongInfoCacheKey(videoId),
TimeSpan.FromSeconds(configuration.SongInfoCacheSeconds),
() => FetchSongAsync(videoId, cancellationToken));
}
private async Task<ErrorOr<YouTubeSongResult>> FetchSongAsync(string videoId, CancellationToken cancellationToken)
{
try
{
var info = await client.GetSongVideoInfoAsync(videoId, cancellationToken);
return new YouTubeSongResult(
videoId,
info.Name,
string.Join(", ", info.Artists.Select(artist => artist.Name)),
info.Album?.Name ?? string.Empty,
(int)info.Duration.TotalSeconds,
PickThumbnail(info.Thumbnails),
info.IsExplicit,
info.Artists
.Select(artist => new YouTubeArtistRef(
string.IsNullOrEmpty(artist.Id)? null: artist.Id,
artist.Name))
.ToList());
}
catch (Exception exception)
{
logger.LogError(exception, "Failed to load YouTube song info for '{VideoId}'", videoId);
return Error.Failure(description: $"Failed to load YouTube song info for '{videoId}': {exception.Message}");
}
}
public string ResolveArtworkUrl(string thumbnailUrl) =>
YouTubeThumbNailHelper.WithSize(thumbnailUrl, configuration.ArtworkSize);
private string PickThumbnail(IEnumerable<YouTubeMusicAPI.Models.Thumbnail> thumbnails)
{
var ordered = thumbnails.OrderBy(thumbnail => thumbnail.Width).ToList();
var square = ordered.Where(thumbnail => YouTubeThumbNailHelper.IsSquare(thumbnail.Url)).ToList();
var candidates = square.Count > 0 ? square : ordered;
var chosen = candidates.FirstOrDefault(thumbnail => thumbnail.Width >= configuration.ThumbnailSize)
?? candidates.LastOrDefault();
return chosen ==null? string.Empty
: YouTubeThumbNailHelper.WithSize(chosen.Url, configuration.ThumbnailSize);
}
private static string SearchCacheKey(string token) => $"yt-search:{token}";
private static string SearchResultsCacheKey(string query) => $"yt-search-results:{query.Trim().ToLowerInvariant()}";
private static string AlbumSearchResultsCacheKey(string query) => $"yt-album-search:{query.Trim().ToLowerInvariant()}";
private static string AlbumCacheKey(string albumId) => $"yt-album:{albumId}";
private static string SongInfoCacheKey(string videoId) => $"yt-song:{videoId}";
private static string StreamCacheKey(string videoId) => $"yt-stream:{videoId}";
}
}