diff --git a/README.md b/README.md index 615faf1..73f1799 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ Usage: sldl [OPTIONS] the directory fails to download. Set to 'delete' to delete the files instead. Set to 'disable' keep it where it is. Default: {configured output dir}/failed + --album-parallel-search Run album searches in parallel ``` #### Aggregate Download Options ``` @@ -357,7 +358,8 @@ The search query is determined as follows: - --concurrent-downloads - set it to 4 or more - --max-stale-time is set to 50 seconds by default, so it will wait a long time before giving up on a file - - --searches-per-time - increase at the risk of bans. + - --searches-per-time - increase at the risk of bans + - --album-parallel-search - enables parallel searching for album entries ## File conditions diff --git a/slsk-batchdl/Config.cs b/slsk-batchdl/Config.cs index 031fa4b..451ca04 100644 --- a/slsk-batchdl/Config.cs +++ b/slsk-batchdl/Config.cs @@ -7,7 +7,10 @@ using System.Text.RegularExpressions; public class Config { - public FileConditions necessaryCond = new(); + public FileConditions necessaryCond = new() + { + Formats = new string[] { ".mp3", ".flac", ".ogg", ".m4a", ".opus", ".wav", ".aac", ".alac" }, + }; public FileConditions preferredCond = new() { @@ -79,6 +82,7 @@ public class Config public bool writePlaylist = false; public bool skipExisting = true; public bool writeIndex = true; + public bool parallelAlbumSearch = false; public int downrankOn = -1; public int ignoreOn = -2; public int minAlbumTrackCount = -1; @@ -96,6 +100,7 @@ public class Config public int searchesPerTime = 34; public int searchRenewTime = 220; public int aggregateLengthTol = 3; + public int parallelAlbumSearchProcesses = 5; public double fastSearchMinUpSpeed = 1.0; public Track regexToReplace = new(); public Track regexReplaceBy = new(); @@ -1257,6 +1262,14 @@ public class Config case "--aggregate-length-tol": aggregateLengthTol = int.Parse(args[++i]); break; + case "--aps": + case "--album-parallel-search": + setFlag(ref parallelAlbumSearch, ref i); + break; + case "--apsc": + case "--album-parallel-search-count": + parallelAlbumSearchProcesses = int.Parse(args[++i]); + break; default: throw new ArgumentException($"Unknown argument: {args[i]}"); } diff --git a/slsk-batchdl/Extractors/Spotify.cs b/slsk-batchdl/Extractors/Spotify.cs index 699f310..c70e997 100644 --- a/slsk-batchdl/Extractors/Spotify.cs +++ b/slsk-batchdl/Extractors/Spotify.cs @@ -92,7 +92,7 @@ namespace Extractors else throw; } - tle.defaultFolderName = playlistName; + tle.defaultFolderName = playlistName.ReplaceInvalidChars(' ').Trim(); tle.enablesIndexByDefault = true; tle.list.Add(tracks); } diff --git a/slsk-batchdl/Extractors/YouTube.cs b/slsk-batchdl/Extractors/YouTube.cs index 3bb7ea8..f10ed6a 100644 --- a/slsk-batchdl/Extractors/YouTube.cs +++ b/slsk-batchdl/Extractors/YouTube.cs @@ -607,7 +607,7 @@ namespace Extractors } } - public static async Task> YtdlpSearch(Track track) + public static async Task> YtdlpSearch(Track track, bool printCommand = false) { Process process = new Process(); ProcessStartInfo startInfo = new ProcessStartInfo(); @@ -623,6 +623,8 @@ namespace Extractors process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); }; process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); }; + Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand); + process.Start(); List<(int, string, string)> results = new List<(int, string, string)>(); @@ -644,7 +646,7 @@ namespace Extractors return results; } - public static async Task YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "") + public static async Task YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "", bool printCommand = false) { Process process = new Process(); ProcessStartInfo startInfo = new ProcessStartInfo(); @@ -666,6 +668,8 @@ namespace Extractors //process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); }; //process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); }; + Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand); + process.Start(); process.WaitForExit(); diff --git a/slsk-batchdl/FileManager.cs b/slsk-batchdl/FileManager.cs index 947a574..c9a7bc7 100644 --- a/slsk-batchdl/FileManager.cs +++ b/slsk-batchdl/FileManager.cs @@ -117,7 +117,14 @@ public class FileManager if (track.DownloadPath.Length == 0) return; - string newFilePath = Path.Join(parent, Path.GetFileName(track.DownloadPath)); + string? part = null; + + if (remoteCommonDir != null && Utils.IsInDirectory(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename), remoteCommonDir, true)) + { + part = Utils.GetFileNameSlsk(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename)); + } + + string newFilePath = Path.Join(parent, part, Path.GetFileName(track.DownloadPath)); try { diff --git a/slsk-batchdl/Help.cs b/slsk-batchdl/Help.cs index c1fc984..f020348 100644 --- a/slsk-batchdl/Help.cs +++ b/slsk-batchdl/Help.cs @@ -4,7 +4,7 @@ // --invalid-replace-str, --cond, --pref // --fast-search-delay, --fast-search-min-up-speed // --min-album-track-count, --max-album-track-count, --extract-max-track-count, --extract-min-track-count -// --skip-mode-music-dir, --skip-mode-output-dir +// --skip-mode-music-dir, --skip-mode-output-dir, --album-parallel-search-count public static class Help { @@ -168,6 +168,7 @@ public static class Help the directory fails to download. Set to 'delete' to delete the files instead. Set to the empty string """" to disable. Default: {configured output dir}/failed + --album-parallel-search Run album searches in parallel Aggregate Download -g, --aggregate Aggregate download mode: Find and download all distinct @@ -333,7 +334,8 @@ public static class Help --concurrent-downloads - set it to 4 or more --max-stale-time is set to 50 seconds by default, so it will wait a long time before giving up on a file - --searches-per-time - increase at the risk of bans. + --searches-per-time - increase at the risk of bans + --album-parallel-search - enables parallel searching for album entries "; const string fileConditionsHelp = @" diff --git a/slsk-batchdl/Models/TrackListEntry.cs b/slsk-batchdl/Models/TrackListEntry.cs index e7b39ea..4e064a4 100644 --- a/slsk-batchdl/Models/TrackListEntry.cs +++ b/slsk-batchdl/Models/TrackListEntry.cs @@ -22,6 +22,8 @@ namespace Models public FileSkipper? outputDirSkipper = null; public FileSkipper? musicDirSkipper = null; + public bool CanParallelSearch => source.Type == TrackType.Album || source.Type == TrackType.Aggregate; + public TrackListEntry(TrackType trackType) { list = new List>(); diff --git a/slsk-batchdl/Program.cs b/slsk-batchdl/Program.cs index a4e4ca8..ace2bab 100644 --- a/slsk-batchdl/Program.cs +++ b/slsk-batchdl/Program.cs @@ -15,6 +15,7 @@ using static Printing; using Directory = System.IO.Directory; using File = System.IO.File; using SlFile = Soulseek.File; +using Konsole; static partial class Program { @@ -54,9 +55,7 @@ static partial class Program trackLists.UpgradeListTypes(config.aggregate, config.album); trackLists.SetListEntryOptions(); - InitConfigs(config); - - await MainLoop(); + await MainLoop(config); WriteLineIf("Mainloop done", config.debugInfo); } @@ -105,63 +104,63 @@ static partial class Program } + static void InitEditors(TrackListEntry tle, Config config) + { + tle.playlistEditor = new M3uEditor(trackLists, config.writePlaylist ? M3uOption.Playlist : M3uOption.None, config.offset); + tle.indexEditor = new M3uEditor(trackLists, config.writeIndex ? M3uOption.Index : M3uOption.None); + } + + static void InitFileSkippers(TrackListEntry tle, Config config) + { + if (config.skipExisting) + { + FileConditions? cond = null; + + if (config.skipCheckPrefCond) + { + cond = config.necessaryCond.With(config.preferredCond); + } + else if (config.skipCheckCond) + { + cond = config.necessaryCond; + } + + tle.outputDirSkipper = FileSkipperRegistry.GetSkipper(config.skipMode, config.parentDir, cond, tle.indexEditor); + + if (config.skipMusicDir.Length > 0) + { + if (!Directory.Exists(config.skipMusicDir)) + Console.WriteLine("Error: Music directory does not exist"); + else + tle.musicDirSkipper = FileSkipperRegistry.GetSkipper(config.skipModeMusicDir, config.skipMusicDir, cond, tle.indexEditor); + } + } + } + static void InitConfigs(Config defaultConfig) { - if (trackLists.Count == 0) - return; + //if (trackLists.Count == 0) + // return; - void initEditors(TrackListEntry tle, Config config) - { - tle.playlistEditor = new M3uEditor(trackLists, config.writePlaylist ? M3uOption.Playlist : M3uOption.None, config.offset); - tle.indexEditor = new M3uEditor(trackLists, config.writeIndex ? M3uOption.Index : M3uOption.None); - } + //foreach (var tle in trackLists.lists) + //{ + // tle.config = defaultConfig.Copy(); + // tle.config.UpdateProfiles(tle); - void initFileSkippers(TrackListEntry tle, Config config) - { - if (config.skipExisting) - { - FileConditions? cond = null; + // if (tle.extractorCond != null) + // { + // tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond); + // tle.extractorCond = null; + // } + // if (tle.extractorPrefCond != null) + // { + // tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond); + // tle.extractorPrefCond = null; + // } - if (config.skipCheckPrefCond) - { - cond = config.necessaryCond.With(config.preferredCond); - } - else if (config.skipCheckCond) - { - cond = config.necessaryCond; - } - - tle.outputDirSkipper = FileSkipperRegistry.GetSkipper(config.skipMode, config.parentDir, cond, tle.indexEditor); - - if (config.skipMusicDir.Length > 0) - { - if (!Directory.Exists(config.skipMusicDir)) - Console.WriteLine("Error: Music directory does not exist"); - else - tle.musicDirSkipper = FileSkipperRegistry.GetSkipper(config.skipModeMusicDir, config.skipMusicDir, cond, tle.indexEditor); - } - } - } - - foreach (var tle in trackLists.lists) - { - tle.config = defaultConfig.Copy(); - tle.config.UpdateProfiles(tle); - - if (tle.extractorCond != null) - { - tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond); - tle.extractorCond = null; - } - if (tle.extractorPrefCond != null) - { - tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond); - tle.extractorPrefCond = null; - } - - initEditors(tle, tle.config); - initFileSkippers(tle, tle.config); - } + // initEditors(tle, tle.config); + // initFileSkippers(tle, tle.config); + //} //defaultConfig.UpdateProfiles(trackLists[0]); //trackLists[0].config = defaultConfig; @@ -266,40 +265,66 @@ static partial class Program } - static void PrepareListEntry(Config config, TrackListEntry tle, bool isFirstEntry) + static void PrepareListEntry(Config prevConfig, TrackListEntry tle) { + tle.config = prevConfig.Copy(); + tle.config.UpdateProfiles(tle); + + if (tle.extractorCond != null) + { + tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond); + tle.extractorCond = null; + } + if (tle.extractorPrefCond != null) + { + tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond); + tle.extractorPrefCond = null; + } + + InitEditors(tle, tle.config); + InitFileSkippers(tle, tle.config); + string m3uPath, indexPath; - if (config.m3uFilePath.Length > 0) - m3uPath = config.m3uFilePath; + if (tle.config.m3uFilePath.Length > 0) + m3uPath = tle.config.m3uFilePath; else - m3uPath = Path.Join(config.parentDir, tle.defaultFolderName, "_playlist.m3u8"); + m3uPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_playlist.m3u8"); - if (config.indexFilePath.Length > 0) - indexPath = config.indexFilePath; + if (tle.config.indexFilePath.Length > 0) + indexPath = tle.config.indexFilePath; else - indexPath = Path.Join(config.parentDir, tle.defaultFolderName, "_index.sldl"); + indexPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_index.sldl"); - if (config.writePlaylist) + if (tle.config.writePlaylist) tle.playlistEditor?.SetPathAndLoad(m3uPath); - if (config.writeIndex) + if (tle.config.writeIndex) tle.indexEditor?.SetPathAndLoad(indexPath); - PreprocessTracks(config, tle); + PreprocessTracks(tle.config, tle); } - static async Task MainLoop() + static async Task MainLoop(Config defaultConfig) { + if (trackLists.Count == 0) return; + + PrepareListEntry(defaultConfig, trackLists[0]); + var firstConfig = trackLists.lists[0].config; + + bool enableParallelSearch = firstConfig.parallelAlbumSearch && !firstConfig.PrintResults && !firstConfig.PrintTracks && trackLists.lists.Any(x => x.CanParallelSearch); + var parallelSearches = new List<(TrackListEntry tle, Task<(bool, ResponseData)> task)>(); + var parallelSearchSemaphore = new SemaphoreSlim(firstConfig.parallelAlbumSearchProcesses); + for (int i = 0; i < trackLists.lists.Count; i++) { - Console.WriteLine(); + if (!enableParallelSearch) Console.WriteLine(); + + if (i > 0) PrepareListEntry(trackLists[i-1].config, trackLists[i]); var tle = trackLists[i]; var config = tle.config; - PrepareListEntry(config, tle, isFirstEntry: i == 0); - var existing = new List(); var notFound = new List(); @@ -361,59 +386,87 @@ static partial class Program { await InitClientAndUpdateIfNeeded(config); - Console.WriteLine($"{tle.source.Type} download: {tle.source.ToString(true)}, searching.."); + ProgressBar? progress = null; - bool foundSomething = false; - var responseData = new ResponseData(); + async Task<(bool, ResponseData)> sourceSearch() + { + await parallelSearchSemaphore.WaitAsync(); - if (tle.source.Type == TrackType.Album) - { - tle.list = await Search.GetAlbumDownloads(tle.source, responseData, config); - foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0; - } - else if (tle.source.Type == TrackType.Aggregate) - { - tle.list.Insert(0, await Search.GetAggregateTracks(tle.source, responseData, config)); - foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0; - } - else if (tle.source.Type == TrackType.AlbumAggregate) - { - var res = await Search.GetAggregateAlbums(tle.source, responseData, config); + progress = enableParallelSearch ? Printing.GetProgressBar(config) : null; + Printing.RefreshOrPrint(progress, 0, $"{tle.source.Type} download: {tle.source.ToString(true)}, searching..", print: true); - foreach (var item in res) + bool foundSomething = false; + var responseData = new ResponseData(); + + if (tle.source.Type == TrackType.Album) { - var newSource = new Track(tle.source) { Type = TrackType.Album }; - var albumTle = new TrackListEntry(item, newSource, needSourceSearch: false, sourceCanBeSkipped: true); - albumTle.defaultFolderName = tle.defaultFolderName; - trackLists.AddEntry(albumTle); + tle.list = await Search.GetAlbumDownloads(tle.source, responseData, config); + foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0; + } + else if (tle.source.Type == TrackType.Aggregate) + { + tle.list.Insert(0, await Search.GetAggregateTracks(tle.source, responseData, config)); + foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0; + } + else if (tle.source.Type == TrackType.AlbumAggregate) + { + var res = await Search.GetAggregateAlbums(tle.source, responseData, config); + + foreach (var item in res) + { + var newSource = new Track(tle.source) { Type = TrackType.Album }; + var albumTle = new TrackListEntry(item, newSource, needSourceSearch: false, sourceCanBeSkipped: true); + albumTle.defaultFolderName = tle.defaultFolderName; + trackLists.AddEntry(albumTle); + } + + foundSomething = res.Count > 0; } - foundSomething = res.Count > 0; - } + tle.needSourceSearch = false; - if (!foundSomething) - { - var lockedFiles = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : ""; - Console.WriteLine($"No results.{lockedFiles}"); - - if (!config.PrintResults) + if (!foundSomething) { - tle.source.State = TrackState.Failed; - tle.source.FailureReason = FailureReason.NoSuitableFileFound; - tle.indexEditor?.Update(); + var lockedFiles = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : ""; + var str = progress != null ? $"{tle.source}: " : ""; + Printing.RefreshOrPrint(progress, 0, $"{str}No results.{lockedFiles}", true); } - - continue; + + parallelSearchSemaphore.Release(); + + return (foundSomething, responseData); } - if (config.skipExisting && tle.needSkipExistingAfterSearch) + if (!enableParallelSearch || !tle.CanParallelSearch) { - foreach (var tracks in tle.list) - existing.AddRange(DoSkipExisting(tle, config, tracks)); - } + (bool foundSomething, ResponseData responseData) = await sourceSearch(); - if (tle.gotoNextAfterSearch) + if (!foundSomething) + { + if (!config.PrintResults) + { + tle.source.State = TrackState.Failed; + tle.source.FailureReason = FailureReason.NoSuitableFileFound; + tle.indexEditor?.Update(); + } + + continue; + } + + if (config.skipExisting && tle.needSkipExistingAfterSearch) + { + foreach (var tracks in tle.list) + existing.AddRange(DoSkipExisting(tle, config, tracks)); + } + + if (tle.gotoNextAfterSearch) + { + continue; + } + } + else { + parallelSearches.Add((tle, sourceSearch())); continue; } } @@ -424,6 +477,29 @@ static partial class Program continue; } + if (parallelSearches.Count > 0 && !tle.CanParallelSearch) + { + await parallelDownloads(); + } + + if (!enableParallelSearch || !tle.CanParallelSearch) + { + await download(tle, config, notFound, existing); + } + } + + if (parallelSearches.Count > 0) + { + await parallelDownloads(); + } + + if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any())) + { + PrintComplete(trackLists); + } + + async Task download(TrackListEntry tle, Config config, List? notFound, List? existing) + { tle.indexEditor?.Update(); tle.playlistEditor?.Update(); @@ -432,9 +508,9 @@ static partial class Program PrintTracksTbd(tle.list[0].Where(t => t.State == TrackState.Initial).ToList(), existing, notFound, tle.source.Type, config); } - if (notFound.Count + existing.Count >= tle.list.Sum(x => x.Count)) + if (notFound != null && existing != null && notFound.Count + existing.Count >= tle.list.Sum(x => x.Count)) { - continue; + return; } await InitClientAndUpdateIfNeeded(config); @@ -453,9 +529,38 @@ static partial class Program } } - if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any())) + async Task parallelDownloads() { - PrintComplete(trackLists); + await Task.WhenAll(parallelSearches.Select(x => x.task)); + + Console.WriteLine(); + + foreach (var (tle, task) in parallelSearches) + { + (bool foundSomething, var responseData) = task.Result; + + if (foundSomething) + { + Console.WriteLine($"Downloading: {tle.source}"); + await download(tle, tle.config, null, null); + } + else + { + if (!tle.config.PrintResults) + { + tle.source.State = TrackState.Failed; + tle.source.FailureReason = FailureReason.NoSuitableFileFound; + tle.indexEditor?.Update(); + } + if (tle.config.skipExisting && tle.needSkipExistingAfterSearch) + { + foreach (var tracks in tle.list) + DoSkipExisting(tle, tle.config, tracks); + } + } + } + + parallelSearches.Clear(); } } @@ -749,6 +854,11 @@ static partial class Program Console.WriteLine("No images found"); return downloadedImages; } + else if (!albumArts.Skip(1).Any() && albumArts.First().All(y => y.State != TrackState.Initial)) + { + Console.WriteLine("No additional images found"); + return downloadedImages; + } if (option == AlbumArtOption.Largest) { diff --git a/slsk-batchdl/Search.cs b/slsk-batchdl/Search.cs index 4c69632..f5cce52 100644 --- a/slsk-batchdl/Search.cs +++ b/slsk-batchdl/Search.cs @@ -102,7 +102,7 @@ static class Search }, fileFilter: (file) => { - return Utils.IsMusicFile(file.Filename) && necCond.FileSatisfies(file, track, null); + return necCond.FileSatisfies(file, track, null); }); } @@ -214,7 +214,7 @@ static class Search try { Printing.RefreshOrPrint(progress, 0, $"yt-dlp search: {track}", true); - var ytResults = await Extractors.YouTube.YtdlpSearch(track); + var ytResults = await Extractors.YouTube.YtdlpSearch(track, printCommand: config.debugInfo); if (ytResults.Count > 0) { @@ -224,8 +224,8 @@ static class Search { string saveFilePathNoExt = organizer.GetSavePathNoExt(title); downloading = 1; - Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}", true); - saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument); + Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}, filename: {saveFilePathNoExt}", true); + saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument, printCommand: config.debugInfo); Printing.RefreshOrPrint(progress, 100, $"Succeded: yt-dlp completed download for {track}", true); break; } @@ -418,7 +418,7 @@ static class Search }, fileFilter: (file) => { - return Utils.IsMusicFile(file.Filename) && nec.FileSatisfies(file, track, null); + return nec.FileSatisfies(file, track, null); } ); void handler(SlResponse r) @@ -678,6 +678,7 @@ static class Search bool albumMode = false) { bool useBracketCheck = true; + if (albumMode) { useBracketCheck = false; @@ -881,7 +882,7 @@ static class Search }, fileFilter: (file) => { - return Utils.IsMusicFile(file.Filename) && (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull); + return (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull); }); } diff --git a/slsk-batchdl/Utilities/Utils.cs b/slsk-batchdl/Utilities/Utils.cs index c8ff60b..5c423e0 100644 --- a/slsk-batchdl/Utilities/Utils.cs +++ b/slsk-batchdl/Utilities/Utils.cs @@ -634,6 +634,13 @@ public static class Utils return path.Replace('\\', '/').TrimEnd('/').Trim(); } + public static bool IsInDirectory(string path, string dir, bool strict) + { + path = NormalizedPath(path); + dir = NormalizedPath(dir); + return strict ? path.StartsWith(dir + '/') : path.StartsWith(dir); + } + public static bool SequenceEqualUpToPermutation(this IEnumerable list1, IEnumerable list2) { var cnt = new Dictionary();