parallel album searching

README.md

@@ -198,6 +198,7 @@ Usage: sldl <input> [OPTIONS]
                                    the directory fails to download. Set to 'delete' to delete
                                    the files instead. Set to 'disable' keep it where it is.
                                    Default: {configured output dir}/failed
+    --album-parallel-search        Run album searches in parallel
 ```
 #### Aggregate Download Options
 ```
@@ -357,7 +358,8 @@ The search query is determined as follows:
   - --concurrent-downloads - set it to 4 or more
   - --max-stale-time is set to 50 seconds by default, so it will wait a long time before giving
     up on a file
-  - --searches-per-time - increase at the risk of bans.
+  - --searches-per-time - increase at the risk of bans
+  - --album-parallel-search - enables parallel searching for album entries
 
 ## File conditions
 

slsk-batchdl/Config.cs

@@ -7,7 +7,10 @@ using System.Text.RegularExpressions;
 
 public class Config
 {
-    public FileConditions necessaryCond = new();
+    public FileConditions necessaryCond = new() 
+    {
+        Formats = new string[] { ".mp3", ".flac", ".ogg", ".m4a", ".opus", ".wav", ".aac", ".alac" },
+    };
 
     public FileConditions preferredCond = new()
     {
@@ -79,6 +82,7 @@ public class Config
     public bool writePlaylist = false;
     public bool skipExisting = true;
     public bool writeIndex = true;
+    public bool parallelAlbumSearch = false;
     public int downrankOn = -1;
     public int ignoreOn = -2;
     public int minAlbumTrackCount = -1;
@@ -96,6 +100,7 @@ public class Config
     public int searchesPerTime = 34;
     public int searchRenewTime = 220;
     public int aggregateLengthTol = 3;
+    public int parallelAlbumSearchProcesses = 5;
     public double fastSearchMinUpSpeed = 1.0;
     public Track regexToReplace = new();
     public Track regexReplaceBy = new();
@@ -1257,6 +1262,14 @@ public class Config
                     case "--aggregate-length-tol":
                         aggregateLengthTol = int.Parse(args[++i]);
                         break;
+                    case "--aps":
+                    case "--album-parallel-search":
+                        setFlag(ref parallelAlbumSearch, ref i);
+                        break;
+                    case "--apsc":
+                    case "--album-parallel-search-count":
+                        parallelAlbumSearchProcesses = int.Parse(args[++i]);
+                        break;
                     default:
                         throw new ArgumentException($"Unknown argument: {args[i]}");
                 }

slsk-batchdl/Extractors/Spotify.cs

@@ -92,7 +92,7 @@ namespace Extractors
                     else throw;
                 }
 
-                tle.defaultFolderName = playlistName;
+                tle.defaultFolderName = playlistName.ReplaceInvalidChars(' ').Trim();
                 tle.enablesIndexByDefault = true;
                 tle.list.Add(tracks);
             }

slsk-batchdl/Extractors/YouTube.cs

@@ -607,7 +607,7 @@ namespace Extractors
             }
         }
 
-        public static async Task<List<(int length, string id, string title)>> YtdlpSearch(Track track)
+        public static async Task<List<(int length, string id, string title)>> YtdlpSearch(Track track, bool printCommand = false)
         {
             Process process = new Process();
             ProcessStartInfo startInfo = new ProcessStartInfo();
@@ -623,6 +623,8 @@ namespace Extractors
             process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
             process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
 
+            Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand);
+
             process.Start();
 
             List<(int, string, string)> results = new List<(int, string, string)>();
@@ -644,7 +646,7 @@ namespace Extractors
             return results;
         }
 
-        public static async Task<string> YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "")
+        public static async Task<string> YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "", bool printCommand = false)
         {
             Process process = new Process();
             ProcessStartInfo startInfo = new ProcessStartInfo();
@@ -666,6 +668,8 @@ namespace Extractors
             //process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
             //process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
 
+            Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand);
+
             process.Start();
             process.WaitForExit();
 

slsk-batchdl/FileManager.cs

@@ -117,7 +117,14 @@ public class FileManager
         if (track.DownloadPath.Length == 0)
             return;
 
-        string newFilePath = Path.Join(parent, Path.GetFileName(track.DownloadPath));
+        string? part = null;
+
+        if (remoteCommonDir != null && Utils.IsInDirectory(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename), remoteCommonDir, true))
+        {
+            part = Utils.GetFileNameSlsk(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename));
+        }
+
+        string newFilePath = Path.Join(parent, part, Path.GetFileName(track.DownloadPath));
 
         try
         {

slsk-batchdl/Help.cs

@@ -4,7 +4,7 @@
 // --invalid-replace-str, --cond, --pref
 // --fast-search-delay, --fast-search-min-up-speed
 // --min-album-track-count, --max-album-track-count, --extract-max-track-count, --extract-min-track-count 
-// --skip-mode-music-dir, --skip-mode-output-dir
+// --skip-mode-music-dir, --skip-mode-output-dir, --album-parallel-search-count
 
 public static class Help
 {
@@ -168,6 +168,7 @@ public static class Help
                                        the directory fails to download. Set to 'delete' to delete
                                        the files instead. Set to the empty string """" to disable.
                                        Default: {configured output dir}/failed
+        --album-parallel-search        Run album searches in parallel
           
       Aggregate Download
         -g, --aggregate                Aggregate download mode: Find and download all distinct 
@@ -333,7 +334,8 @@ public static class Help
         --concurrent-downloads - set it to 4 or more
         --max-stale-time is set to 50 seconds by default, so it will wait a long time before giving
           up on a file
-        --searches-per-time - increase at the risk of bans.
+        --searches-per-time - increase at the risk of bans
+        --album-parallel-search - enables parallel searching for album entries
     ";
 
     const string fileConditionsHelp = @"

slsk-batchdl/Models/TrackListEntry.cs

@@ -22,6 +22,8 @@ namespace Models
         public FileSkipper? outputDirSkipper = null;
         public FileSkipper? musicDirSkipper = null;
 
+        public bool CanParallelSearch => source.Type == TrackType.Album || source.Type == TrackType.Aggregate;
+
         public TrackListEntry(TrackType trackType)
         {
             list = new List<List<Track>>();

slsk-batchdl/Program.cs

@@ -15,6 +15,7 @@ using static Printing;
 using Directory = System.IO.Directory;
 using File = System.IO.File;
 using SlFile = Soulseek.File;
+using Konsole;
 
 static partial class Program
 {
@@ -54,9 +55,7 @@ static partial class Program
         trackLists.UpgradeListTypes(config.aggregate, config.album);
         trackLists.SetListEntryOptions();
 
-        InitConfigs(config);
+        await MainLoop(config);
-
-        await MainLoop();
 
         WriteLineIf("Mainloop done", config.debugInfo);
     }
@@ -105,18 +104,13 @@ static partial class Program
     }
 
 
-    static void InitConfigs(Config defaultConfig)
+    static void InitEditors(TrackListEntry tle, Config config)
-    {
-        if (trackLists.Count == 0)
-            return;
-
-        void initEditors(TrackListEntry tle, Config config)
     {
         tle.playlistEditor = new M3uEditor(trackLists, config.writePlaylist ? M3uOption.Playlist : M3uOption.None, config.offset);
         tle.indexEditor = new M3uEditor(trackLists, config.writeIndex ? M3uOption.Index : M3uOption.None);
     }
 
-        void initFileSkippers(TrackListEntry tle, Config config)
+    static void InitFileSkippers(TrackListEntry tle, Config config)
     {
         if (config.skipExisting)
         {
@@ -143,25 +137,30 @@ static partial class Program
         }
     }
 
-        foreach (var tle in trackLists.lists)
+    static void InitConfigs(Config defaultConfig)
     {
-            tle.config = defaultConfig.Copy();
+        //if (trackLists.Count == 0)
-            tle.config.UpdateProfiles(tle);
+        //    return;
 
-            if (tle.extractorCond != null)
+        //foreach (var tle in trackLists.lists)
-            {
+        //{
-                tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
+        //    tle.config = defaultConfig.Copy();
-                tle.extractorCond = null;
+        //    tle.config.UpdateProfiles(tle);
-            }
-            if (tle.extractorPrefCond != null)
-            {
-                tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
-                tle.extractorPrefCond = null;
-            }
 
-            initEditors(tle, tle.config);
+        //    if (tle.extractorCond != null)
-            initFileSkippers(tle, tle.config);
+        //    {
-        }
+        //        tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
+        //        tle.extractorCond = null;
+        //    }
+        //    if (tle.extractorPrefCond != null)
+        //    {
+        //        tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
+        //        tle.extractorPrefCond = null;
+        //    }
+
+        //    initEditors(tle, tle.config);
+        //    initFileSkippers(tle, tle.config);
+        //}
 
         //defaultConfig.UpdateProfiles(trackLists[0]);
         //trackLists[0].config = defaultConfig;
@@ -266,40 +265,66 @@ static partial class Program
     }
 
 
-    static void PrepareListEntry(Config config, TrackListEntry tle, bool isFirstEntry)
+    static void PrepareListEntry(Config prevConfig, TrackListEntry tle)
     {
+        tle.config = prevConfig.Copy();
+        tle.config.UpdateProfiles(tle);
+
+        if (tle.extractorCond != null)
+        {
+            tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
+            tle.extractorCond = null;
+        }
+        if (tle.extractorPrefCond != null)
+        {
+            tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
+            tle.extractorPrefCond = null;
+        }
+
+        InitEditors(tle, tle.config);
+        InitFileSkippers(tle, tle.config);
+
         string m3uPath, indexPath;
 
-        if (config.m3uFilePath.Length > 0)
+        if (tle.config.m3uFilePath.Length > 0)
-            m3uPath = config.m3uFilePath;
+            m3uPath = tle.config.m3uFilePath;
         else
-            m3uPath = Path.Join(config.parentDir, tle.defaultFolderName, "_playlist.m3u8");
+            m3uPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_playlist.m3u8");
 
-        if (config.indexFilePath.Length > 0)
+        if (tle.config.indexFilePath.Length > 0)
-            indexPath = config.indexFilePath;
+            indexPath = tle.config.indexFilePath;
         else
-            indexPath = Path.Join(config.parentDir, tle.defaultFolderName, "_index.sldl");
+            indexPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_index.sldl");
 
-        if (config.writePlaylist)
+        if (tle.config.writePlaylist)
             tle.playlistEditor?.SetPathAndLoad(m3uPath);
-        if (config.writeIndex)
+        if (tle.config.writeIndex)
             tle.indexEditor?.SetPathAndLoad(indexPath);
 
-        PreprocessTracks(config, tle);
+        PreprocessTracks(tle.config, tle);
     }
 
 
-    static async Task MainLoop()
+    static async Task MainLoop(Config defaultConfig)
     {
+        if (trackLists.Count == 0) return;
+
+        PrepareListEntry(defaultConfig, trackLists[0]);
+        var firstConfig = trackLists.lists[0].config;
+
+        bool enableParallelSearch = firstConfig.parallelAlbumSearch && !firstConfig.PrintResults && !firstConfig.PrintTracks && trackLists.lists.Any(x => x.CanParallelSearch);
+        var parallelSearches = new List<(TrackListEntry tle, Task<(bool, ResponseData)> task)>();
+        var parallelSearchSemaphore = new SemaphoreSlim(firstConfig.parallelAlbumSearchProcesses);
+
         for (int i = 0; i < trackLists.lists.Count; i++)
         {
-            Console.WriteLine();
+            if (!enableParallelSearch) Console.WriteLine();
+
+            if (i > 0) PrepareListEntry(trackLists[i-1].config, trackLists[i]);
 
             var tle = trackLists[i];
             var config = tle.config;
 
-            PrepareListEntry(config, tle, isFirstEntry: i == 0);
-
             var existing = new List<Track>();
             var notFound = new List<Track>();
 
@@ -361,7 +386,14 @@ static partial class Program
             {
                 await InitClientAndUpdateIfNeeded(config);
 
-                Console.WriteLine($"{tle.source.Type} download: {tle.source.ToString(true)}, searching..");
+                ProgressBar? progress = null;
+
+                async Task<(bool, ResponseData)> sourceSearch()
+                {
+                    await parallelSearchSemaphore.WaitAsync();
+
+                    progress = enableParallelSearch ? Printing.GetProgressBar(config) : null;
+                    Printing.RefreshOrPrint(progress, 0, $"{tle.source.Type} download: {tle.source.ToString(true)}, searching..", print: true);
 
                     bool foundSomething = false;
                     var responseData = new ResponseData();
@@ -391,11 +423,26 @@ static partial class Program
                         foundSomething = res.Count > 0;
                     }
 
+                    tle.needSourceSearch = false;
+
                     if (!foundSomething)
                     {
                         var lockedFiles = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : "";
-                    Console.WriteLine($"No results.{lockedFiles}");
+                        var str = progress != null ? $"{tle.source}: " : "";
+                        Printing.RefreshOrPrint(progress, 0, $"{str}No results.{lockedFiles}", true);
+                    }
 
+                    parallelSearchSemaphore.Release();
+
+                    return (foundSomething, responseData);
+                }
+
+                if (!enableParallelSearch || !tle.CanParallelSearch)
+                {
+                    (bool foundSomething, ResponseData responseData) = await sourceSearch();
+
+                    if (!foundSomething)
+                    {
                         if (!config.PrintResults)
                         {
                             tle.source.State = TrackState.Failed;
@@ -417,6 +464,12 @@ static partial class Program
                         continue;
                     }
                 }
+                else
+                {
+                    parallelSearches.Add((tle, sourceSearch()));
+                    continue;
+                }
+            }
 
             if (config.PrintResults)
             {
@@ -424,6 +477,29 @@ static partial class Program
                 continue;
             }
 
+            if (parallelSearches.Count > 0 && !tle.CanParallelSearch)
+            {
+                await parallelDownloads();
+            }
+
+            if (!enableParallelSearch || !tle.CanParallelSearch)
+            {
+                await download(tle, config, notFound, existing);
+            }
+        }
+
+        if (parallelSearches.Count > 0)
+        {
+            await parallelDownloads();
+        }
+
+        if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any()))
+        {
+            PrintComplete(trackLists);
+        }
+
+        async Task download(TrackListEntry tle, Config config, List<Track>? notFound, List<Track>? existing)
+        {
             tle.indexEditor?.Update();
             tle.playlistEditor?.Update();
 
@@ -432,9 +508,9 @@ static partial class Program
                 PrintTracksTbd(tle.list[0].Where(t => t.State == TrackState.Initial).ToList(), existing, notFound, tle.source.Type, config);
             }
 
-            if (notFound.Count + existing.Count >= tle.list.Sum(x => x.Count))
+            if (notFound != null && existing != null && notFound.Count + existing.Count >= tle.list.Sum(x => x.Count))
             {
-                continue;
+                return;
             }
 
             await InitClientAndUpdateIfNeeded(config);
@@ -453,9 +529,38 @@ static partial class Program
             }
         }
 
-        if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any()))
+        async Task parallelDownloads()
         {
-            PrintComplete(trackLists);
+            await Task.WhenAll(parallelSearches.Select(x => x.task));
+
+            Console.WriteLine();
+
+            foreach (var (tle, task) in parallelSearches)
+            {
+                (bool foundSomething, var responseData) = task.Result;
+
+                if (foundSomething)
+                {
+                    Console.WriteLine($"Downloading: {tle.source}");
+                    await download(tle, tle.config, null, null);
+                }
+                else
+                {
+                    if (!tle.config.PrintResults)
+                    {
+                        tle.source.State = TrackState.Failed;
+                        tle.source.FailureReason = FailureReason.NoSuitableFileFound;
+                        tle.indexEditor?.Update();
+                    }
+                    if (tle.config.skipExisting && tle.needSkipExistingAfterSearch)
+                    {
+                        foreach (var tracks in tle.list)
+                            DoSkipExisting(tle, tle.config, tracks);
+                    }
+                }
+            }
+
+            parallelSearches.Clear();
         }
     }
 
@@ -749,6 +854,11 @@ static partial class Program
             Console.WriteLine("No images found");
             return downloadedImages;
         }
+        else if (!albumArts.Skip(1).Any() && albumArts.First().All(y => y.State != TrackState.Initial))
+        {
+            Console.WriteLine("No additional images found");
+            return downloadedImages;
+        }
 
         if (option == AlbumArtOption.Largest)
         {

slsk-batchdl/Search.cs

@@ -102,7 +102,7 @@ static class Search
                 },
                 fileFilter: (file) =>
                 {
-                    return Utils.IsMusicFile(file.Filename) && necCond.FileSatisfies(file, track, null);
+                    return necCond.FileSatisfies(file, track, null);
                 });
         }
 
@@ -214,7 +214,7 @@ static class Search
             try
             {
                 Printing.RefreshOrPrint(progress, 0, $"yt-dlp search: {track}", true);
-                var ytResults = await Extractors.YouTube.YtdlpSearch(track);
+                var ytResults = await Extractors.YouTube.YtdlpSearch(track, printCommand: config.debugInfo);
 
                 if (ytResults.Count > 0)
                 {
@@ -224,8 +224,8 @@ static class Search
                         {
                             string saveFilePathNoExt = organizer.GetSavePathNoExt(title);
                             downloading = 1;
-                            Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}", true);
+                            Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}, filename: {saveFilePathNoExt}", true);
-                            saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument);
+                            saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument, printCommand: config.debugInfo);
                             Printing.RefreshOrPrint(progress, 100, $"Succeded: yt-dlp completed download for {track}", true);
                             break;
                         }
@@ -418,7 +418,7 @@ static class Search
                 },
                 fileFilter: (file) =>
                 {
-                    return Utils.IsMusicFile(file.Filename) && nec.FileSatisfies(file, track, null);
+                    return nec.FileSatisfies(file, track, null);
                 }
             );
         void handler(SlResponse r)
@@ -678,6 +678,7 @@ static class Search
         bool albumMode = false)
     {
         bool useBracketCheck = true;
+
         if (albumMode)
         {
             useBracketCheck = false;
@@ -881,7 +882,7 @@ static class Search
                     },
                     fileFilter: (file) =>
                     {
-                        return Utils.IsMusicFile(file.Filename) && (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull);
+                        return (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull);
                     });
             }
 

slsk-batchdl/Utilities/Utils.cs

@@ -634,6 +634,13 @@ public static class Utils
         return path.Replace('\\', '/').TrimEnd('/').Trim();
     }
 
+    public static bool IsInDirectory(string path, string dir, bool strict)
+    {
+        path = NormalizedPath(path);
+        dir = NormalizedPath(dir);
+        return strict ? path.StartsWith(dir + '/') : path.StartsWith(dir);
+    }
+
     public static bool SequenceEqualUpToPermutation<T>(this IEnumerable<T> list1, IEnumerable<T> list2)
     {
         var cnt = new Dictionary<T, int>();