commit

README.md

@@ -144,7 +144,7 @@ Options:
                                  names.
 
   --format <format>              Accepted file format(s), comma-separated
-  --length-tol <sec>             Length tolerance in seconds (default: 3)
+  --length-tol <sec>             Length tolerance in seconds
   --min-bitrate <rate>           Minimum file bitrate
   --max-bitrate <rate>           Maximum file bitrate
   --min-samplerate <rate>        Minimum file sample rate
@@ -154,7 +154,7 @@ Options:
   --banned-users <list>          Comma-separated list of users to ignore
 
   --pref-format <format>         Preferred file format(s), comma-separated (default: mp3)
-  --pref-length-tol <sec>        Preferred length tolerance in seconds (default: 2)
+  --pref-length-tol <sec>        Preferred length tolerance in seconds (default: 3)
   --pref-min-bitrate <rate>      Preferred minimum bitrate (default: 200)
   --pref-max-bitrate <rate>      Preferred maximum bitrate (default: 2500)
   --pref-min-samplerate <rate>   Preferred minimum sample rate
@@ -244,8 +244,7 @@ Options:
 ```
 
 ### File conditions
-Files not satisfying the conditions will not be downloaded. For example, `--length-tol` is set to 3 by default, meaning that files whose duration differs from the supplied duration by more than 3 seconds will not be downloaded (can be disabled by setting it to -1).  
+Files not satisfying the conditions will not be downloaded. Files satisfying `pref-` conditions will be preferred; setting `--pref-format "flac,wav"` will make it download high quality files if they exist, and only download low quality files if there's nothing else. Conditions can also be supplied as a semicolon-delimited string to `--cond` and `--pref`, e.g `--cond "br>=320;f=mp3,ogg;sr<96000"`. See the start of `Program.cs` for the default file conditions.
-Files satisfying `pref-` conditions will be preferred; setting `--pref-format "flac,wav"` will make it download high quality files if they exist, and only download low quality files if there's nothing else. Conditions can also be supplied as a semicolon-delimited string to `--cond` and `--pref`, e.g `--cond "br>=320;f=mp3,ogg;sr<96000"`. See the start of `Program.cs` for the default file conditions.
 
 **Important note**: Some info may be unavailable depending on the client used by the peer. For example, the default Soulseek client does not share the file bitrate. By default, if `--min-bitrate` is set, then files with unknown bitrate will still be downloaded. You can configure it to reject all files where one of the checked properties is unavailable by enabling `--strict-conditions`. (As a consequence, if `--min-bitrate` is also set then any files shared by users with the default client will be ignored)
 
@@ -262,7 +261,7 @@ The following options will make it go faster, but may decrease search result qua
 - `--searches-per-time` increase at the risk of ban, see the notes section for details.
 
 ### Quality vs Quantity
-The options `--strict-title`, `--strict-artist` and `--strict-album` will filter any file that does not contain the title/artist/album in the filename (ignoring case, bounded by boundary chars). Since by default such files will be ranked lower anyways and may actually be correct, these options are only recommended when you want to minimize false downloads as much as possible.
+The options `--strict-title`, `--strict-artist` and `--strict-album` will filter any file that does not contain the title/artist/album in the filename (ignoring case, bounded by boundary chars). Since by default such files will be ranked lower anyways and may actually be correct, these options are only recommended when you want to minimize false downloads as much as possible. Another way to prevent false downloads is to set `--length-tol` to 3 or less to make it ignore any songs that differ from the input by more than 3 seconds. 
 
 ## Configuration  
 Create a file named `sldl.conf` in the same directory as the executable and write your arguments there, e.g:

slsk-batchdl/Program.cs

@@ -15,16 +15,18 @@ using SlFile = Soulseek.File;
 using File = System.IO.File;
 using Directory = System.IO.Directory;
 using SlDictionary = System.Collections.Concurrent.ConcurrentDictionary<string, (Soulseek.SearchResponse, Soulseek.File)>;
+using System.Runtime.CompilerServices;
 
 
 // todo
 // - Why does it use so much CPU and memory?
 // - Very slow startup time on linux
+// - Uses more threads than allowed after a hundred or so downloads
 
 // undocumented options
 // --on-complete
 // --artist-col, --title-col, --album-col, --length-col, --yt-desc-col, --yt-id-col, --album-track-count-col
-// --input-type, --login, --random-login, --no-modify-share-count --fast-search-delay,
+// --input-type, --login, --random-login, --no-modify-share-count, --unknown-error-retries
 // --fails-to-deprioritize (=1), --fails-to-ignore (=2), --invalid-replace-str
 // --cond, --pref, --danger-words, --pref-danger-words, --strict-title, --strict-artist, --strict-album
 // --fast-search-delay, --fast-search-min-up-speed
@@ -34,13 +36,13 @@ static class Program
 {
     static FileConditions necessaryCond = new()
     {
-        LengthTolerance = 3,
+
     };
 
     static FileConditions preferredCond = new()
     {
         Formats = new string[] { "mp3" },
-        LengthTolerance = 2,
+        LengthTolerance = 3,
         MinBitrate = 200,
         MaxBitrate = 2500,
         MaxSampleRate = 48000,
@@ -133,6 +135,7 @@ static class Program
     static int updateDelay = 100;
     static int searchTimeout = 5000;
     static int maxConcurrentProcesses = 2;
+    static int unknownErrorRetries = 2;
     static int maxRetriesPerTrack = 30;
     static int listenPort = 50000;
 
@@ -214,7 +217,7 @@ static class Program
                             "\n                                 names." +
                             "\n" +
                             "\n  --format <format>              Accepted file format(s), comma-separated" +
-                            "\n  --length-tol <sec>             Length tolerance in seconds (default: 3)" +
+                            "\n  --length-tol <sec>             Length tolerance in seconds" +
                             "\n  --min-bitrate <rate>           Minimum file bitrate" +
                             "\n  --max-bitrate <rate>           Maximum file bitrate" +
                             "\n  --min-samplerate <rate>        Minimum file sample rate" +
@@ -224,7 +227,7 @@ static class Program
                             "\n  --banned-users <list>          Comma-separated list of users to ignore" +
                             "\n" +
                             "\n  --pref-format <format>         Preferred file format(s), comma-separated (default: mp3)" +
-                            "\n  --pref-length-tol <sec>        Preferred length tolerance in seconds (default: 2)" +
+                            "\n  --pref-length-tol <sec>        Preferred length tolerance in seconds (default: 3)" +
                             "\n  --pref-min-bitrate <rate>      Preferred minimum bitrate (default: 200)" +
                             "\n  --pref-max-bitrate <rate>      Preferred maximum bitrate (default: 2500)" +
                             "\n  --pref-min-samplerate <rate>   Preferred minimum sample rate" +
@@ -914,6 +917,9 @@ static class Program
                     case "--fails-to-ignore":
                         ignoreOn = -int.Parse(args[++i]);
                         break;
+                    case "--unknown-error-retries":
+                        unknownErrorRetries = int.Parse(args[++i]);
+                        break;
                     default:
                         throw new ArgumentException($"Unknown argument: {args[i]}");
                 }
@@ -1515,39 +1521,57 @@ static class Program
         {
             if (track.TrackState == Track.State.Exists || track.TrackState == Track.State.NotFoundLastTime)
                 return;
+
             await semaphore.WaitAsync();
-            int tries = 2;
-        retry:
-            await WaitForLogin();
 
-            try
+            int tries = unknownErrorRetries;
+            string savedFilePath = "";
+
+            while (tries > 0)
             {
-                WriteLine($"Search and download {track}", debugOnly: true);
+                await WaitForLogin();
-                var savedFilePath = await SearchAndDownload(track);
-                lock (trackLists) { tracks[index] = new Track(track) { TrackState=Track.State.Downloaded, DownloadPath=savedFilePath }; }
 
-                if (removeTracksFromSource && !string.IsNullOrEmpty(spotifyUrl))
+                try
-                    spotifyClient.RemoveTrackFromPlaylist(playlistUri, track.URI);
+                {
+                    WriteLine($"Search and download {track}", debugOnly: true);
+                    savedFilePath = await SearchAndDownload(track);
+                }
+                catch (Exception ex)
+                {
+                    WriteLine($"Exception thrown: {ex}", debugOnly: true);
+                    if (!client.State.HasFlag(SoulseekClientStates.LoggedIn))
+                    {
+                        continue;
+                    }
+                    else if (ex is SearchAndDownloadException)
+                    {
+                        lock (trackLists) { tracks[index] = new Track(track) { TrackState = Track.State.Failed, FailureReason = ex.Message }; }
+                    }
+                    else
+                    {
+                        WriteLine($"\n{ex.Message}\n{ex.StackTrace}\n", ConsoleColor.DarkYellow, true);
+                        tries--;
+                        continue;
+                    }
+                }
+
+                break;
             }
-            catch (Exception ex)
+
+            if (savedFilePath != "")
             {
-                WriteLine($"Exception thrown: {ex}", debugOnly: true);
+                try
-                if (!client.State.HasFlag(SoulseekClientStates.LoggedIn))
                 {
-                    goto retry;
+                    lock (trackLists) { tracks[index] = new Track(track) { TrackState = Track.State.Downloaded, DownloadPath = savedFilePath }; }
+
+                    if (removeTracksFromSource && !string.IsNullOrEmpty(spotifyUrl))
+                        spotifyClient.RemoveTrackFromPlaylist(playlistUri, track.URI);
                 }
-                else if (ex is SearchAndDownloadException)
+                catch (Exception ex) 
-                {
-                    lock (trackLists) { tracks[index] = new Track(track) { TrackState = Track.State.Failed, FailureReason = ex.Message }; }
-                }
-                else
                 {
                     WriteLine($"\n{ex.Message}\n{ex.StackTrace}\n", ConsoleColor.DarkYellow, true);
-                    if (tries-- > 0)
-                        goto retry;
                 }
             }
-            finally { semaphore.Release(); }
 
             m3uEditor.Update();
 
@@ -1555,6 +1579,8 @@ static class Program
             {
                 OnComplete(onComplete, tracks[index]);
             }
+
+            semaphore.Release();
         });
 
         await Task.WhenAll(downloadTasks);
@@ -1634,15 +1660,62 @@ static class Program
                         return;
 
                     await semaphore.WaitAsync(mainLoopCts.Token);
-                    int tries = 2;
                     
-                retry:
+                    int tries = unknownErrorRetries;
-                    await WaitForLogin();
+                    string savedFilePath = "";
-                    mainLoopCts.Token.ThrowIfCancellationRequested();
 
-                    try
+                    while (tries > 0)
+                    {
+                        await WaitForLogin();
+                        mainLoopCts.Token.ThrowIfCancellationRequested();
+
+                        try
+                        {
+                            savedFilePath = await SearchAndDownload(track);
+                        }
+                        catch (Exception ex)
+                        {
+                            if (!client.State.HasFlag(SoulseekClientStates.LoggedIn))
+                            {
+                                continue;
+                            }
+                            else if (ex is SearchAndDownloadException)
+                            {
+                                lock (trackLists)
+                                {
+                                    tracks[index] = new Track(track) { TrackState = Track.State.Failed, FailureReason = ex.Message };
+                                    if (downloadingImages)
+                                        ReplaceTrack(listRef, track, tracks[index]); // shitty shortcut
+                                }
+
+                                if (!albumIgnoreFails)
+                                {
+                                    mainLoopCts.Cancel();
+                                    foreach (var (key, dl) in downloads)
+                                    {
+                                        lock (dl)
+                                        {
+                                            dl.cts.Cancel();
+                                            if (File.Exists(dl.savePath)) File.Delete(dl.savePath);
+                                            downloads.TryRemove(key, out _);
+                                        }
+                                    }
+                                    throw new OperationCanceledException();
+                                }
+                            }
+                            else
+                            {
+                                WriteLine($"\n{ex.Message}\n{ex.StackTrace}\n", ConsoleColor.DarkYellow, true);
+                                tries--;
+                                continue;
+                            }
+                        }
+
+                        break;
+                    } 
+
+                    if (savedFilePath != "")
                     {
-                        var savedFilePath = await SearchAndDownload(track);
                         dlFiles.TryAdd(savedFilePath, true);
 
                         lock (trackLists)
@@ -1655,48 +1728,13 @@ static class Program
                             }
                         }
                     }
-                    catch (Exception ex)
-                    {
-                        if (!client.State.HasFlag(SoulseekClientStates.LoggedIn))
-                        {
-                            goto retry;
-                        }
-                        else if (ex is SearchAndDownloadException)
-                        {
-                            lock (trackLists)
-                            {
-                                tracks[index] = new Track(track) { TrackState = Track.State.Failed, FailureReason = ex.Message };
-                                if (downloadingImages)
-                                    ReplaceTrack(listRef, track, tracks[index]); // shitty shortcut
-                            }
-                        }
-                        else
-                        {
-                            WriteLine($"\n{ex.Message}\n{ex.StackTrace}\n", ConsoleColor.DarkYellow, true);
-                            if (tries-- > 0)
-                                goto retry;
-                        }
-
-                        if (!albumIgnoreFails)
-                        {
-                            mainLoopCts.Cancel();
-                            lock (downloads)
-                            {
-                                foreach (var (key, dl) in downloads)
-                                {
-                                    dl.cts.Cancel();
-                                    if (File.Exists(dl.savePath)) File.Delete(dl.savePath);
-                                }
-                            }
-                            throw new OperationCanceledException();
-                        }
-                    }
-                    finally { semaphore.Release(); }
 
                     if (onComplete != "")
                     {
                         OnComplete(onComplete, tracks[index]);
                     }
+
+                    semaphore.Release();
                 });
 
                 await Task.WhenAll(downloadTasks);
@@ -2472,10 +2510,13 @@ static class Program
                 .ThenByDescending(x => (x.file.Length != null && x.file.Length > 0) || preferredCond.AcceptNoLength)
                 .ThenByDescending(x => !useBracketCheck || BracketCheck(track, inferredTrack(x).Item1)) // deprioritize result if it contains '(' or '[' and the title does not (avoid remixes)
                 .ThenByDescending(x => preferredCond.StrictTitleSatisfies(x.file.Filename, track.Title))
+                .ThenByDescending(x => preferredCond.StrictArtistSatisfies(x.file.Filename, track.Title))
                 .ThenByDescending(x => preferredCond.LengthToleranceSatisfies(x.file, track.Length))
                 .ThenByDescending(x => preferredCond.FormatSatisfies(x.file.Filename))
                 .ThenByDescending(x => preferredCond.StrictAlbumSatisfies(x.file.Filename, track.Album))
                 .ThenByDescending(x => preferredCond.BitrateSatisfies(x.file))
+                .ThenByDescending(x => preferredCond.SampleRateSatisfies(x.file))
+                .ThenByDescending(x => preferredCond.BitDepthSatisfies(x.file))
                 .ThenByDescending(x => preferredCond.FileSatisfies(x.file, track, x.response))
                 .ThenByDescending(x => x.response.HasFreeUploadSlot)
                 .ThenByDescending(x => x.response.UploadSpeed / 1024 / 650)
@@ -2500,14 +2541,6 @@ static class Program
         if (!t2.Contains('('))
             return true;
 
-        string ar = track.Artist.Replace('[', '(');
-        if (ar.Contains('(') && !t2.Replace(ar, "").Contains('('))
-            return true;
-
-        string al = track.Album.Replace('[', '(');
-        if (al.Contains('(') && !t2.Replace(al, "").Contains('('))
-            return true;
-
         return false;
     }
 
@@ -2842,21 +2875,20 @@ static class Program
             throw new Exception();
 
         await WaitForLogin();
-        System.IO.Directory.CreateDirectory(Path.GetDirectoryName(filePath));
+        Directory.CreateDirectory(Path.GetDirectoryName(filePath));
         string origPath = filePath;
         filePath += ".incomplete";
 
-        bool transferSet = false;
         var transferOptions = new TransferOptions(
             stateChanged: (state) =>
             {
-                if (downloads.ContainsKey(file.Filename) && !transferSet)
+                if (downloads.TryGetValue(file.Filename, out var x))
-                    downloads[file.Filename].transfer = state.Transfer;
+                    x.transfer = state.Transfer;
             },
             progressUpdated: (progress) =>
             {
-                if (downloads.ContainsKey(file.Filename))
+                if (downloads.TryGetValue(file.Filename, out var x))
-                    downloads[file.Filename].bytesTransferred = progress.PreviousBytesTransferred;
+                    x.bytesTransferred = progress.PreviousBytesTransferred;
             }
         );
 
@@ -2869,21 +2901,29 @@ static class Program
         }
         catch
         {
-            if (System.IO.File.Exists(filePath))
+            if (File.Exists(filePath))
-                try { System.IO.File.Delete(filePath); } catch { }
+                try { File.Delete(filePath); } catch { }
-            if (downloads.ContainsKey(file.Filename))
+            downloads.TryRemove(file.Filename, out var d);
-                downloads[file.Filename].UpdateText();
+            if (d != null)
-            downloads.TryRemove(file.Filename, out _);
+                lock (d) { d.UpdateText(); }
             throw;
         }
 
         try { searchCts?.Cancel(); }
         catch { }
-        try { System.IO.File.Move(filePath, origPath, true); }
+
+        try { Utils.Move(filePath, origPath); }
         catch (IOException) { WriteLine($"Failed to rename .incomplete file", ConsoleColor.DarkYellow, true); }
-        downloads[file.Filename].success = true;
+
-        downloads[file.Filename].UpdateText();
+        downloads.TryRemove(file.Filename, out var x);
-        downloads.TryRemove(file.Filename, out _);
+        if (x != null)
+        {
+            lock (x)
+            {
+                x.success = true;
+                x.UpdateText();
+            }
+        }
     }
 
 
@@ -2897,25 +2937,30 @@ static class Program
                 {
                     if (client.State.HasFlag(SoulseekClientStates.LoggedIn))
                     {
-                        foreach (var (key, val) in searches) // shouldn't this give "collection was modified" errors? whatever..
+                        foreach (var (key, val) in searches)
                         {
                             if (val == null)
-                                searches.TryRemove(key, out _);
+                                searches.TryRemove(key, out _); // reminder: removing from a dict in a foreach is allowed in newer .net versions
                         }
 
                         foreach (var (key, val) in downloads)
                         {
                             if (val != null)
                             {
-                                val.UpdateText();
+                                lock (val)
-
-                                if ((DateTime.Now - val.UpdateLastChangeTime()).TotalMilliseconds > downloadMaxStaleTime)
                                 {
-                                    val.stalled = true;
+                                    if ((DateTime.Now - val.UpdateLastChangeTime()).TotalMilliseconds > downloadMaxStaleTime)
-                                    val.UpdateText();
+                                    {
+                                        val.stalled = true;
+                                        val.UpdateText();
 
-                                    try { val.cts.Cancel(); } catch { }
+                                        try { val.cts.Cancel(); } catch { }
-                                    downloads.TryRemove(key, out _);
+                                        downloads.TryRemove(key, out _);
+                                    }
+                                    else
+                                    {
+                                        val.UpdateText();
+                                    }
                                 }
                             }
                             else
@@ -3592,14 +3637,27 @@ static class Program
         if (nameFormat == "" || !Utils.IsMusicFile(filepath))
             return filepath;
 
-        string add = Path.GetRelativePath(outputFolder, Path.GetDirectoryName(filepath));
+        string dir = Path.GetDirectoryName(filepath) ?? "";
+        string add = dir != "" ? Path.GetRelativePath(outputFolder, dir) : "";
         string newFilePath = NamingFormat(filepath, nameFormat, track);
+
         if (filepath != newFilePath)
         {
-            Directory.CreateDirectory(Path.GetDirectoryName(newFilePath));
+            dir = Path.GetDirectoryName(newFilePath) ?? "";
-            Utils.Move(filepath, newFilePath);
+            if (dir != "") Directory.CreateDirectory(dir);
+
+            try
+            {
+                Utils.Move(filepath, newFilePath);
+            }
+            catch (Exception ex)
+            {
+                WriteLine($"\nFailed to move: {ex.Message}\n", ConsoleColor.DarkYellow, true);
+                return filepath;
+            }
+
             if (add != "" && add != "." && Utils.GetRecursiveFileCount(Path.Join(outputFolder, add)) == 0)
-                Directory.Delete(Path.Join(outputFolder, add), true);
+                try { Directory.Delete(Path.Join(outputFolder, add), true); } catch { }
         }
 
         return newFilePath;
@@ -3659,7 +3717,7 @@ static class Program
             char dirsep = Path.DirectorySeparatorChar;
             newName = newName.Replace('/', dirsep);
             var x = newName.Split(dirsep, StringSplitOptions.RemoveEmptyEntries);
-            newName = string.Join(dirsep, x.Select(x => x.ReplaceInvalidChars(invalidReplaceStr)));
+            newName = string.Join(dirsep, x.Select(x => x.ReplaceInvalidChars(invalidReplaceStr).Trim(' ', '.')));
             string newFilePath = Path.Combine(directory, newName + extension);
             return newFilePath;
         }
@@ -4790,7 +4848,10 @@ class RateLimitedSemaphore
             var currentTime = DateTimeOffset.UtcNow;
             if (currentTime.UtcTicks > Interlocked.Read(ref this.nextResetTimeTicks))
             {
-                this.semaphore.Release(this.maxCount - this.semaphore.CurrentCount);
+                int releaseCount = this.maxCount - this.semaphore.CurrentCount;
+                if (releaseCount > 0)
+                    this.semaphore.Release(releaseCount);
+
                 var newResetTimeTicks = (currentTime + this.resetTimeSpan).UtcTicks;
                 Interlocked.Exchange(ref this.nextResetTimeTicks, newResetTimeTicks);
             }