From 4e3ed2ec466db652843574272136efb15f0fb065 Mon Sep 17 00:00:00 2001 From: fiso64 Date: Wed, 28 Aug 2024 13:49:39 +0200 Subject: [PATCH] improve result ordering code --- README.md | 12 ++- slsk-batchdl/Config.cs | 11 --- slsk-batchdl/Data.cs | 25 +++--- slsk-batchdl/FileConditions.cs | 92 +++++----------------- slsk-batchdl/Help.cs | 12 ++- slsk-batchdl/Program.cs | 54 +++++++------ slsk-batchdl/SearchAndDownload.cs | 122 +++++++++++++----------------- 7 files changed, 131 insertions(+), 197 deletions(-) diff --git a/README.md b/README.md index f9d7982..9cebebd 100644 --- a/README.md +++ b/README.md @@ -105,8 +105,8 @@ Usage: sldl [OPTIONS] on a per-track basis, so it is best kept off in that case. -d, --desperate Tries harder to find the desired track by searching for the artist/album/title only, then filtering. (slower search) - --fails-to-downrank Number of fails to downrank a user's uploads (default: 1) - --fails-to-ignore Number of fails to ban/ignore a user's uploads (default: 2) + --fails-to-downrank Number of fails to downrank a user's shares (default: 1) + --fails-to-ignore Number of fails to ban/ignore a user's shares (default: 2) --yt-dlp Use yt-dlp to download tracks that weren't found on Soulseek. yt-dlp must be available from the command line. @@ -367,8 +367,12 @@ accept-no-length = false ``` sldl will therefore prefer mp3 files with bitrate between 200 and 2500 kbps, and whose length differs from the supplied length by no more than 3 seconds. It will also prefer files whose -paths contain the supplied artist and album (ignoring case, and bounded by boundary characters) -and which have a non-null length. Changing the last three preferred conditions is not recommended. +paths contain the supplied title and album (ignoring case, and bounded by boundary characters) +and which have non-null length. Changing the last three preferred conditions is not recommended. +Note that files satisfying a subset of the preferred conditions will still be preferred over files +that don't satisfy any condition, but some conditions have precedence over others. For instance, +a file that only satisfies strict-title (if enabled) will always be preferred over a file that +only satisfies the format condition. Run with --print "results-full" to reveal the sorting logic. ### Important note Some info may be unavailable depending on the client used by the peer. For example, the standard diff --git a/slsk-batchdl/Config.cs b/slsk-batchdl/Config.cs index b8613f2..b663f2b 100644 --- a/slsk-batchdl/Config.cs +++ b/slsk-batchdl/Config.cs @@ -537,9 +537,6 @@ static class Config case "bannedusers": cond.BannedUsers = value.Split(',', tr); break; - case "dangerwords": - cond.DangerWords = value.Split(',', tr); - break; case "stricttitle": cond.StrictTitle = bool.Parse(value); break; @@ -967,10 +964,6 @@ static class Config case "--pref-max-bitdepth": preferredCond.MaxBitDepth = int.Parse(args[++i]); break; - case "--pdw": - case "--pref-danger-words": - preferredCond.DangerWords = args[++i].Split(','); - break; case "--pst": case "--pstt": case "--pref-strict-title": @@ -1022,10 +1015,6 @@ static class Config case "--max-bitdepth": necessaryCond.MaxBitDepth = int.Parse(args[++i]); break; - case "--dw": - case "--danger-words": - necessaryCond.DangerWords = args[++i].Split(','); - break; case "--stt": case "--strict-title": setFlag(ref necessaryCond.StrictTitle, ref i); diff --git a/slsk-batchdl/Data.cs b/slsk-batchdl/Data.cs index 298283c..51355de 100644 --- a/slsk-batchdl/Data.cs +++ b/slsk-batchdl/Data.cs @@ -1,6 +1,6 @@ using Enums; +using Soulseek; -using SlDictionary = System.Collections.Concurrent.ConcurrentDictionary; namespace Data { @@ -21,11 +21,11 @@ namespace Data public TrackType Type = TrackType.Normal; public FailureReason FailureReason = FailureReason.None; public TrackState State = TrackState.Initial; - public SlDictionary? Downloads = null; + public List<(SearchResponse, Soulseek.File)>? Downloads = null; public bool OutputsDirectory => Type != TrackType.Normal; - public Soulseek.File? FirstDownload => Downloads?.FirstOrDefault().Value.Item2; - public string? FirstUsername => Downloads?.FirstOrDefault().Value.Item1.Username; + public Soulseek.File? FirstDownload => Downloads?.FirstOrDefault().Item2; + public string? FirstUsername => Downloads?.FirstOrDefault().Item1?.Username; public Track() { } @@ -64,13 +64,13 @@ namespace Data public string ToString(bool noInfo = false) { - if (IsNotAudio && Downloads != null && !Downloads.IsEmpty) - return $"{Utils.GetFileNameSlsk(Downloads.First().Value.Item2.Filename)}"; + if (IsNotAudio && Downloads != null && Downloads.Count > 0) + return $"{Utils.GetFileNameSlsk(Downloads[0].Item2.Filename)}"; string str = Artist; - if (Type == TrackType.Normal && Title.Length == 0 && Downloads != null && !Downloads.IsEmpty) + if (Type == TrackType.Normal && Title.Length == 0 && Downloads != null && Downloads.Count > 0) { - str = $"{Utils.GetFileNameSlsk(Downloads.First().Value.Item2.Filename)}"; + str = $"{Utils.GetFileNameSlsk(Downloads[0].Item2.Filename)}"; } else if (Title.Length > 0 || Album.Length > 0) { @@ -313,12 +313,14 @@ namespace Data } } - public class TrackStringComparer : IEqualityComparer + public class TrackComparer : IEqualityComparer { private bool _ignoreCase = false; - public TrackStringComparer(bool ignoreCase = false) + private int _lenTol = -1; + public TrackComparer(bool ignoreCase = false, int lenTol = -1) { _ignoreCase = ignoreCase; + _lenTol = lenTol; } public bool Equals(Track a, Track b) @@ -330,7 +332,8 @@ namespace Data return string.Equals(a.Title, b.Title, comparer) && string.Equals(a.Artist, b.Artist, comparer) - && string.Equals(a.Album, b.Album, comparer); + && string.Equals(a.Album, b.Album, comparer) + && _lenTol == -1 || (a.Length == -1 && b.Length == -1) || (a.Length != -1 && b.Length != -1 && Math.Abs(a.Length - b.Length) <= _lenTol); } public int GetHashCode(Track a) diff --git a/slsk-batchdl/FileConditions.cs b/slsk-batchdl/FileConditions.cs index 90176e6..5fd317c 100644 --- a/slsk-batchdl/FileConditions.cs +++ b/slsk-batchdl/FileConditions.cs @@ -17,10 +17,8 @@ public class FileConditions public bool StrictTitle = false; public bool StrictArtist = false; public bool StrictAlbum = false; - public string[] DangerWords = Array.Empty(); public string[] Formats = Array.Empty(); public string[] BannedUsers = Array.Empty(); - public string StrictStringRegexRemove = string.Empty; public bool StrictStringDiacrRemove = true; public bool AcceptNoLength = true; public bool AcceptMissingProps = true; @@ -40,7 +38,6 @@ public class FileConditions MinBitDepth = other.MinBitDepth; MaxBitDepth = other.MaxBitDepth; Formats = other.Formats.ToArray(); - DangerWords = other.DangerWords.ToArray(); BannedUsers = other.BannedUsers.ToArray(); } @@ -58,12 +55,10 @@ public class FileConditions StrictTitle == other.StrictTitle && StrictArtist == other.StrictArtist && StrictAlbum == other.StrictAlbum && - StrictStringRegexRemove == other.StrictStringRegexRemove && StrictStringDiacrRemove == other.StrictStringDiacrRemove && AcceptNoLength == other.AcceptNoLength && AcceptMissingProps == other.AcceptMissingProps && Formats.SequenceEqual(other.Formats) && - DangerWords.SequenceEqual(other.DangerWords) && BannedUsers.SequenceEqual(other.BannedUsers); } return false; @@ -81,7 +76,7 @@ public class FileConditions public bool FileSatisfies(Soulseek.File file, Track track, SearchResponse? response) { - return DangerWordSatisfies(file.Filename, track.Title, track.Artist) && FormatSatisfies(file.Filename) + return FormatSatisfies(file.Filename) && LengthToleranceSatisfies(file, track.Length) && BitrateSatisfies(file) && SampleRateSatisfies(file) && StrictTitleSatisfies(file.Filename, track.Title) && StrictArtistSatisfies(file.Filename, track.Artist) && StrictAlbumSatisfies(file.Filename, track.Album) && BannedUsersSatisfies(response) && BitDepthSatisfies(file); @@ -89,7 +84,7 @@ public class FileConditions public bool FileSatisfies(TagLib.File file, Track track, bool filenameChecks = false) { - return DangerWordSatisfies(file.Name, track.Title, track.Artist) && FormatSatisfies(file.Name) + return FormatSatisfies(file.Name) && LengthToleranceSatisfies(file, track.Length) && BitrateSatisfies(file) && SampleRateSatisfies(file) && BitDepthSatisfies(file) && (!filenameChecks || StrictTitleSatisfies(file.Name, track.Title) && StrictArtistSatisfies(file.Name, track.Artist) && StrictAlbumSatisfies(file.Name, track.Album)); @@ -97,44 +92,19 @@ public class FileConditions public bool FileSatisfies(SimpleFile file, Track track, bool filenameChecks = false) { - return DangerWordSatisfies(file.Path, track.Title, track.Artist) && FormatSatisfies(file.Path) + return FormatSatisfies(file.Path) && LengthToleranceSatisfies(file, track.Length) && BitrateSatisfies(file) && SampleRateSatisfies(file) && BitDepthSatisfies(file) && (!filenameChecks || StrictTitleSatisfies(file.Path, track.Title) && StrictArtistSatisfies(file.Path, track.Artist) && StrictAlbumSatisfies(file.Path, track.Album)); } - public bool DangerWordSatisfies(string fname, string tname, string aname) - { - if (tname.Length == 0) - return true; - - fname = Utils.GetFileNameWithoutExtSlsk(fname).Replace(" — ", " - "); - tname = tname.Replace(" — ", " - "); - - foreach (var word in DangerWords) - { - if (fname.ContainsIgnoreCase(word) ^ tname.ContainsIgnoreCase(word)) - { - if (!(fname.Contains(" - ") && fname.ContainsIgnoreCase(word) && aname.ContainsIgnoreCase(word))) - { - if (word == "mix") - return fname.ContainsIgnoreCase("original mix") || tname.ContainsIgnoreCase("original mix"); - else - return false; - } - } - } - - return true; - } - public bool StrictTitleSatisfies(string fname, string tname, bool noPath = true) { if (!StrictTitle || tname.Length == 0) return true; fname = noPath ? Utils.GetFileNameWithoutExtSlsk(fname) : fname; - return StrictString(fname, tname, StrictStringRegexRemove, StrictStringDiacrRemove, ignoreCase: true); + return StrictString(fname, tname, StrictStringDiacrRemove, ignoreCase: true); } public bool StrictArtistSatisfies(string fname, string aname) @@ -142,7 +112,7 @@ public class FileConditions if (!StrictArtist || aname.Length == 0) return true; - return StrictString(fname, aname, StrictStringRegexRemove, StrictStringDiacrRemove, ignoreCase: true, boundarySkipWs: false); + return StrictString(fname, aname, StrictStringDiacrRemove, ignoreCase: true, boundarySkipWs: false); } public bool StrictAlbumSatisfies(string fname, string alname) @@ -150,25 +120,24 @@ public class FileConditions if (!StrictAlbum || alname.Length == 0) return true; - return StrictString(Utils.GetDirectoryNameSlsk(fname), alname, StrictStringRegexRemove, StrictStringDiacrRemove, ignoreCase: true); + return StrictString(Utils.GetDirectoryNameSlsk(fname), alname, StrictStringDiacrRemove, ignoreCase: true); } - public static string StrictStringPreprocess(string str, string regexRemove = "", bool diacrRemove = true) + public static string StrictStringPreprocess(string str, bool diacrRemove = true) { str = str.Replace('_', ' ').ReplaceInvalidChars(' ', true, false); - str = regexRemove.Length > 0 ? Regex.Replace(str, regexRemove, "") : str; str = diacrRemove ? str.RemoveDiacritics() : str; str = str.Trim().RemoveConsecutiveWs(); return str; } - public static bool StrictString(string fname, string tname, string regexRemove = "", bool diacrRemove = true, bool ignoreCase = true, bool boundarySkipWs = true) + public static bool StrictString(string fname, string tname, bool diacrRemove = true, bool ignoreCase = true, bool boundarySkipWs = true) { if (tname.Length == 0) return true; - fname = StrictStringPreprocess(fname, regexRemove, diacrRemove); - tname = StrictStringPreprocess(tname, regexRemove, diacrRemove); + fname = StrictStringPreprocess(fname, diacrRemove); + tname = StrictStringPreprocess(tname, diacrRemove); if (boundarySkipWs) return fname.ContainsWithBoundaryIgnoreWs(tname, ignoreCase, acceptLeftDigit: true); @@ -252,43 +221,24 @@ public class FileConditions public string GetNotSatisfiedName(Soulseek.File file, Track track, SearchResponse? response) { - if (!DangerWordSatisfies(file.Filename, track.Title, track.Artist)) - return "DangerWord fails"; - if (!FormatSatisfies(file.Filename)) - return "Format fails"; - if (!LengthToleranceSatisfies(file, track.Length)) - return "Length fails"; - if (!BitrateSatisfies(file)) - return "Bitrate fails"; - if (!SampleRateSatisfies(file)) - return "SampleRate fails"; - if (!StrictTitleSatisfies(file.Filename, track.Title)) - return "StrictTitle fails"; - if (!StrictArtistSatisfies(file.Filename, track.Artist)) - return "StrictArtist fails"; - if (!BitDepthSatisfies(file)) - return "BitDepth fails"; if (!BannedUsersSatisfies(response)) return "BannedUsers fails"; - return "Satisfied"; - } - - public string GetNotSatisfiedName(TagLib.File file, Track track) - { - if (!DangerWordSatisfies(file.Name, track.Title, track.Artist)) - return "DangerWord fails"; - if (!FormatSatisfies(file.Name)) - return "Format fails"; + if (!StrictTitleSatisfies(file.Filename, track.Title)) + return "StrictTitle fails"; + if (track.Type == Enums.TrackType.Album && !StrictAlbumSatisfies(file.Filename, track.Artist)) + return "StrictAlbum fails"; + if (!StrictArtistSatisfies(file.Filename, track.Artist)) + return "StrictArtist fails"; if (!LengthToleranceSatisfies(file, track.Length)) - return "Length fails"; + return "LengthTolerance fails"; + if (!FormatSatisfies(file.Filename)) + return "Format fails"; + if (track.Type != Enums.TrackType.Album && !StrictAlbumSatisfies(file.Filename, track.Artist)) + return "StrictAlbum fails"; if (!BitrateSatisfies(file)) return "Bitrate fails"; if (!SampleRateSatisfies(file)) return "SampleRate fails"; - if (!StrictTitleSatisfies(file.Name, track.Title)) - return "StrictTitle fails"; - if (!StrictArtistSatisfies(file.Name, track.Artist)) - return "StrictArtist fails"; if (!BitDepthSatisfies(file)) return "BitDepth fails"; return "Satisfied"; diff --git a/slsk-batchdl/Help.cs b/slsk-batchdl/Help.cs index fc57940..11e2a89 100644 --- a/slsk-batchdl/Help.cs +++ b/slsk-batchdl/Help.cs @@ -83,8 +83,8 @@ public static class Help on a per-track basis, so it is best kept off in that case. -d, --desperate Tries harder to find the desired track by searching for the artist/album/title only, then filtering. (slower search) - --fails-to-downrank Number of fails to downrank a user's uploads (default: 1) - --fails-to-ignore Number of fails to ban/ignore a user's uploads (default: 2) + --fails-to-downrank Number of fails to downrank a user's shares (default: 1) + --fails-to-ignore Number of fails to ban/ignore a user's shares (default: 2) --yt-dlp Use yt-dlp to download tracks that weren't found on Soulseek. yt-dlp must be available from the command line. @@ -339,8 +339,12 @@ public static class Help sldl will therefore prefer mp3 files with bitrate between 200 and 2500 kbps, and whose length differs from the supplied length by no more than 3 seconds. It will also prefer files whose - paths contain the supplied artist and album (ignoring case, and bounded by boundary characters) - and which have a non-null length. Changing the last three preferred conditions is not recommended. + paths contain the supplied title and album (ignoring case, and bounded by boundary characters) + and which have non-null length. Changing the last three preferred conditions is not recommended. + Note that files satisfying a subset of the preferred conditions will still be preferred over files + that don't satisfy any condition, but some conditions have precedence over others. For instance, + a file that only satisfies strict-title (if enabled) will always be preferred over a file that + only satisfies the format condition. Run with --print ""results-full"" to reveal the sorting logic. Important note Some info may be unavailable depending on the client used by the peer. For example, the standard diff --git a/slsk-batchdl/Program.cs b/slsk-batchdl/Program.cs index 9d50ce8..2f6a320 100644 --- a/slsk-batchdl/Program.cs +++ b/slsk-batchdl/Program.cs @@ -350,12 +350,16 @@ static partial class Program else if (tle.source.Type == TrackType.Album) { Console.WriteLine(new string('-', 60)); - Console.WriteLine($"Results for album {tle.source.ToString(true)}:"); + + if (!Config.printOption.HasFlag(PrintOption.Full)) + Console.WriteLine($"Result 1 of {tle.list.Count} for album {tle.source.ToString(true)}:"); + else + Console.WriteLine($"Results ({tle.list.Count}) for album {tle.source.ToString(true)}:"); if (tle.list.Count > 0 && tle.list[0].Count > 0) { if (!Config.noBrowseFolder) - Console.WriteLine("[Skipped full folder retrieval]"); + Console.WriteLine("[Skipping full folder retrieval]"); foreach (var ls in tle.list) { @@ -430,12 +434,12 @@ static partial class Program static void PrintAlbum(List albumTracks, bool retrieveAll = false) { - if (albumTracks.Count == 0 && albumTracks[0].Downloads.IsEmpty) + if (albumTracks.Count == 0 && albumTracks[0].Downloads.Count == 0) return; - var response = albumTracks[0].Downloads.First().Value.Item1; + var response = albumTracks[0].Downloads[0].Item1; string userInfo = $"{response.Username} ({((float)response.UploadSpeed / (1024 * 1024)):F3}MB/s)"; - var (parents, props) = FolderInfo(albumTracks.SelectMany(x => x.Downloads.Select(d => d.Value.Item2))); + var (parents, props) = FolderInfo(albumTracks.SelectMany(x => x.Downloads.Select(d => d.Item2))); Console.WriteLine(); WriteLine($"User : {userInfo}\nFolder: {parents}\nProps : {props}", ConsoleColor.White); @@ -623,23 +627,23 @@ static partial class Program { var albumArtList = list //.Where(tracks => tracks) - .Select(tracks => tracks.Where(t => Utils.IsImageFile(t.Downloads.First().Value.Item2.Filename))) + .Select(tracks => tracks.Where(t => Utils.IsImageFile(t.Downloads[0].Item2.Filename))) .Where(tracks => tracks.Any()); if (option == AlbumArtOption.Largest) { list = albumArtList - .OrderByDescending(tracks => tracks.Select(t => t.Downloads.First().Value.Item2.Size).Max() / 1024 / 100) - .ThenByDescending(tracks => tracks.First().Downloads.First().Value.Item1.UploadSpeed / 1024 / 300) - .ThenByDescending(tracks => tracks.Select(t => t.Downloads.First().Value.Item2.Size).Sum() / 1024 / 100) + .OrderByDescending(tracks => tracks.Select(t => t.Downloads[0].Item2.Size).Max() / 1024 / 100) + .ThenByDescending(tracks => tracks.First().Downloads[0].Item1.UploadSpeed / 1024 / 300) + .ThenByDescending(tracks => tracks.Select(t => t.Downloads[0].Item2.Size).Sum() / 1024 / 100) .Select(x => x.ToList()).ToList(); } else if (option == AlbumArtOption.Most) { list = albumArtList .OrderByDescending(tracks => tracks.Count()) - .ThenByDescending(tracks => tracks.First().Downloads.First().Value.Item1.UploadSpeed / 1024 / 300) - .ThenByDescending(tracks => tracks.Select(t => t.Downloads.First().Value.Item2.Size).Sum() / 1024 / 100) + .ThenByDescending(tracks => tracks.First().Downloads[0].Item1.UploadSpeed / 1024 / 300) + .ThenByDescending(tracks => tracks.Select(t => t.Downloads[0].Item2.Size).Sum() / 1024 / 100) .Select(x => x.ToList()).ToList(); } } @@ -655,7 +659,7 @@ static partial class Program else if (option == AlbumArtOption.Largest) { long curMax = dlFiles.Keys.Where(x => Utils.IsImageFile(x) && File.Exists(x)).Max(x => new FileInfo(x).Length); - need = curMax < list[0].Max(t => t.Downloads.First().Value.Item2.Size) - 1024 * 50; + need = curMax < list[0].Max(t => t.Downloads[0].Item2.Size) - 1024 * 50; } return need; @@ -693,7 +697,7 @@ static partial class Program if (!Config.noBrowseFolder && !Config.interactiveMode && !retrievedFolders.Contains(soulseekFolderPathPrefix)) { Console.WriteLine("Getting all files in folder..."); - var response = tracks[0].Downloads.First().Value.Item1; + var response = tracks[0].Downloads[0].Item1; await CompleteFolder(tracks, response, soulseekFolderPathPrefix); retrievedFolders.Add(soulseekFolderPathPrefix); } @@ -865,9 +869,9 @@ static partial class Program static string GetCommonPathPrefix(List tracks) { if (tracks.Count == 1) - return Utils.GetDirectoryNameSlsk(tracks.First().Downloads.First().Value.Item2.Filename); + return Utils.GetDirectoryNameSlsk(tracks.First().Downloads[0].Item2.Filename); else - return Utils.GreatestCommonPath(tracks.SelectMany(x => x.Downloads.Select(y => y.Value.Item2.Filename)), dirsep: '\\'); + return Utils.GreatestCommonPath(tracks.SelectMany(x => x.Downloads.Select(y => y.Item2.Filename)), dirsep: '\\'); } @@ -897,7 +901,7 @@ static partial class Program { Console.WriteLine(); var tracks = list[aidx]; - var response = tracks[0].Downloads.First().Value.Item1; + var response = tracks[0].Downloads[0].Item1; var folder = GetCommonPathPrefix(tracks); if (retrieveFolder && !Config.noBrowseFolder && !retrievedFolders.Contains(folder)) @@ -908,7 +912,7 @@ static partial class Program } string userInfo = $"{response.Username} ({((float)response.UploadSpeed / (1024 * 1024)):F3}MB/s)"; - var (parents, props) = FolderInfo(tracks.SelectMany(x => x.Downloads.Select(d => d.Value.Item2))); + var (parents, props) = FolderInfo(tracks.SelectMany(x => x.Downloads.Select(d => d.Item2))); WriteLine($"[{aidx + 1} / {list.Count}]", ConsoleColor.DarkGray); WriteLine($"User : {userInfo}\nFolder: {parents}\nProps : {props}", ConsoleColor.White); @@ -1396,7 +1400,7 @@ static partial class Program string ancestor = ""; if (showAncestors) - ancestor = Utils.GreatestCommonPath(tracks.SelectMany(x => x.Downloads.Select(y => y.Value.Item2.Filename)), Path.DirectorySeparatorChar); + ancestor = Utils.GreatestCommonPath(tracks.SelectMany(x => x.Downloads.Select(y => y.Item2.Filename)), Path.DirectorySeparatorChar); if (pathsOnly) { @@ -1405,9 +1409,9 @@ static partial class Program foreach (var x in tracks[i].Downloads) { if (ancestor.Length == 0) - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser)); else - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, customPath: x.Value.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); } } } @@ -1447,23 +1451,23 @@ static partial class Program foreach (var x in tracks[i].Downloads) { if (ancestor.Length == 0) - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser)); else - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, customPath: x.Value.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); } if (tracks[i].Downloads?.Count > 0) Console.WriteLine(); } } else { - Console.WriteLine($" File: {Utils.GetFileNameSlsk(tracks[i].Downloads.First().Value.Item2.Filename)}"); + Console.WriteLine($" File: {Utils.GetFileNameSlsk(tracks[i].Downloads[0].Item2.Filename)}"); Console.WriteLine($" Shares: {tracks[i].Downloads.Count}"); foreach (var x in tracks[i].Downloads) { if (ancestor.Length == 0) - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser)); else - Console.WriteLine(" " + DisplayString(tracks[i], x.Value.Item2, x.Value.Item1, customPath: x.Value.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); + Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, ""), infoFirst: infoFirst, showUser: showUser)); } Console.WriteLine(); } diff --git a/slsk-batchdl/SearchAndDownload.cs b/slsk-batchdl/SearchAndDownload.cs index 2e824ff..b546867 100644 --- a/slsk-batchdl/SearchAndDownload.cs +++ b/slsk-batchdl/SearchAndDownload.cs @@ -22,6 +22,7 @@ static partial class Program throw new Exception(); responseData ??= new ResponseData(); + IEnumerable<(SlResponse response, SlFile file)>? orderedResults = null; var progress = GetProgressBar(Config.displayMode); var results = new SlDictionary(); var fsResults = new SlDictionary(); @@ -38,7 +39,7 @@ static partial class Program if (track.Downloads != null) { - results = track.Downloads; + orderedResults = track.Downloads; goto downloads; } @@ -141,9 +142,10 @@ static partial class Program downloads: - if (downloading == 0 && !results.IsEmpty) + if (downloading == 0 && (!results.IsEmpty || orderedResults != null)) { - var orderedResults = OrderedResults(results, track, true); + if (orderedResults == null) + orderedResults = OrderedResults(results, track, useInfer: true); int trackTries = Config.maxRetriesPerTrack; async Task process(SlResponse response, SlFile file) @@ -376,13 +378,12 @@ static partial class Program Album = track.Album, Length = x.file.Length ?? -1, IsNotAudio = !Utils.IsMusicFile(x.file.Filename), - Downloads = new ConcurrentDictionary( - new Dictionary { { x.response.Username + '\\' + x.file.Filename, x } }) + Downloads = new() { x }, }; ls.Add(t); } - ls = ls.OrderBy(t => t.IsNotAudio).ThenBy(t => t.Downloads.First().Value.Item2.Filename).ToList(); + ls = ls.OrderBy(t => t.IsNotAudio).ThenBy(t => t.Downloads[0].Item2.Filename).ToList(); result.Add(ls); } @@ -396,9 +397,9 @@ static partial class Program static async Task> GetAggregateTracks(Track track, ResponseData responseData) { - var results = new ConcurrentDictionary(); + var results = new SlDictionary(); SearchOptions getSearchOptions(int timeout, FileConditions nec, FileConditions prf) => - new SearchOptions( + new ( minimumResponseFileCount: 1, minimumPeerUploadSpeed: 1, removeSingleCharacterSearchTerms: Config.removeSingleCharacterSearchTerms, @@ -410,9 +411,6 @@ static partial class Program fileFilter: (file) => { return Utils.IsMusicFile(file.Filename) && nec.FileSatisfies(file, track, null); - //&& FileConditions.StrictString(file.Filename, track.ArtistName, ignoreCase: true) - //&& FileConditions.StrictString(file.Filename, track.TrackTitle, ignoreCase: true) - //&& FileConditions.StrictString(file.Filename, track.Album, ignoreCase: true); } ); void handler(SlResponse r) @@ -433,11 +431,8 @@ static partial class Program string trackName = track.Title.Trim(); string albumName = track.Album.Trim(); - //var orderedResults = OrderedResults(results, track, false, false, false); - - var fileResponses = results.Select(x => x.Value); - - var equivalentFiles = EquivalentFiles(track, fileResponses).ToList(); + var equivalentFiles = EquivalentFiles(track, results.Select(x => x.Value)) + .Select(x => (x.Item1, OrderedResults(x.Item2, track, false, false, false))).ToList(); if (!Config.relax) { @@ -452,8 +447,7 @@ static partial class Program var tracks = equivalentFiles .Select(kvp => { - kvp.Item1.Downloads = new SlDictionary( - kvp.Item2.ToDictionary(item => { return item.response.Username + "\\" + item.file.Filename; }, item => item)); + kvp.Item1.Downloads = kvp.Item2.ToList(); return kvp.Item1; }).ToList(); @@ -498,6 +492,7 @@ static partial class Program sortedLengthLists.Add((sortedLengths, album, user)); } + var usernamesList = new List>(); var lengthsList = new List(); var res = new List>>(); @@ -511,8 +506,8 @@ static partial class Program { if (lengths.Length == 1 && lengthsList[i].Length == 1) { - var t1 = InferTrack(album[0].Downloads.First().Value.Item2.Filename, new Track()); - var t2 = InferTrack(res[i][0][0].Downloads.First().Value.Item2.Filename, new Track()); + var t1 = InferTrack(album[0].Downloads[0].Item2.Filename, new Track()); + var t2 = InferTrack(res[i][0][0].Downloads[0].Item2.Filename, new Track()); if ((t2.Artist.ContainsIgnoreCase(t1.Artist) || t1.Artist.ContainsIgnoreCase(t2.Artist)) && (t2.Title.ContainsIgnoreCase(t1.Title) || t1.Title.ContainsIgnoreCase(t2.Title))) @@ -527,6 +522,7 @@ static partial class Program if (found) { + usernamesList[i].Add(user); res[i].Add(album); break; } @@ -539,12 +535,17 @@ static partial class Program } else { + usernamesList.Add(new() { user }); lengthsList.Add(lengths); res.Add(new List> { album }); } } - res = res.Where(x => x.Count >= Config.minSharesAggregate).OrderByDescending(x => x.Count).ToList(); + res = res.Select((x, i) => (x, i)) + .Where(x => usernamesList[x.i].Count >= Config.minSharesAggregate) + .OrderByDescending(x => usernamesList[x.i].Count) + .Select(x => x.x) + .ToList(); return res; // Note: The nested lists are still ordered according to OrderedResults } @@ -586,7 +587,7 @@ static partial class Program if (allFiles.Count > tracks.Count) { - var paths = tracks.Select(x => x.Downloads.First().Value.Item2.Filename).ToHashSet(); + var paths = tracks.Select(x => x.Downloads[0].Item2.Filename).ToHashSet(); var first = tracks[0]; foreach ((var dir, var file) in allFiles) @@ -600,8 +601,7 @@ static partial class Program Artist = first.Artist, Album = first.Album, IsNotAudio = !Utils.IsMusicFile(file.Filename), - Downloads = new ConcurrentDictionary( - new Dictionary { { response.Username + '\\' + fullPath, (response, newFile) } }) + Downloads = new() { (response, newFile) } }; tracks.Add(t); } @@ -615,7 +615,7 @@ static partial class Program } - static IOrderedEnumerable<(Track, IEnumerable<(SlResponse response, SlFile file)>)> EquivalentFiles(Track track, + static IEnumerable<(Track, IEnumerable<(SlResponse response, SlFile file)>)> EquivalentFiles(Track track, IEnumerable<(SlResponse, SlFile)> fileResponses, int minShares = -1) { if (minShares == -1) @@ -628,53 +628,32 @@ static partial class Program return t; } - var res = fileResponses - .GroupBy(inferTrack, new TrackStringComparer(ignoreCase: true)) - .Where(group => group.Select(x => x.Item1.Username).Distinct().Count() >= minShares) - .SelectMany(group => + var groups = fileResponses + .GroupBy(inferTrack, new TrackComparer(ignoreCase: true, Config.necessaryCond.LengthTolerance)) + .Select(x => (x, x.Select(y => y.Item1.Username).Distinct().Count())) + .Where(x => x.Item2 >= minShares) + .OrderByDescending(x => x.Item2) + .Select(x => x.x) + .Select(x => { - var sortedTracks = group.OrderBy(t => t.Item2.Length).Where(x => x.Item2.Length != null).ToList(); - var groups = new List<(Track, List<(SearchResponse, Soulseek.File)>)>(); - var noLengthGroup = group.Where(x => x.Item2.Length == null); - for (int i = 0; i < sortedTracks.Count;) - { - var subGroup = new List<(SearchResponse, Soulseek.File)> { sortedTracks[i] }; - int j = i + 1; - while (j < sortedTracks.Count) - { - int l1 = (int)sortedTracks[j].Item2.Length; - int l2 = (int)sortedTracks[i].Item2.Length; - if (Config.necessaryCond.LengthTolerance == -1 || Math.Abs(l1 - l2) <= Config.necessaryCond.LengthTolerance) - { - subGroup.Add(sortedTracks[j]); - j++; - } - else break; - } - var t = new Track(group.Key); - t.Length = (int)sortedTracks[i].Item2.Length; - groups.Add((t, subGroup)); - i = j; - } + if (x.Key.Length == -1) + x.Key.Length = x.FirstOrDefault(y => y.Item2.Length != null).Item2?.Length ?? -1; + return (x.Key, x.AsEnumerable()); + }); - if (noLengthGroup.Any()) - { - if (groups.Count > 0 && !Config.preferredCond.AcceptNoLength) - groups.First().Item2.AddRange(noLengthGroup); - else - groups.Add((group.Key, noLengthGroup.ToList())); - } - - return groups.Where(subGroup => subGroup.Item2.Select(x => x.Item1.Username).Distinct().Count() >= minShares) - .Select(subGroup => (subGroup.Item1, subGroup.Item2.AsEnumerable())); - }).OrderByDescending(x => x.Item2.Count()); - - return res; + return groups; } static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable> results, Track track, bool useInfer = false, bool useLevenshtein = true, bool albumMode = false) + { + return OrderedResults(results.Select(x => x.Value), track, useInfer, useLevenshtein, albumMode); + } + + + static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable<(SlResponse, SlFile)> results, + Track track, bool useInfer = false, bool useLevenshtein = true, bool albumMode = false) { bool useBracketCheck = true; if (albumMode) @@ -685,9 +664,10 @@ static partial class Program } Dictionary? infTracksAndCounts = null; - if (useInfer) + + if (useInfer) // this is very slow { - var equivalentFiles = EquivalentFiles(track, results.Select(x => x.Value), 1); + var equivalentFiles = EquivalentFiles(track, results, 1); infTracksAndCounts = equivalentFiles .SelectMany(t => t.Item2, (t, f) => new { t.Item1, f.response.Username, f.file.Filename, Count = t.Item2.Count() }) .ToSafeDictionary(x => $"{x.Username}\\{x.Filename}", y => (y.Item1, y.Count)); @@ -710,7 +690,7 @@ static partial class Program } var random = new Random(); - return results.Select(kvp => (response: kvp.Value.Item1, file: kvp.Value.Item2)) + return results.Select(x => (response: x.Item1, file: x.Item2)) .Where(x => userSuccessCount.GetValueOrDefault(x.response.Username, 0) > Config.ignoreOn) .OrderByDescending(x => userSuccessCount.GetValueOrDefault(x.response.Username, 0) > Config.downrankOn) .ThenByDescending(x => Config.necessaryCond.FileSatisfies(x.file, track, x.response)) @@ -718,10 +698,11 @@ static partial class Program .ThenByDescending(x => (x.file.Length != null && x.file.Length > 0) || Config.preferredCond.AcceptNoLength) .ThenByDescending(x => !useBracketCheck || FileConditions.BracketCheck(track, inferredTrack(x).Item1)) // downrank result if it contains '(' or '[' and the title does not (avoid remixes) .ThenByDescending(x => Config.preferredCond.StrictTitleSatisfies(x.file.Filename, track.Title)) + .ThenByDescending(x => !albumMode || Config.preferredCond.StrictAlbumSatisfies(x.file.Filename, track.Album)) .ThenByDescending(x => Config.preferredCond.StrictArtistSatisfies(x.file.Filename, track.Title)) .ThenByDescending(x => Config.preferredCond.LengthToleranceSatisfies(x.file, track.Length)) .ThenByDescending(x => Config.preferredCond.FormatSatisfies(x.file.Filename)) - .ThenByDescending(x => Config.preferredCond.StrictAlbumSatisfies(x.file.Filename, track.Album)) + .ThenByDescending(x => albumMode || Config.preferredCond.StrictAlbumSatisfies(x.file.Filename, track.Album)) .ThenByDescending(x => Config.preferredCond.BitrateSatisfies(x.file)) .ThenByDescending(x => Config.preferredCond.SampleRateSatisfies(x.file)) .ThenByDescending(x => Config.preferredCond.BitDepthSatisfies(x.file)) @@ -901,7 +882,7 @@ static partial class Program } else { - var orderedResults = OrderedResults(results, track, true); + var orderedResults = OrderedResults(results, track, useInfer: true); int count = 0; Console.WriteLine(); foreach (var (response, file) in orderedResults) @@ -971,7 +952,6 @@ static partial class Program filename = Utils.GetFileNameWithoutExtSlsk(filename).Replace(" — ", " - ").Replace('_', ' ').Trim().RemoveConsecutiveWs(); var trackNumStart = new Regex(@"^(?:(?:[0-9][-\.])?\d{2,3}[. -]|\b\d\.\s|\b\d\s-\s)(?=.+\S)"); - //var trackNumMiddle = new Regex(@"\s+-\s+(\d{2,3})(?: -|\.|)\s+|\s+-(\d{2,3})-\s+"); var trackNumMiddle = new Regex(@"(?<= - )((\d-)?\d{2,3}|\d{2,3}\.?)\s+"); var trackNumMiddleAlt = new Regex(@"\s+-(\d{2,3})-\s+");