mirror/slsk-batchdl
1
0
Fork 0
mirror of https://github.com/fiso64/slsk-batchdl.git synced 2024-12-22 06:22:41 +00:00
Code Issues Activity

parallel album searching

Browse source
This commit is contained in:
fiso64 2024-12-15 22:15:51 +01:00
parent 85571115cf
commit 111442b40c
10 changed files with 274 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -198,6 +198,7 @@ Usage: sldl <input> [OPTIONS]
the directory fails to download. Set to 'delete' to delete
the files instead. Set to 'disable' keep it where it is.
Default: {configured output dir}/failed
--album-parallel-search Run album searches in parallel
```
#### Aggregate Download Options
```
@ -357,7 +358,8 @@ The search query is determined as follows:
- --concurrent-downloads - set it to 4 or more
- --max-stale-time is set to 50 seconds by default, so it will wait a long time before giving
up on a file
- --searches-per-time - increase at the risk of bans.
- --searches-per-time - increase at the risk of bans
- --album-parallel-search - enables parallel searching for album entries
## File conditions

15
slsk-batchdl/Config.cs
View file

@ -7,7 +7,10 @@ using System.Text.RegularExpressions;
public class Config
{
public FileConditions necessaryCond = new();
public FileConditions necessaryCond = new()
{
Formats = new string[] { ".mp3", ".flac", ".ogg", ".m4a", ".opus", ".wav", ".aac", ".alac" },
};
public FileConditions preferredCond = new()
{
@ -79,6 +82,7 @@ public class Config
public bool writePlaylist = false;
public bool skipExisting = true;
public bool writeIndex = true;
public bool parallelAlbumSearch = false;
public int downrankOn = -1;
public int ignoreOn = -2;
public int minAlbumTrackCount = -1;
@ -96,6 +100,7 @@ public class Config
public int searchesPerTime = 34;
public int searchRenewTime = 220;
public int aggregateLengthTol = 3;
public int parallelAlbumSearchProcesses = 5;
public double fastSearchMinUpSpeed = 1.0;
public Track regexToReplace = new();
public Track regexReplaceBy = new();
@ -1257,6 +1262,14 @@ public class Config
case "--aggregate-length-tol":
aggregateLengthTol = int.Parse(args[++i]);
break;
case "--aps":
case "--album-parallel-search":
setFlag(ref parallelAlbumSearch, ref i);
break;
case "--apsc":
case "--album-parallel-search-count":
parallelAlbumSearchProcesses = int.Parse(args[++i]);
break;
default:
throw new ArgumentException($"Unknown argument: {args[i]}");
}

2
slsk-batchdl/Extractors/Spotify.cs
View file

@ -92,7 +92,7 @@ namespace Extractors
else throw;
}
tle.defaultFolderName = playlistName;
tle.defaultFolderName = playlistName.ReplaceInvalidChars(' ').Trim();
tle.enablesIndexByDefault = true;
tle.list.Add(tracks);
}

8
slsk-batchdl/Extractors/YouTube.cs
View file

@ -607,7 +607,7 @@ namespace Extractors
}
}
public static async Task<List<(int length, string id, string title)>> YtdlpSearch(Track track)
public static async Task<List<(int length, string id, string title)>> YtdlpSearch(Track track, bool printCommand = false)
{
Process process = new Process();
ProcessStartInfo startInfo = new ProcessStartInfo();
@ -623,6 +623,8 @@ namespace Extractors
process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand);
process.Start();
List<(int, string, string)> results = new List<(int, string, string)>();
@ -644,7 +646,7 @@ namespace Extractors
return results;
}
public static async Task<string> YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "")
public static async Task<string> YtdlpDownload(string id, string savePathNoExt, string ytdlpArgument = "", bool printCommand = false)
{
Process process = new Process();
ProcessStartInfo startInfo = new ProcessStartInfo();
@ -666,6 +668,8 @@ namespace Extractors
//process.OutputDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
//process.ErrorDataReceived += (sender, e) => { Console.WriteLine(e.Data); };
Printing.WriteLineIf($"{startInfo.FileName} {startInfo.Arguments}", printCommand);
process.Start();
process.WaitForExit();

9
slsk-batchdl/FileManager.cs
View file

@ -117,7 +117,14 @@ public class FileManager
if (track.DownloadPath.Length == 0)
return;
string newFilePath = Path.Join(parent, Path.GetFileName(track.DownloadPath));
string? part = null;
if (remoteCommonDir != null && Utils.IsInDirectory(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename), remoteCommonDir, true))
{
part = Utils.GetFileNameSlsk(Utils.GetDirectoryNameSlsk(track.FirstDownload.Filename));
}
string newFilePath = Path.Join(parent, part, Path.GetFileName(track.DownloadPath));
try
{

6
slsk-batchdl/Help.cs
View file

@ -4,7 +4,7 @@
// --invalid-replace-str, --cond, --pref
// --fast-search-delay, --fast-search-min-up-speed
// --min-album-track-count, --max-album-track-count, --extract-max-track-count, --extract-min-track-count
// --skip-mode-music-dir, --skip-mode-output-dir
// --skip-mode-music-dir, --skip-mode-output-dir, --album-parallel-search-count
public static class Help
{
@ -168,6 +168,7 @@ public static class Help
the directory fails to download. Set to 'delete' to delete
the files instead. Set to the empty string """" to disable.
Default: {configured output dir}/failed
--album-parallel-search Run album searches in parallel
Aggregate Download
-g, --aggregate Aggregate download mode: Find and download all distinct
@ -333,7 +334,8 @@ public static class Help
--concurrent-downloads - set it to 4 or more
--max-stale-time is set to 50 seconds by default, so it will wait a long time before giving
up on a file
--searches-per-time - increase at the risk of bans.
--searches-per-time - increase at the risk of bans
--album-parallel-search - enables parallel searching for album entries
";
const string fileConditionsHelp = @"

2
slsk-batchdl/Models/TrackListEntry.cs
View file

@ -22,6 +22,8 @@ namespace Models
public FileSkipper? outputDirSkipper = null;
public FileSkipper? musicDirSkipper = null;
public bool CanParallelSearch => source.Type == TrackType.Album || source.Type == TrackType.Aggregate;
public TrackListEntry(TrackType trackType)
{
list = new List<List<Track>>();

334
slsk-batchdl/Program.cs
View file

@ -15,6 +15,7 @@ using static Printing;
using Directory = System.IO.Directory;
using File = System.IO.File;
using SlFile = Soulseek.File;
using Konsole;
static partial class Program
{
@ -54,9 +55,7 @@ static partial class Program
trackLists.UpgradeListTypes(config.aggregate, config.album);
trackLists.SetListEntryOptions();
InitConfigs(config);
await MainLoop();
await MainLoop(config);
WriteLineIf("Mainloop done", config.debugInfo);
}
@ -105,63 +104,63 @@ static partial class Program
}
static void InitEditors(TrackListEntry tle, Config config)
{
tle.playlistEditor = new M3uEditor(trackLists, config.writePlaylist ? M3uOption.Playlist : M3uOption.None, config.offset);
tle.indexEditor = new M3uEditor(trackLists, config.writeIndex ? M3uOption.Index : M3uOption.None);
}
static void InitFileSkippers(TrackListEntry tle, Config config)
{
if (config.skipExisting)
{
FileConditions? cond = null;
if (config.skipCheckPrefCond)
{
cond = config.necessaryCond.With(config.preferredCond);
}
else if (config.skipCheckCond)
{
cond = config.necessaryCond;
}
tle.outputDirSkipper = FileSkipperRegistry.GetSkipper(config.skipMode, config.parentDir, cond, tle.indexEditor);
if (config.skipMusicDir.Length > 0)
{
if (!Directory.Exists(config.skipMusicDir))
Console.WriteLine("Error: Music directory does not exist");
else
tle.musicDirSkipper = FileSkipperRegistry.GetSkipper(config.skipModeMusicDir, config.skipMusicDir, cond, tle.indexEditor);
}
}
}
static void InitConfigs(Config defaultConfig)
{
if (trackLists.Count == 0)
return;
//if (trackLists.Count == 0)
// return;
void initEditors(TrackListEntry tle, Config config)
{
tle.playlistEditor = new M3uEditor(trackLists, config.writePlaylist ? M3uOption.Playlist : M3uOption.None, config.offset);
tle.indexEditor = new M3uEditor(trackLists, config.writeIndex ? M3uOption.Index : M3uOption.None);
}
//foreach (var tle in trackLists.lists)
//{
// tle.config = defaultConfig.Copy();
// tle.config.UpdateProfiles(tle);
void initFileSkippers(TrackListEntry tle, Config config)
{
if (config.skipExisting)
{
FileConditions? cond = null;
// if (tle.extractorCond != null)
// {
// tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
// tle.extractorCond = null;
// }
// if (tle.extractorPrefCond != null)
// {
// tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
// tle.extractorPrefCond = null;
// }
if (config.skipCheckPrefCond)
{
cond = config.necessaryCond.With(config.preferredCond);
}
else if (config.skipCheckCond)
{
cond = config.necessaryCond;
}
tle.outputDirSkipper = FileSkipperRegistry.GetSkipper(config.skipMode, config.parentDir, cond, tle.indexEditor);
if (config.skipMusicDir.Length > 0)
{
if (!Directory.Exists(config.skipMusicDir))
Console.WriteLine("Error: Music directory does not exist");
else
tle.musicDirSkipper = FileSkipperRegistry.GetSkipper(config.skipModeMusicDir, config.skipMusicDir, cond, tle.indexEditor);
}
}
}
foreach (var tle in trackLists.lists)
{
tle.config = defaultConfig.Copy();
tle.config.UpdateProfiles(tle);
if (tle.extractorCond != null)
{
tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
tle.extractorCond = null;
}
if (tle.extractorPrefCond != null)
{
tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
tle.extractorPrefCond = null;
}
initEditors(tle, tle.config);
initFileSkippers(tle, tle.config);
}
// initEditors(tle, tle.config);
// initFileSkippers(tle, tle.config);
//}
//defaultConfig.UpdateProfiles(trackLists[0]);
//trackLists[0].config = defaultConfig;
@ -266,40 +265,66 @@ static partial class Program
}
static void PrepareListEntry(Config config, TrackListEntry tle, bool isFirstEntry)
static void PrepareListEntry(Config prevConfig, TrackListEntry tle)
{
tle.config = prevConfig.Copy();
tle.config.UpdateProfiles(tle);
if (tle.extractorCond != null)
{
tle.config.necessaryCond = tle.config.necessaryCond.With(tle.extractorCond);
tle.extractorCond = null;
}
if (tle.extractorPrefCond != null)
{
tle.config.preferredCond = tle.config.preferredCond.With(tle.extractorPrefCond);
tle.extractorPrefCond = null;
}
InitEditors(tle, tle.config);
InitFileSkippers(tle, tle.config);
string m3uPath, indexPath;
if (config.m3uFilePath.Length > 0)
m3uPath = config.m3uFilePath;
if (tle.config.m3uFilePath.Length > 0)
m3uPath = tle.config.m3uFilePath;
else
m3uPath = Path.Join(config.parentDir, tle.defaultFolderName, "_playlist.m3u8");
m3uPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_playlist.m3u8");
if (config.indexFilePath.Length > 0)
indexPath = config.indexFilePath;
if (tle.config.indexFilePath.Length > 0)
indexPath = tle.config.indexFilePath;
else
indexPath = Path.Join(config.parentDir, tle.defaultFolderName, "_index.sldl");
indexPath = Path.Join(tle.config.parentDir, tle.defaultFolderName, "_index.sldl");
if (config.writePlaylist)
if (tle.config.writePlaylist)
tle.playlistEditor?.SetPathAndLoad(m3uPath);
if (config.writeIndex)
if (tle.config.writeIndex)
tle.indexEditor?.SetPathAndLoad(indexPath);
PreprocessTracks(config, tle);
PreprocessTracks(tle.config, tle);
}
static async Task MainLoop()
static async Task MainLoop(Config defaultConfig)
{
if (trackLists.Count == 0) return;
PrepareListEntry(defaultConfig, trackLists[0]);
var firstConfig = trackLists.lists[0].config;
bool enableParallelSearch = firstConfig.parallelAlbumSearch && !firstConfig.PrintResults && !firstConfig.PrintTracks && trackLists.lists.Any(x => x.CanParallelSearch);
var parallelSearches = new List<(TrackListEntry tle, Task<(bool, ResponseData)> task)>();
var parallelSearchSemaphore = new SemaphoreSlim(firstConfig.parallelAlbumSearchProcesses);
for (int i = 0; i < trackLists.lists.Count; i++)
{
Console.WriteLine();
if (!enableParallelSearch) Console.WriteLine();
if (i > 0) PrepareListEntry(trackLists[i-1].config, trackLists[i]);
var tle = trackLists[i];
var config = tle.config;
PrepareListEntry(config, tle, isFirstEntry: i == 0);
var existing = new List<Track>();
var notFound = new List<Track>();
@ -361,59 +386,87 @@ static partial class Program
{
await InitClientAndUpdateIfNeeded(config);
Console.WriteLine($"{tle.source.Type} download: {tle.source.ToString(true)}, searching..");
ProgressBar? progress = null;
bool foundSomething = false;
var responseData = new ResponseData();
async Task<(bool, ResponseData)> sourceSearch()
{
await parallelSearchSemaphore.WaitAsync();
if (tle.source.Type == TrackType.Album)
{
tle.list = await Search.GetAlbumDownloads(tle.source, responseData, config);
foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0;
}
else if (tle.source.Type == TrackType.Aggregate)
{
tle.list.Insert(0, await Search.GetAggregateTracks(tle.source, responseData, config));
foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0;
}
else if (tle.source.Type == TrackType.AlbumAggregate)
{
var res = await Search.GetAggregateAlbums(tle.source, responseData, config);
progress = enableParallelSearch ? Printing.GetProgressBar(config) : null;
Printing.RefreshOrPrint(progress, 0, $"{tle.source.Type} download: {tle.source.ToString(true)}, searching..", print: true);
foreach (var item in res)
bool foundSomething = false;
var responseData = new ResponseData();
if (tle.source.Type == TrackType.Album)
{
var newSource = new Track(tle.source) { Type = TrackType.Album };
var albumTle = new TrackListEntry(item, newSource, needSourceSearch: false, sourceCanBeSkipped: true);
albumTle.defaultFolderName = tle.defaultFolderName;
trackLists.AddEntry(albumTle);
tle.list = await Search.GetAlbumDownloads(tle.source, responseData, config);
foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0;
}
else if (tle.source.Type == TrackType.Aggregate)
{
tle.list.Insert(0, await Search.GetAggregateTracks(tle.source, responseData, config));
foundSomething = tle.list.Count > 0 && tle.list[0].Count > 0;
}
else if (tle.source.Type == TrackType.AlbumAggregate)
{
var res = await Search.GetAggregateAlbums(tle.source, responseData, config);
foreach (var item in res)
{
var newSource = new Track(tle.source) { Type = TrackType.Album };
var albumTle = new TrackListEntry(item, newSource, needSourceSearch: false, sourceCanBeSkipped: true);
albumTle.defaultFolderName = tle.defaultFolderName;
trackLists.AddEntry(albumTle);
}
foundSomething = res.Count > 0;
}
foundSomething = res.Count > 0;
}
tle.needSourceSearch = false;
if (!foundSomething)
{
var lockedFiles = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : "";
Console.WriteLine($"No results.{lockedFiles}");
if (!config.PrintResults)
if (!foundSomething)
{
tle.source.State = TrackState.Failed;
tle.source.FailureReason = FailureReason.NoSuitableFileFound;
tle.indexEditor?.Update();
var lockedFiles = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : "";
var str = progress != null ? $"{tle.source}: " : "";
Printing.RefreshOrPrint(progress, 0, $"{str}No results.{lockedFiles}", true);
}
continue;
parallelSearchSemaphore.Release();
return (foundSomething, responseData);
}
if (config.skipExisting && tle.needSkipExistingAfterSearch)
if (!enableParallelSearch || !tle.CanParallelSearch)
{
foreach (var tracks in tle.list)
existing.AddRange(DoSkipExisting(tle, config, tracks));
}
(bool foundSomething, ResponseData responseData) = await sourceSearch();
if (tle.gotoNextAfterSearch)
if (!foundSomething)
{
if (!config.PrintResults)
{
tle.source.State = TrackState.Failed;
tle.source.FailureReason = FailureReason.NoSuitableFileFound;
tle.indexEditor?.Update();
}
continue;
}
if (config.skipExisting && tle.needSkipExistingAfterSearch)
{
foreach (var tracks in tle.list)
existing.AddRange(DoSkipExisting(tle, config, tracks));
}
if (tle.gotoNextAfterSearch)
{
continue;
}
}
else
{
parallelSearches.Add((tle, sourceSearch()));
continue;
}
}
@ -424,6 +477,29 @@ static partial class Program
continue;
}
if (parallelSearches.Count > 0 && !tle.CanParallelSearch)
{
await parallelDownloads();
}
if (!enableParallelSearch || !tle.CanParallelSearch)
{
await download(tle, config, notFound, existing);
}
}
if (parallelSearches.Count > 0)
{
await parallelDownloads();
}
if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any()))
{
PrintComplete(trackLists);
}
async Task download(TrackListEntry tle, Config config, List<Track>? notFound, List<Track>? existing)
{
tle.indexEditor?.Update();
tle.playlistEditor?.Update();
@ -432,9 +508,9 @@ static partial class Program
PrintTracksTbd(tle.list[0].Where(t => t.State == TrackState.Initial).ToList(), existing, notFound, tle.source.Type, config);
}
if (notFound.Count + existing.Count >= tle.list.Sum(x => x.Count))
if (notFound != null && existing != null && notFound.Count + existing.Count >= tle.list.Sum(x => x.Count))
{
continue;
return;
}
await InitClientAndUpdateIfNeeded(config);
@ -453,9 +529,38 @@ static partial class Program
}
}
if (!trackLists[^1].config.DoNotDownload && (trackLists.lists.Count > 0 || trackLists.Flattened(false, false).Skip(1).Any()))
async Task parallelDownloads()
{
PrintComplete(trackLists);
await Task.WhenAll(parallelSearches.Select(x => x.task));
Console.WriteLine();
foreach (var (tle, task) in parallelSearches)
{
(bool foundSomething, var responseData) = task.Result;
if (foundSomething)
{
Console.WriteLine($"Downloading: {tle.source}");
await download(tle, tle.config, null, null);
}
else
{
if (!tle.config.PrintResults)
{
tle.source.State = TrackState.Failed;
tle.source.FailureReason = FailureReason.NoSuitableFileFound;
tle.indexEditor?.Update();
}
if (tle.config.skipExisting && tle.needSkipExistingAfterSearch)
{
foreach (var tracks in tle.list)
DoSkipExisting(tle, tle.config, tracks);
}
}
}
parallelSearches.Clear();
}
}
@ -749,6 +854,11 @@ static partial class Program
Console.WriteLine("No images found");
return downloadedImages;
}
else if (!albumArts.Skip(1).Any() && albumArts.First().All(y => y.State != TrackState.Initial))
{
Console.WriteLine("No additional images found");
return downloadedImages;
}
if (option == AlbumArtOption.Largest)
{

13
slsk-batchdl/Search.cs
View file

@ -102,7 +102,7 @@ static class Search
},
fileFilter: (file) =>
{
return Utils.IsMusicFile(file.Filename) && necCond.FileSatisfies(file, track, null);
return necCond.FileSatisfies(file, track, null);
});
}
@ -214,7 +214,7 @@ static class Search
try
{
Printing.RefreshOrPrint(progress, 0, $"yt-dlp search: {track}", true);
var ytResults = await Extractors.YouTube.YtdlpSearch(track);
var ytResults = await Extractors.YouTube.YtdlpSearch(track, printCommand: config.debugInfo);
if (ytResults.Count > 0)
{
@ -224,8 +224,8 @@ static class Search
{
string saveFilePathNoExt = organizer.GetSavePathNoExt(title);
downloading = 1;
Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}", true);
saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument);
Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}, filename: {saveFilePathNoExt}", true);
saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, config.ytdlpArgument, printCommand: config.debugInfo);
Printing.RefreshOrPrint(progress, 100, $"Succeded: yt-dlp completed download for {track}", true);
break;
}
@ -418,7 +418,7 @@ static class Search
},
fileFilter: (file) =>
{
return Utils.IsMusicFile(file.Filename) && nec.FileSatisfies(file, track, null);
return nec.FileSatisfies(file, track, null);
}
);
void handler(SlResponse r)
@ -678,6 +678,7 @@ static class Search
bool albumMode = false)
{
bool useBracketCheck = true;
if (albumMode)
{
useBracketCheck = false;
@ -881,7 +882,7 @@ static class Search
},
fileFilter: (file) =>
{
return Utils.IsMusicFile(file.Filename) && (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull);
return (necCond.FileSatisfies(file, track, null) || config.PrintResultsFull);
});
}

7
slsk-batchdl/Utilities/Utils.cs
View file

@ -634,6 +634,13 @@ public static class Utils
return path.Replace('\\', '/').TrimEnd('/').Trim();
}
public static bool IsInDirectory(string path, string dir, bool strict)
{
path = NormalizedPath(path);
dir = NormalizedPath(dir);
return strict ? path.StartsWith(dir + '/') : path.StartsWith(dir);
}
public static bool SequenceEqualUpToPermutation<T>(this IEnumerable<T> list1, IEnumerable<T> list2)
{
var cnt = new Dictionary<T, int>();