mirror/slsk-batchdl
1
0
Fork 0
mirror of https://github.com/fiso64/slsk-batchdl.git synced 2024-12-22 14:32:40 +00:00
Code Issues Activity

commit

Browse source
This commit is contained in:
fiso64 2024-08-30 10:57:18 +02:00
parent 4e3ed2ec46
commit 82591b6569
15 changed files with 1428 additions and 1282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
README.md
View file

@ -191,7 +191,6 @@ Usage: sldl <input> [OPTIONS]
'default': No additional images
'largest': Download from the folder with the largest image
'most': Download from the folder containing the most images
'most-largest': Do most, then largest
--album-art-only Only download album art for the provided album
--no-browse-folder Do not automatically browse user shares to get all files in
in the folder
@ -250,7 +249,7 @@ The id and secret can be obtained at https://developer.spotify.com/dashboard/app
Create an app and add http://localhost:48721/callback as a redirect url in its settings.
### Bandcamp
A bandcamp url: Download a single track, and album, or an artist's entire discography.
A bandcamp url: Download a single track, an album, or an artist's entire discography.
Extracts the artist name, album name and sets --album-track-count="n+", where n is the
number of visible tracks on the bandcamp page.
@ -296,8 +295,7 @@ Two files are considered equal if their inferred track title and artist name are
(ignoring case and some special characters), and their lengths are within --length-tol of each
other.
Note that this mode is not 100% reliable, which is why --min-shares-aggregate is set to 2 by
default, i.e. any song that is shared only once will be ignored. Enable --relax-filtering to
make the file filtering less aggressive.
default, i.e. any song that is shared only once will be ignored.
### Album Aggregate
Activated when --album and --aggregate are enabled, in this mode sldl searches for the query
@ -356,14 +354,14 @@ files if available, and only download lossy files if there's nothing else.
There are no default required conditions. The default preferred conditions are:
```
format = mp3
length-tol = 3
min-bitrate = 200
max-bitrate = 2500
max-samplerate = 48000
strict-title = true
strict-album = true
accept-no-length = false
pref-format = mp3
pref-length-tol = 3
pref-min-bitrate = 200
pref-max-bitrate = 2500
pref-max-samplerate = 48000
pref-strict-title = true
pref-strict-album = true
pref-accept-no-length = false
```
sldl will therefore prefer mp3 files with bitrate between 200 and 2500 kbps, and whose length
differs from the supplied length by no more than 3 seconds. It will also prefer files whose
@ -419,7 +417,7 @@ year Track year or date
track Track number
disc Disc number
filename Soulseek filename without extension
foldername Soulseek folder name (only available for album downloads)
foldername Soulseek folder name
default-foldername Default sldl folder name
extractor Name of the extractor used (CSV/Spotify/YouTube/etc)
```
@ -494,11 +492,11 @@ profile-cond = input-type == "youtube"
path = ~/downloads/sldl-youtube
# download to another location for youtube
```
The following operators are supported: &&, ||, ==, !=, ! (negation for bools).
The following variables are available for use in profile-cond:
The following operators are supported for use in profile-cond: &&, ||, ==, !=, !{bool}.
The following variables are available:
```
input-type ( = "youtube"|"csv"|"string"|"bandcamp"|"spotify")
download-mode ( = "normal"|"aggregate"|"album"|"album-aggregate")
input-type ("youtube"|"csv"|"string"|"bandcamp"|"spotify")
download-mode ("normal"|"aggregate"|"album"|"album-aggregate")
interactive (bool)
```
## Examples

16
slsk-batchdl/Config.cs
View file

@ -236,6 +236,9 @@ static class Config
}
}
if (albumArtOnly && albumArtOption == AlbumArtOption.Default)
albumArtOption = AlbumArtOption.Largest;
parentFolder = Utils.ExpandUser(parentFolder);
m3uFilePath = Utils.ExpandUser(m3uFilePath);
musicDir = Utils.ExpandUser(musicDir);
@ -250,6 +253,7 @@ static class Config
folderName = folderName.Replace('/', Path.DirectorySeparatorChar);
outputFolder = Path.Join(parentFolder, folderName);
nameFormat = nameFormat.Trim();
if (m3uFilePath.Length == 0)
m3uFilePath = Path.Join(outputFolder, (folderName.Length == 0 ? "playlist" : folderName) + ".m3u8");
@ -319,7 +323,7 @@ static class Config
if (newProfiles.Count > 0)
{
//appliedProfiles.Clear();
appliedProfiles.Union(newProfiles);
appliedProfiles.UnionWith(newProfiles);
ApplyProfile(profile);
ProcessArgs(arguments);
PostProcessArgs();
@ -551,6 +555,7 @@ static class Config
cond.AcceptNoLength = bool.Parse(value);
break;
case "strict":
case "strictconditions":
case "acceptmissing":
case "acceptmissingprops":
cond.AcceptMissingProps = bool.Parse(value);
@ -908,7 +913,6 @@ static class Config
"default" => AlbumArtOption.Default,
"largest" => AlbumArtOption.Largest,
"most" => AlbumArtOption.Most,
"most-largest" => AlbumArtOption.MostLargest,
_ => throw new ArgumentException($"Invalid album art download mode '{args[i]}'"),
};
break;
@ -977,6 +981,10 @@ static class Config
case "--pref-strict-album":
setFlag(ref preferredCond.StrictAlbum, ref i);
break;
case "--panl":
case "--pref-accept-no-length":
setFlag(ref preferredCond.AcceptNoLength, ref i);
break;
case "--pbu":
case "--pref-banned-users":
preferredCond.BannedUsers = args[++i].Split(',');
@ -1031,6 +1039,10 @@ static class Config
case "--banned-users":
necessaryCond.BannedUsers = args[++i].Split(',');
break;
case "--anl":
case "--accept-no-length":
setFlag(ref necessaryCond.AcceptNoLength, ref i);
break;
case "--c":
case "--cond":
case "--conditions":

7
slsk-batchdl/Data.cs
View file

@ -25,6 +25,7 @@ namespace Data
public bool OutputsDirectory => Type != TrackType.Normal;
public Soulseek.File? FirstDownload => Downloads?.FirstOrDefault().Item2;
public SearchResponse? FirstResponse => Downloads?.FirstOrDefault().Item1;
public string? FirstUsername => Downloads?.FirstOrDefault().Item1?.Username;
public Track() { }
@ -106,7 +107,6 @@ namespace Data
public bool needSkipExistingAfterSearch = false;
public bool gotoNextAfterSearch = false;
public bool placeInSubdir = false;
public bool useRemoteDirname = false;
public TrackListEntry()
{
@ -141,14 +141,13 @@ namespace Data
}
public TrackListEntry(List<List<Track>> list, Track source, bool needSearch, bool placeInSubdir,
bool useRemoteDirname, bool canBeSkipped, bool needSkipExistingAfterSearch, bool gotoNextAfterSearch)
bool sourceCanBeSkipped, bool needSkipExistingAfterSearch, bool gotoNextAfterSearch)
{
this.list = list;
this.source = source;
this.needSourceSearch = needSearch;
this.placeInSubdir = placeInSubdir;
this.useRemoteDirname = useRemoteDirname;
this.sourceCanBeSkipped = canBeSkipped;
this.sourceCanBeSkipped = sourceCanBeSkipped;
this.needSkipExistingAfterSearch = needSkipExistingAfterSearch;
this.gotoNextAfterSearch = gotoNextAfterSearch;
}

221
slsk-batchdl/Download.cs Normal file
View file

@ -0,0 +1,221 @@
using Soulseek;
using System.Collections.Concurrent;
using System.Text.RegularExpressions;
using Data;
using Enums;
using static Program;
using File = System.IO.File;
using Directory = System.IO.Directory;
using ProgressBar = Konsole.ProgressBar;
using SearchResponse = Soulseek.SearchResponse;
using SlResponse = Soulseek.SearchResponse;
using SlFile = Soulseek.File;
using SlDictionary = System.Collections.Concurrent.ConcurrentDictionary<string, (Soulseek.SearchResponse, Soulseek.File)>;
static class Download
{
public static async Task DownloadFile(SearchResponse response, Soulseek.File file, string filePath, Track track, ProgressBar progress, CancellationTokenSource? searchCts = null)
{
if (Config.DoNotDownload)
throw new Exception();
await Program.WaitForLogin();
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
string origPath = filePath;
filePath += ".incomplete";
var transferOptions = new TransferOptions(
stateChanged: (state) =>
{
if (Program.downloads.TryGetValue(file.Filename, out var x))
x.transfer = state.Transfer;
},
progressUpdated: (progress) =>
{
if (downloads.TryGetValue(file.Filename, out var x))
x.bytesTransferred = progress.PreviousBytesTransferred;
}
);
try
{
using var cts = new CancellationTokenSource();
using var outputStream = new FileStream(filePath, FileMode.Create);
var wrapper = new DownloadWrapper(origPath, response, file, track, cts, progress);
downloads.TryAdd(file.Filename, wrapper);
// Attempt to make it resume downloads after a network interruption.
// Does not work: The resumed download will be queued until it goes stale.
// The host (slskd) reports that "Another upload to {user} is already in progress"
// when attempting to resume. Must wait until timeout, which can take minutes.
int maxRetries = 3;
int retryCount = 0;
while (true)
{
try
{
await client.DownloadAsync(response.Username, file.Filename,
() => Task.FromResult((Stream)outputStream),
file.Size, startOffset: outputStream.Position,
options: transferOptions, cancellationToken: cts.Token);
break;
}
catch (SoulseekClientException)
{
retryCount++;
if (retryCount >= maxRetries || IsConnectedAndLoggedIn())
throw;
await WaitForLogin();
}
}
}
catch
{
if (File.Exists(filePath))
try { File.Delete(filePath); } catch { }
downloads.TryRemove(file.Filename, out var d);
if (d != null)
lock (d) { d.UpdateText(); }
throw;
}
try { searchCts?.Cancel(); }
catch { }
try { Utils.Move(filePath, origPath); }
catch (IOException) { Printing.WriteLine($"Failed to rename .incomplete file", ConsoleColor.DarkYellow, true); }
downloads.TryRemove(file.Filename, out var x);
if (x != null)
{
lock (x)
{
x.success = true;
x.UpdateText();
}
}
}
}
public class DownloadWrapper
{
public string savePath;
public string displayText = "";
public int downloadRotatingBarState = 0;
public Soulseek.File file;
public Transfer? transfer;
public SearchResponse response;
public ProgressBar progress;
public Track track;
public long bytesTransferred = 0;
public bool stalled = false;
public bool queued = false;
public bool success = false;
public CancellationTokenSource cts;
public DateTime startTime = DateTime.Now;
public DateTime lastChangeTime = DateTime.Now;
TransferStates? prevTransferState = null;
long prevBytesTransferred = 0;
bool updatedTextDownload = false;
bool updatedTextSuccess = false;
readonly char[] bars = { '|', '/', '—', '\\' };
public DownloadWrapper(string savePath, SearchResponse response, Soulseek.File file, Track track, CancellationTokenSource cts, ProgressBar progress)
{
this.savePath = savePath;
this.response = response;
this.file = file;
this.cts = cts;
this.track = track;
this.progress = progress;
this.displayText = Printing.DisplayString(track, file, response);
Printing.RefreshOrPrint(progress, 0, "Initialize: " + displayText, true);
Printing.RefreshOrPrint(progress, 0, displayText, false);
}
public void UpdateText()
{
downloadRotatingBarState++;
downloadRotatingBarState %= bars.Length;
float? percentage = bytesTransferred / (float)file.Size;
queued = (transfer?.State & TransferStates.Queued) != 0;
string bar;
string state;
bool downloading = false;
if (stalled)
{
state = "Stalled";
bar = "";
}
else if (transfer != null)
{
if (queued)
state = "Queued";
else if ((transfer.State & TransferStates.Initializing) != 0)
state = "Initialize";
else if ((transfer.State & TransferStates.Completed) != 0)
{
var flag = transfer.State & (TransferStates.Succeeded | TransferStates.Cancelled
| TransferStates.TimedOut | TransferStates.Errored | TransferStates.Rejected
| TransferStates.Aborted);
state = flag.ToString();
if (flag == TransferStates.Succeeded)
success = true;
}
else
{
state = transfer.State.ToString();
if ((transfer.State & TransferStates.InProgress) != 0)
downloading = true;
}
bar = success ? "" : bars[downloadRotatingBarState] + " ";
}
else
{
state = "NullState";
bar = "";
}
string txt = $"{bar}{state}:".PadRight(14) + $" {displayText}";
bool needSimplePrintUpdate = (downloading && !updatedTextDownload) || (success && !updatedTextSuccess);
updatedTextDownload |= downloading;
updatedTextSuccess |= success;
Console.ResetColor();
Printing.RefreshOrPrint(progress, (int)((percentage ?? 0) * 100), txt, needSimplePrintUpdate, needSimplePrintUpdate);
}
public DateTime UpdateLastChangeTime(bool updateAllFromThisUser = true, bool forceChanged = false)
{
bool changed = prevTransferState != transfer?.State || prevBytesTransferred != bytesTransferred;
if (changed || forceChanged)
{
lastChangeTime = DateTime.Now;
stalled = false;
if (updateAllFromThisUser)
{
foreach (var (_, dl) in downloads)
{
if (dl != this && dl.response.Username == response.Username)
dl.UpdateLastChangeTime(updateAllFromThisUser: false, forceChanged: true);
}
}
}
prevTransferState = transfer?.State;
prevBytesTransferred = bytesTransferred;
return lastChangeTime;
}
}

1
slsk-batchdl/Enums.cs
View file

@ -68,7 +68,6 @@ namespace Enums
Default,
Most,
Largest,
MostLargest,
}
public enum DisplayMode

2
slsk-batchdl/Extractors/Csv.cs
View file

@ -166,7 +166,7 @@ namespace Extractors
}
catch
{
Program.WriteLine($"Couldn't parse track length \"{values[lengthIndex]}\" with format \"{timeUnit}\" for \"{track}\"", ConsoleColor.DarkYellow);
Printing.WriteLine($"Couldn't parse track length \"{values[lengthIndex]}\" with format \"{timeUnit}\" for \"{track}\"", ConsoleColor.DarkYellow);
}
}

271
slsk-batchdl/FileManager.cs Normal file
View file

@ -0,0 +1,271 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using Data;
using Enums;
public class FileManager
{
readonly TrackListEntry? tle;
readonly HashSet<Track> organized = new();
string? remoteCommonDir;
public FileManager() { }
public FileManager(TrackListEntry tle)
{
this.tle = tle;
}
public string GetSavePath(string sourceFname)
{
return $"{GetSavePathNoExt(sourceFname)}{Path.GetExtension(sourceFname)}";
}
public string GetSavePathNoExt(string sourceFname)
{
string name = Utils.GetFileNameWithoutExtSlsk(sourceFname);
name = name.ReplaceInvalidChars(Config.invalidReplaceStr);
string parent = Config.outputFolder;
if (tle != null && tle.placeInSubdir && remoteCommonDir != null)
{
string dirname = Path.GetFileName(remoteCommonDir);
string relpath = Path.GetRelativePath(remoteCommonDir, Utils.NormalizedPath(sourceFname));
parent = Path.Join(parent, dirname, Path.GetDirectoryName(relpath));
}
return Path.Join(parent, name);
}
public void SetRemoteCommonDir(string? remoteCommonDir)
{
this.remoteCommonDir = remoteCommonDir != null ? Utils.NormalizedPath(remoteCommonDir) : null;
}
public void OrganizeAlbum(List<Track> tracks, List<Track>? additionalImages, bool remainingOnly = true)
{
if (tle == null)
throw new NullReferenceException("TrackListEntry should not be null.");
string outputFolder = Config.outputFolder;
foreach (var track in tracks.Where(t => !t.IsNotAudio))
{
if (remainingOnly && organized.Contains(track))
continue;
OrganizeAudio(tle, track, track.FirstDownload);
}
bool onlyAdditionalImages = Config.nameFormat.Length == 0;
var nonAudioToOrganize = onlyAdditionalImages ? additionalImages : tracks.Where(t => t.IsNotAudio);
if (nonAudioToOrganize == null || !nonAudioToOrganize.Any())
return;
string parent = Utils.GreatestCommonDirectory(
tracks.Where(t => !t.IsNotAudio && t.State == TrackState.Downloaded && t.DownloadPath.Length > 0).Select(t => t.DownloadPath));
foreach (var track in nonAudioToOrganize)
{
if (remainingOnly && organized.Contains(track))
continue;
OrganizeNonAudio(track, parent);
}
}
public void OrganizeAudio(TrackListEntry tle, Track track, Soulseek.File? file)
{
if (track.DownloadPath.Length == 0 || !Utils.IsMusicFile(track.DownloadPath))
{
return;
}
else if (Config.nameFormat.Length == 0)
{
organized.Add(track);
return;
}
string pathPart = SubstituteValues(Config.nameFormat, track, file);
string newFilePath = Path.Join(Config.outputFolder, pathPart + Path.GetExtension(track.DownloadPath));
try
{
MoveAndDeleteParent(track.DownloadPath, newFilePath);
}
catch (Exception ex)
{
Printing.WriteLine($"\nFailed to move: {ex.Message}\n", ConsoleColor.DarkYellow, true);
return;
}
track.DownloadPath = newFilePath;
organized.Add(track);
}
public void OrganizeNonAudio(Track track, string parent)
{
if (track.DownloadPath.Length == 0)
return;
string newFilePath = Path.Join(parent, Path.GetFileName(track.DownloadPath));
try
{
MoveAndDeleteParent(track.DownloadPath, newFilePath);
}
catch (Exception ex)
{
Printing.WriteLine($"\nFailed to move: {ex.Message}\n", ConsoleColor.DarkYellow, true);
return;
}
track.DownloadPath = newFilePath;
organized.Add(track);
}
static void MoveAndDeleteParent(string oldPath, string newPath)
{
if (Utils.NormalizedPath(oldPath) != Utils.NormalizedPath(newPath))
{
Directory.CreateDirectory(Path.GetDirectoryName(newPath));
Utils.Move(oldPath, newPath);
string x = Utils.NormalizedPath(Path.GetFullPath(Config.outputFolder));
string y = Utils.NormalizedPath(Path.GetDirectoryName(oldPath));
while (x.Length > 0 && y.StartsWith(x + '/') && Utils.FileCountRecursive(y) == 0)
{
Directory.Delete(y, true); // hopefully this is fine
y = Utils.NormalizedPath(Path.GetDirectoryName(y));
}
}
}
string SubstituteValues(string format, Track track, Soulseek.File? slfile)
{
string newName = format;
TagLib.File? file = null;
try { file = TagLib.File.Create(track.DownloadPath); }
catch { }
var regex = new Regex(@"(\{(?:\{??[^\{]*?\}))");
var matches = regex.Matches(newName);
while (matches.Count > 0)
{
foreach (var match in matches.Cast<Match>())
{
string inner = match.Groups[1].Value;
inner = inner[1..^1];
var options = inner.Split('|');
string chosenOpt = "";
foreach (var opt in options)
{
string[] parts = Regex.Split(opt, @"\([^\)]*\)");
string[] result = parts.Where(part => !string.IsNullOrWhiteSpace(part)).ToArray();
if (result.All(x => GetVarValue(x, file, slfile, track).Length > 0))
{
chosenOpt = opt;
break;
}
}
chosenOpt = Regex.Replace(chosenOpt, @"\([^()]*\)|[^()]+", match =>
{
if (match.Value.StartsWith("(") && match.Value.EndsWith(")"))
return match.Value[1..^1].ReplaceInvalidChars(Config.invalidReplaceStr, removeSlash: false);
else
return GetVarValue(match.Value, file, slfile, track);
});
string old = match.Groups[1].Value;
old = old.StartsWith("{{") ? old[1..] : old;
newName = newName.Replace(old, chosenOpt);
}
matches = regex.Matches(newName);
}
if (newName != format)
{
char dirsep = Path.DirectorySeparatorChar;
newName = newName.Replace('/', dirsep).Replace('\\', dirsep);
var x = newName.Split(dirsep, StringSplitOptions.RemoveEmptyEntries);
newName = string.Join(dirsep, x.Select(x => x.ReplaceInvalidChars(Config.invalidReplaceStr).Trim(' ', '.')));
return newName;
}
return format;
}
string GetVarValue(string x, TagLib.File? file, Soulseek.File? slfile, Track track)
{
string res;
switch (x)
{
case "artist":
res = file?.Tag.FirstPerformer ?? ""; break;
case "artists":
res = file != null ? string.Join(" & ", file.Tag.Performers) : ""; break;
case "albumartist":
res = file?.Tag.FirstAlbumArtist ?? ""; break;
case "albumartists":
res = file != null ? string.Join(" & ", file.Tag.AlbumArtists) : ""; break;
case "title":
res = file?.Tag.Title ?? ""; break;
case "album":
res = file?.Tag.Album ?? ""; break;
case "sartist":
case "sartists":
res = track.Artist; break;
case "stitle":
res = track.Title; break;
case "salbum":
res = track.Album; break;
case "year":
res = file?.Tag.Year.ToString() ?? ""; break;
case "track":
res = file?.Tag.Track.ToString("D2") ?? ""; break;
case "disc":
res = file?.Tag.Disc.ToString() ?? ""; break;
case "filename":
res = Utils.GetFileNameWithoutExtSlsk(slfile?.Filename ?? ""); break;
case "foldername":
if (remoteCommonDir == null || slfile == null)
{
return Utils.GetBaseNameSlsk(Utils.GetDirectoryNameSlsk(slfile?.Filename ?? ""));
}
else
{
string d = Path.GetDirectoryName(Utils.NormalizedPath(slfile.Filename));
string r = Path.GetFileName(remoteCommonDir);
return Path.Join(r, Path.GetRelativePath(remoteCommonDir, d));
}
case "default-foldername":
res = Config.defaultFolderName; break;
case "extractor":
res = Config.inputType.ToString(); break;
default:
res = ""; break;
}
return res.ReplaceInvalidChars(Config.invalidReplaceStr);
}
}

44
slsk-batchdl/ExistingChecker.cs → slsk-batchdl/FileSkipper.cs
View file

@ -3,39 +3,39 @@ using Data;
using Enums;
using System.IO;
namespace ExistingCheckers
namespace FileSkippers
{
public static class ExistingCheckerRegistry
public static class FileSkipperRegistry
{
public static ExistingChecker GetChecker(SkipMode mode, string dir, FileConditions conditions, M3uEditor m3uEditor)
public static FileSkipper GetChecker(SkipMode mode, string dir, FileConditions conditions, M3uEditor m3uEditor)
{
bool noConditions = conditions.Equals(new FileConditions());
return mode switch
{
SkipMode.Name => new NameExistingChecker(dir),
SkipMode.NameCond => noConditions ? new NameExistingChecker(dir) : new NameConditionExistingChecker(dir, conditions),
SkipMode.Tag => new TagExistingChecker(dir),
SkipMode.TagCond => noConditions ? new TagExistingChecker(dir) : new TagConditionExistingChecker(dir, conditions),
SkipMode.M3u => new M3uExistingChecker(m3uEditor, false),
SkipMode.M3uCond => noConditions ? new M3uExistingChecker(m3uEditor, true) : new M3uConditionExistingChecker(m3uEditor, conditions),
SkipMode.Name => new NameSkipper(dir),
SkipMode.NameCond => noConditions ? new NameSkipper(dir) : new NameConditionalSkipper(dir, conditions),
SkipMode.Tag => new TagSkipper(dir),
SkipMode.TagCond => noConditions ? new TagSkipper(dir) : new TagConditionalSkipper(dir, conditions),
SkipMode.M3u => new M3uSkipper(m3uEditor, false),
SkipMode.M3uCond => noConditions ? new M3uSkipper(m3uEditor, true) : new M3uConditionalSkipper(m3uEditor, conditions),
};
}
}
public abstract class ExistingChecker
public abstract class FileSkipper
{
public abstract bool TrackExists(Track track, out string? foundPath);
public virtual void BuildIndex() { IndexIsBuilt = true; }
public bool IndexIsBuilt { get; protected set; } = false;
}
public class NameExistingChecker : ExistingChecker
public class NameSkipper : FileSkipper
{
readonly string[] ignore = new string[] { "_", "-", ".", "(", ")", "[", "]" };
readonly string dir;
readonly List<(string, string, string)> index = new(); // (Path, PreprocessedPath, PreprocessedName)
public NameExistingChecker(string dir)
public NameSkipper(string dir)
{
this.dir = dir;
}
@ -99,14 +99,14 @@ namespace ExistingCheckers
}
}
public class NameConditionExistingChecker : ExistingChecker
public class NameConditionalSkipper : FileSkipper
{
readonly string[] ignore = new string[] { "_", "-", ".", "(", ")", "[", "]" };
readonly string dir;
readonly List<(string, string, SimpleFile)> index = new(); // (PreprocessedPath, PreprocessedName, file)
FileConditions conditions;
public NameConditionExistingChecker(string dir, FileConditions conditions)
public NameConditionalSkipper(string dir, FileConditions conditions)
{
this.dir = dir;
this.conditions = conditions;
@ -175,12 +175,12 @@ namespace ExistingCheckers
}
}
public class TagExistingChecker : ExistingChecker
public class TagSkipper : FileSkipper
{
readonly string dir;
readonly List<(string, string, string)> index = new(); // (Path, PreprocessedArtist, PreprocessedTitle)
public TagExistingChecker(string dir)
public TagSkipper(string dir)
{
this.dir = dir;
}
@ -240,13 +240,13 @@ namespace ExistingCheckers
}
}
public class TagConditionExistingChecker : ExistingChecker
public class TagConditionalSkipper : FileSkipper
{
readonly string dir;
readonly List<(string, string, SimpleFile)> index = new(); // (PreprocessedArtist, PreprocessedTitle, file)
FileConditions conditions;
public TagConditionExistingChecker(string dir, FileConditions conditions)
public TagConditionalSkipper(string dir, FileConditions conditions)
{
this.dir = dir;
this.conditions = conditions;
@ -307,12 +307,12 @@ namespace ExistingCheckers
}
}
public class M3uExistingChecker : ExistingChecker
public class M3uSkipper : FileSkipper
{
M3uEditor m3uEditor;
bool checkFileExists;
public M3uExistingChecker(M3uEditor m3UEditor, bool checkFileExists)
public M3uSkipper(M3uEditor m3UEditor, bool checkFileExists)
{
this.m3uEditor = m3UEditor;
this.checkFileExists = checkFileExists;
@ -349,12 +349,12 @@ namespace ExistingCheckers
}
}
public class M3uConditionExistingChecker : ExistingChecker
public class M3uConditionalSkipper : FileSkipper
{
M3uEditor m3uEditor;
FileConditions conditions;
public M3uConditionExistingChecker(M3uEditor m3UEditor, FileConditions conditions)
public M3uConditionalSkipper(M3uEditor m3UEditor, FileConditions conditions)
{
this.m3uEditor = m3UEditor;
this.conditions = conditions;

30
slsk-batchdl/Help.cs
View file

@ -164,7 +164,6 @@ public static class Help
'default': No additional images
'largest': Download from the folder with the largest image
'most': Download from the folder containing the most images
'most-largest': Do most, then largest
--album-art-only Only download album art for the provided album
--no-browse-folder Do not automatically browse user shares to get all files in
in the folder
@ -221,7 +220,7 @@ public static class Help
Create an app and add http://localhost:48721/callback as a redirect url in its settings.
Bandcamp
A bandcamp url: Download a single track, and album, or an artist's entire discography.
A bandcamp url: Download a single track, an album, or an artist's entire discography.
Extracts the artist name, album name and sets --album-track-count=""n+"", where n is the
number of visible tracks on the bandcamp page.
@ -266,8 +265,7 @@ public static class Help
(ignoring case and some special characters), and their lengths are within --length-tol of each
other.
Note that this mode is not 100% reliable, which is why --min-shares-aggregate is set to 2 by
default, i.e. any song that is shared only once will be ignored. Enable --relax-filtering to
make the file filtering less aggressive.
default, i.e. any song that is shared only once will be ignored.
Album Aggregate
Activated when --album and --aggregate are enabled, in this mode sldl searches for the query
@ -328,14 +326,14 @@ public static class Help
There are no default required conditions. The default preferred conditions are:
format = mp3
length-tol = 3
min-bitrate = 200
max-bitrate = 2500
max-samplerate = 48000
strict-title = true
strict-album = true
accept-no-length = false
pref-format = mp3
pref-length-tol = 3
pref-min-bitrate = 200
pref-max-bitrate = 2500
pref-max-samplerate = 48000
pref-strict-title = true
pref-strict-album = true
pref-accept-no-length = false
sldl will therefore prefer mp3 files with bitrate between 200 and 2500 kbps, and whose length
differs from the supplied length by no more than 3 seconds. It will also prefer files whose
@ -391,7 +389,7 @@ public static class Help
track Track number
disc Disc number
filename Soulseek filename without extension
foldername Soulseek folder name (only available for album downloads)
foldername Soulseek folder name
default-foldername Default sldl folder name
extractor Name of the extractor used (CSV/Spotify/YouTube/etc)
";
@ -469,11 +467,11 @@ public static class Help
path = ~/downloads/sldl-youtube
# download to another location for youtube
The following operators are supported: &&, ||, ==, !=, ! (negation for bools).
The following operators are supported for use in profile-cond: &&, ||, ==, !=, !{bool}.
The following variables are available for use in profile-cond:
input-type ( = ""youtube""|""csv""|""string""|""bandcamp""|""spotify"")
download-mode ( = ""normal""|""aggregate""|""album""|""album-aggregate"")
input-type (""youtube""|""csv""|""string""|""bandcamp""|""spotify"")
download-mode (""normal""|""aggregate""|""album""|""album-aggregate"")
interactive (bool)
";

8
slsk-batchdl/M3uEditor.cs
View file

@ -20,7 +20,7 @@ public class M3uEditor
this.offset = offset;
this.option = option;
this.path = Path.GetFullPath(m3uPath);
this.parent = Path.GetDirectoryName(path);
this.parent = Utils.NormalizedPath(Path.GetDirectoryName(path));
this.lines = ReadAllLines().ToList();
this.needFirstUpdate = option == M3uOption.All;
@ -133,7 +133,7 @@ public class M3uEditor
return indexTrack == null
|| indexTrack.State != track.State
|| indexTrack.FailureReason != track.FailureReason
|| indexTrack.DownloadPath != track.DownloadPath;
|| Utils.NormalizedPath(indexTrack.DownloadPath) != Utils.NormalizedPath(track.DownloadPath);
}
void updateTrackIfNeeded(Track track)
@ -245,7 +245,7 @@ public class M3uEditor
foreach (var val in previousRunData.Values)
{
string p = val.DownloadPath;
if (p.StartsWith(parent))
if (Utils.NormalizedPath(p).StartsWith(parent))
p = "./" + Path.GetRelativePath(parent, p); // prepend ./ for LoadPreviousResults to recognize that a rel. path is used
var items = new string[]
@ -274,7 +274,7 @@ public class M3uEditor
failureReason = nameof(FailureReason.NoSuitableFileFound);
if (failureReason != null)
return $"# Failed: {track} [{failureReason}]";
return $"#FAIL: {track} [{failureReason}]";
if (track.DownloadPath.Length > 0)
{

362
slsk-batchdl/Printing.cs Normal file
View file

@ -0,0 +1,362 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Konsole;
using Soulseek;
using System.Text.RegularExpressions;
using Data;
using Enums;
using Directory = System.IO.Directory;
using File = System.IO.File;
using ProgressBar = Konsole.ProgressBar;
using SearchResponse = Soulseek.SearchResponse;
using SlFile = Soulseek.File;
using SlResponse = Soulseek.SearchResponse;
public static class Printing
{
static readonly object consoleLock = new();
public static string DisplayString(Track t, Soulseek.File? file = null, SearchResponse? response = null, FileConditions? nec = null,
FileConditions? pref = null, bool fullpath = false, string customPath = "", bool infoFirst = false, bool showUser = true, bool showSpeed = false)
{
if (file == null)
return t.ToString();
string sampleRate = file.SampleRate.HasValue ? $"{(file.SampleRate.Value / 1000.0).Normalize()}kHz" : "";
string bitRate = file.BitRate.HasValue ? $"{file.BitRate}kbps" : "";
string fileSize = $"{file.Size / (float)(1024 * 1024):F1}MB";
string user = showUser && response?.Username != null ? response.Username + "\\" : "";
string speed = showSpeed && response?.Username != null ? $"({response.UploadSpeed / 1024.0 / 1024.0:F2}MB/s) " : "";
string fname = fullpath ? file.Filename : (showUser ? "..\\" : "") + (customPath.Length == 0 ? Utils.GetFileNameSlsk(file.Filename) : customPath);
string length = Utils.IsMusicFile(file.Filename) ? (file.Length ?? -1).ToString() + "s" : "";
string displayText;
if (!infoFirst)
{
string info = string.Join('/', new string[] { length, sampleRate + bitRate, fileSize }.Where(value => value.Length > 0));
displayText = $"{speed}{user}{fname} [{info}]";
}
else
{
string info = string.Join('/', new string[] { length.PadRight(4), (sampleRate + bitRate).PadRight(8), fileSize.PadLeft(6) });
displayText = $"[{info}] {speed}{user}{fname}";
}
string necStr = nec != null ? $"nec:{nec.GetNotSatisfiedName(file, t, response)}, " : "";
string prefStr = pref != null ? $"prf:{pref.GetNotSatisfiedName(file, t, response)}" : "";
string cond = "";
if (nec != null || pref != null)
cond = $" ({(necStr + prefStr).TrimEnd(' ', ',')})";
return displayText + cond;
}
public static void PrintTracks(List<Track> tracks, int number = int.MaxValue, bool fullInfo = false, bool pathsOnly = false, bool showAncestors = true, bool infoFirst = false, bool showUser = true)
{
if (tracks.Count == 0)
return;
number = Math.Min(tracks.Count, number);
string ancestor = "";
if (!showAncestors)
ancestor = Utils.GreatestCommonDirectorySlsk(tracks.SelectMany(x => x.Downloads.Select(y => y.Item2.Filename)));
if (pathsOnly)
{
for (int i = 0; i < number; i++)
{
foreach (var x in tracks[i].Downloads)
{
if (ancestor.Length == 0)
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser));
else
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, "").TrimStart('\\'), infoFirst: infoFirst, showUser: showUser));
}
}
}
else if (!fullInfo)
{
for (int i = 0; i < number; i++)
{
Console.WriteLine($" {tracks[i]}");
}
}
else
{
for (int i = 0; i < number; i++)
{
if (!tracks[i].IsNotAudio)
{
Console.WriteLine($" Artist: {tracks[i].Artist}");
if (!string.IsNullOrEmpty(tracks[i].Title) || tracks[i].Type == TrackType.Normal)
Console.WriteLine($" Title: {tracks[i].Title}");
if (!string.IsNullOrEmpty(tracks[i].Album) || tracks[i].Type == TrackType.Album)
Console.WriteLine($" Album: {tracks[i].Album}");
if (tracks[i].Length > -1 || tracks[i].Type == TrackType.Normal)
Console.WriteLine($" Length: {tracks[i].Length}s");
if (!string.IsNullOrEmpty(tracks[i].DownloadPath))
Console.WriteLine($" Local path: {tracks[i].DownloadPath}");
if (!string.IsNullOrEmpty(tracks[i].URI))
Console.WriteLine($" URL/ID: {tracks[i].URI}");
if (tracks[i].Type != TrackType.Normal)
Console.WriteLine($" Type: {tracks[i].Type}");
if (!string.IsNullOrEmpty(tracks[i].Other))
Console.WriteLine($" Other: {tracks[i].Other}");
if (tracks[i].ArtistMaybeWrong)
Console.WriteLine($" Artist maybe wrong: {tracks[i].ArtistMaybeWrong}");
if (tracks[i].Downloads != null)
{
Console.WriteLine($" Shares: {tracks[i].Downloads.Count}");
foreach (var x in tracks[i].Downloads)
{
if (ancestor.Length == 0)
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser));
else
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, "").TrimStart('\\'), infoFirst: infoFirst, showUser: showUser));
}
if (tracks[i].Downloads?.Count > 0) Console.WriteLine();
}
}
else
{
Console.WriteLine($" File: {Utils.GetFileNameSlsk(tracks[i].Downloads[0].Item2.Filename)}");
Console.WriteLine($" Shares: {tracks[i].Downloads.Count}");
foreach (var x in tracks[i].Downloads)
{
if (ancestor.Length == 0)
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, infoFirst: infoFirst, showUser: showUser));
else
Console.WriteLine(" " + DisplayString(tracks[i], x.Item2, x.Item1, customPath: x.Item2.Filename.Replace(ancestor, "").TrimStart('\\'), infoFirst: infoFirst, showUser: showUser));
}
Console.WriteLine();
}
Console.WriteLine();
}
}
if (number < tracks.Count)
Console.WriteLine($" ... (etc)");
}
public static async Task PrintResults(TrackListEntry tle, List<Track> existing, List<Track> notFound)
{
await Program.InitClientAndUpdateIfNeeded();
if (tle.source.Type == TrackType.Normal)
{
await Search.SearchAndPrintResults(tle.list[0]);
}
else if (tle.source.Type == TrackType.Aggregate)
{
Console.WriteLine(new string('-', 60));
Console.WriteLine($"Results for aggregate {tle.source.ToString(true)}:");
PrintTracksTbd(tle.list[0].Where(t => t.State == TrackState.Initial).ToList(), existing, notFound, tle.source.Type);
}
else if (tle.source.Type == TrackType.Album)
{
Console.WriteLine(new string('-', 60));
if (!Config.printOption.HasFlag(PrintOption.Full))
Console.WriteLine($"Result 1 of {tle.list.Count} for album {tle.source.ToString(true)}:");
else
Console.WriteLine($"Results ({tle.list.Count}) for album {tle.source.ToString(true)}:");
if (tle.list.Count > 0 && tle.list[0].Count > 0)
{
if (!Config.noBrowseFolder)
Console.WriteLine("[Skipping full folder retrieval]");
foreach (var ls in tle.list)
{
PrintAlbum(ls);
if (!Config.printOption.HasFlag(PrintOption.Full))
break;
}
}
else
{
Console.WriteLine("No results.");
}
}
}
public static void PrintComplete(TrackLists trackLists)
{
var ls = trackLists.Flattened(true, true);
int successes = 0, fails = 0;
foreach (var x in ls)
{
if (x.State == TrackState.Downloaded)
successes++;
else if (x.State == TrackState.Failed)
fails++;
}
if (successes + fails > 1)
Console.WriteLine($"\nCompleted: {successes} succeeded, {fails} failed.");
}
public static void PrintTracksTbd(List<Track> toBeDownloaded, List<Track> existing, List<Track> notFound, TrackType type, bool summary = true)
{
if (type == TrackType.Normal && !Config.PrintTracks && toBeDownloaded.Count == 1 && existing.Count + notFound.Count == 0)
return;
string notFoundLastTime = notFound.Count > 0 ? $"{notFound.Count} not found" : "";
string alreadyExist = existing.Count > 0 ? $"{existing.Count} already exist" : "";
notFoundLastTime = alreadyExist.Length > 0 && notFoundLastTime.Length > 0 ? ", " + notFoundLastTime : notFoundLastTime;
string skippedTracks = alreadyExist.Length + notFoundLastTime.Length > 0 ? $" ({alreadyExist}{notFoundLastTime})" : "";
bool full = Config.printOption.HasFlag(PrintOption.Full);
bool allSkipped = existing.Count + notFound.Count > toBeDownloaded.Count;
if (summary && (type == TrackType.Normal || skippedTracks.Length > 0))
Console.WriteLine($"Downloading {toBeDownloaded.Count(x => !x.IsNotAudio)} tracks{skippedTracks}{(allSkipped ? '.' : ':')}");
if (toBeDownloaded.Count > 0)
{
bool showAll = type != TrackType.Normal || Config.PrintTracks || Config.PrintResults;
PrintTracks(toBeDownloaded, showAll ? int.MaxValue : 10, full, infoFirst: Config.PrintTracks);
if (full && (existing.Count > 0 || notFound.Count > 0))
Console.WriteLine("\n-----------------------------------------------\n");
}
if (Config.PrintTracks || Config.PrintResults)
{
if (existing.Count > 0)
{
Console.WriteLine($"\nThe following tracks already exist:");
PrintTracks(existing, fullInfo: full, infoFirst: Config.PrintTracks);
}
if (notFound.Count > 0)
{
Console.WriteLine($"\nThe following tracks were not found during a prior run:");
PrintTracks(notFound, fullInfo: full, infoFirst: Config.PrintTracks);
}
}
Console.WriteLine();
}
public static void PrintAlbum(List<Track> albumTracks)
{
if (albumTracks.Count == 0 && albumTracks[0].Downloads.Count == 0)
return;
var response = albumTracks[0].FirstResponse;
string userInfo = $"{response.Username} ({((float)response.UploadSpeed / (1024 * 1024)):F3}MB/s)";
var (parents, props) = FolderInfo(albumTracks.Select(x => x.FirstDownload));
WriteLine($"User : {userInfo}\nFolder: {parents}\nProps : {props}", ConsoleColor.White);
PrintTracks(albumTracks.ToList(), pathsOnly: true, showAncestors: false, showUser: false);
Console.WriteLine();
}
static (string parents, string props) FolderInfo(IEnumerable<SlFile> files)
{
string res = "";
int totalLengthInSeconds = files.Sum(f => f.Length ?? 0);
var sampleRates = files.Where(f => f.SampleRate.HasValue).Select(f => f.SampleRate.Value).OrderBy(r => r).ToList();
int? modeSampleRate = sampleRates.GroupBy(rate => rate).OrderByDescending(g => g.Count()).Select(g => (int?)g.Key).FirstOrDefault();
var bitRates = files.Where(f => f.BitRate.HasValue).Select(f => f.BitRate.Value).ToList();
double? meanBitrate = bitRates.Count > 0 ? (double?)bitRates.Average() : null;
double totalFileSizeInMB = files.Sum(f => f.Size) / (1024.0 * 1024.0);
TimeSpan totalTimeSpan = TimeSpan.FromSeconds(totalLengthInSeconds);
string totalLengthFormatted;
if (totalTimeSpan.TotalHours >= 1)
totalLengthFormatted = string.Format("{0}:{1:D2}:{2:D2}", (int)totalTimeSpan.TotalHours, totalTimeSpan.Minutes, totalTimeSpan.Seconds);
else
totalLengthFormatted = string.Format("{0:D2}:{1:D2}", totalTimeSpan.Minutes, totalTimeSpan.Seconds);
var mostCommonExtension = files.GroupBy(f => Utils.GetExtensionSlsk(f.Filename))
.OrderByDescending(g => Utils.IsMusicExtension(g.Key)).ThenByDescending(g => g.Count()).First().Key.TrimStart('.');
res = $"[{mostCommonExtension.ToUpper()} / {totalLengthFormatted}";
if (modeSampleRate.HasValue)
res += $" / {(modeSampleRate.Value / 1000.0).Normalize()} kHz";
if (meanBitrate.HasValue)
res += $" / {(int)meanBitrate.Value} kbps";
res += $" / {totalFileSizeInMB:F2} MB]";
string gcp = Utils.GreatestCommonDirectory(files.Select(x => x.Filename)).TrimEnd('\\');
var discPattern = new Regex(@"^(?i)(dis[c|k]|cd)\s*\d{1,2}$");
int lastIndex = gcp.LastIndexOf('\\');
if (lastIndex != -1)
{
int secondLastIndex = gcp.LastIndexOf('\\', lastIndex - 1);
gcp = secondLastIndex == -1 ? gcp[(lastIndex + 1)..] : gcp[(secondLastIndex + 1)..];
}
return (gcp, res);
}
public static void RefreshOrPrint(ProgressBar? progress, int current, string item, bool print = false, bool refreshIfOffscreen = false)
{
if (progress != null && !Console.IsOutputRedirected && (refreshIfOffscreen || progress.Y >= Console.WindowTop))
{
try { progress.Refresh(current, item); }
catch { }
}
else if ((Config.displayMode == DisplayMode.Simple || Console.IsOutputRedirected) && print)
Console.WriteLine(item);
}
public static void WriteLine(string value, ConsoleColor color = ConsoleColor.Gray, bool safe = false, bool debugOnly = false)
{
if (debugOnly && !Config.debugInfo)
return;
if (!safe)
{
Console.ForegroundColor = color;
Console.WriteLine(value);
Console.ResetColor();
}
else
{
Program.skipUpdate = true;
lock (consoleLock)
{
Console.ForegroundColor = color;
Console.WriteLine(value);
Console.ResetColor();
}
Program.skipUpdate = false;
}
}
public static ProgressBar? GetProgressBar(DisplayMode style)
{
lock (consoleLock)
{
ProgressBar? progress = null;
if (style == DisplayMode.Double)
progress = new ProgressBar(PbStyle.DoubleLine, 100, Console.WindowWidth - 40, character: '―');
else if (style != DisplayMode.Simple)
progress = new ProgressBar(PbStyle.SingleLine, 100, Console.WindowWidth - 10, character: ' ');
return progress;
}
}
}

1257
slsk-batchdl/Program.cs
View file

File diff suppressed because it is too large Load diff

330
slsk-batchdl/SearchAndDownload.cs → slsk-batchdl/Search.cs
View file

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using Data;
using Enums;
using static Program;
using File = System.IO.File;
using Directory = System.IO.Directory;
@ -14,28 +15,31 @@ using SlFile = Soulseek.File;
using SlDictionary = System.Collections.Concurrent.ConcurrentDictionary<string, (Soulseek.SearchResponse, Soulseek.File)>;
static partial class Program
static class Search
{
static async Task<string> SearchAndDownload(Track track, ResponseData? responseData = null)
public static RateLimitedSemaphore? searchSemaphore;
// very messy function that does everything
public static async Task<(string, SlFile?)> SearchAndDownload(Track track, FileManager organizer)
{
if (Config.DoNotDownload)
throw new Exception();
responseData ??= new ResponseData();
var responseData = new ResponseData();
IEnumerable<(SlResponse response, SlFile file)>? orderedResults = null;
var progress = GetProgressBar(Config.displayMode);
var progress = Printing.GetProgressBar(Config.displayMode);
var results = new SlDictionary();
var fsResults = new SlDictionary();
var cts = new CancellationTokenSource();
var saveFilePath = "";
SlFile? chosenFile = null;
Task? downloadTask = null;
var fsDownloadLock = new object();
int fsResultsStarted = 0;
int downloading = 0;
bool notFound = false;
bool searchEnded = false;
string fsUser = "";
string fsFile = "";
string? fsUser = null;
if (track.Downloads != null)
{
@ -43,10 +47,7 @@ static partial class Program
goto downloads;
}
RefreshOrPrint(progress, 0, $"Waiting: {track}", false);
string searchText = $"{track.Artist} {track.Title}".Trim();
var removeChars = new string[] { " ", "_", "-" };
Printing.RefreshOrPrint(progress, 0, $"Waiting: {track}", false);
searches.TryAdd(track, new SearchInfo(results, progress));
@ -58,10 +59,10 @@ static partial class Program
{
downloading = 1;
var (r, f) = fsResults.MaxBy(x => x.Value.Item1.UploadSpeed).Value;
saveFilePath = GetSavePath(f.Filename);
saveFilePath = organizer.GetSavePath(f.Filename);
fsUser = r.Username;
fsFile = f.Filename;
downloadTask = DownloadFile(r, f, saveFilePath, track, progress, cts);
chosenFile = f;
downloadTask = Download.DownloadFile(r, f, saveFilePath, track, progress, cts);
}
}
}
@ -109,7 +110,7 @@ static partial class Program
});
}
void onSearch() => RefreshOrPrint(progress, 0, $"Searching: {track}", true);
void onSearch() => Printing.RefreshOrPrint(progress, 0, $"Searching: {track}", true);
await RunSearches(track, results, getSearchOptions, responseHandler, cts.Token, onSearch);
searches.TryRemove(track, out _);
@ -133,10 +134,13 @@ static partial class Program
{
saveFilePath = "";
downloading = 0;
results.TryRemove(fsUser + "\\" + fsFile, out _);
if (chosenFile != null && fsUser != null)
{
results.TryRemove(fsUser + '\\' + chosenFile.Filename, out _);
userSuccessCount.AddOrUpdate(fsUser, -1, (k, v) => v - 1);
}
}
}
cts.Dispose();
@ -150,24 +154,27 @@ static partial class Program
int trackTries = Config.maxRetriesPerTrack;
async Task<bool> process(SlResponse response, SlFile file)
{
saveFilePath = GetSavePath(file.Filename);
saveFilePath = organizer.GetSavePath(file.Filename);
chosenFile = file;
try
{
downloading = 1;
await DownloadFile(response, file, saveFilePath, track, progress);
await Download.DownloadFile(response, file, saveFilePath, track, progress);
userSuccessCount.AddOrUpdate(response.Username, 1, (k, v) => v + 1);
return true;
}
catch (Exception e)
{
chosenFile = null;
saveFilePath = "";
downloading = 0;
if (!IsConnectedAndLoggedIn())
throw;
userSuccessCount.AddOrUpdate(response.Username, -1, (k, v) => v - 1);
if (--trackTries <= 0)
{
RefreshOrPrint(progress, 0, $"Out of download retries: {track}", true);
WriteLine("Last error was: " + e.Message, ConsoleColor.DarkYellow, true);
Printing.RefreshOrPrint(progress, 0, $"Out of download retries: {track}", true);
Printing.WriteLine("Last error was: " + e.Message, ConsoleColor.DarkYellow, true);
throw new SearchAndDownloadException(FailureReason.OutOfDownloadRetries);
}
return false;
@ -206,7 +213,7 @@ static partial class Program
notFound = false;
try
{
RefreshOrPrint(progress, 0, $"yt-dlp search: {track}", true);
Printing.RefreshOrPrint(progress, 0, $"yt-dlp search: {track}", true);
var ytResults = await Extractors.YouTube.YtdlpSearch(track);
if (ytResults.Count > 0)
@ -215,11 +222,11 @@ static partial class Program
{
if (Config.necessaryCond.LengthToleranceSatisfies(length, track.Length))
{
string saveFilePathNoExt = GetSavePathNoExt(title);
string saveFilePathNoExt = organizer.GetSavePathNoExt(title);
downloading = 1;
RefreshOrPrint(progress, 0, $"yt-dlp download: {track}", true);
Printing.RefreshOrPrint(progress, 0, $"yt-dlp download: {track}", true);
saveFilePath = await Extractors.YouTube.YtdlpDownload(id, saveFilePathNoExt, Config.ytdlpArgument);
RefreshOrPrint(progress, 100, $"Succeded: yt-dlp completed download for {track}", true);
Printing.RefreshOrPrint(progress, 100, $"Succeded: yt-dlp completed download for {track}", true);
break;
}
}
@ -229,7 +236,7 @@ static partial class Program
{
saveFilePath = "";
downloading = 0;
RefreshOrPrint(progress, 0, $"{e.Message}", true);
Printing.RefreshOrPrint(progress, 0, $"{e.Message}", true);
throw new SearchAndDownloadException(FailureReason.NoSuitableFileFound);
}
}
@ -239,24 +246,21 @@ static partial class Program
if (notFound)
{
string lockedFilesStr = responseData.lockedFilesCount > 0 ? $" (Found {responseData.lockedFilesCount} locked files)" : "";
RefreshOrPrint(progress, 0, $"Not found: {track}{lockedFilesStr}", true);
Printing.RefreshOrPrint(progress, 0, $"Not found: {track}{lockedFilesStr}", true);
throw new SearchAndDownloadException(FailureReason.NoSuitableFileFound);
}
else
{
RefreshOrPrint(progress, 0, $"All downloads failed: {track}", true);
Printing.RefreshOrPrint(progress, 0, $"All downloads failed: {track}", true);
throw new SearchAndDownloadException(FailureReason.AllDownloadsFailed);
}
}
if (Config.nameFormat.Length > 0)
saveFilePath = ApplyNamingFormat(saveFilePath, track);
return Path.GetFullPath(saveFilePath);
return (Path.GetFullPath(saveFilePath), chosenFile);
}
static async Task<List<List<Track>>> GetAlbumDownloads(Track track, ResponseData responseData)
public static async Task<List<List<Track>>> GetAlbumDownloads(Track track, ResponseData responseData)
{
var results = new ConcurrentDictionary<string, (SearchResponse, Soulseek.File)>();
SearchOptions getSearchOptions(int timeout, FileConditions nec, FileConditions prf) =>
@ -395,7 +399,7 @@ static partial class Program
}
static async Task<List<Track>> GetAggregateTracks(Track track, ResponseData responseData)
public static async Task<List<Track>> GetAggregateTracks(Track track, ResponseData responseData)
{
var results = new SlDictionary();
SearchOptions getSearchOptions(int timeout, FileConditions nec, FileConditions prf) =>
@ -455,7 +459,7 @@ static partial class Program
}
static async Task<List<List<List<Track>>>> GetAggregateAlbums(Track track, ResponseData responseData)
public static async Task<List<List<List<Track>>>> GetAggregateAlbums(Track track, ResponseData responseData)
{
int maxDiff = Config.necessaryCond.LengthTolerance;
@ -551,7 +555,7 @@ static partial class Program
}
static async Task<List<(string dir, SlFile file)>> GetAllFilesInFolder(string user, string folderPrefix)
public static async Task<List<(string dir, SlFile file)>> GetAllFilesInFolder(string user, string folderPrefix)
{
var browseOptions = new BrowseOptions();
var res = new List<(string dir, SlFile file)>();
@ -579,7 +583,7 @@ static partial class Program
}
static async Task CompleteFolder(List<Track> tracks, SearchResponse response, string folder)
public static async Task CompleteFolder(List<Track> tracks, SearchResponse response, string folder)
{
try
{
@ -610,12 +614,12 @@ static partial class Program
}
catch (Exception ex)
{
WriteLine($"Error getting complete list of files: {ex}", ConsoleColor.DarkYellow);
Printing.WriteLine($"Error getting complete list of files: {ex}", ConsoleColor.DarkYellow);
}
}
static IEnumerable<(Track, IEnumerable<(SlResponse response, SlFile file)>)> EquivalentFiles(Track track,
public static IEnumerable<(Track, IEnumerable<(SlResponse response, SlFile file)>)> EquivalentFiles(Track track,
IEnumerable<(SlResponse, SlFile)> fileResponses, int minShares = -1)
{
if (minShares == -1)
@ -645,14 +649,14 @@ static partial class Program
}
static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable<KeyValuePair<string, (SlResponse, SlFile)>> results,
public static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable<KeyValuePair<string, (SlResponse, SlFile)>> results,
Track track, bool useInfer = false, bool useLevenshtein = true, bool albumMode = false)
{
return OrderedResults(results.Select(x => x.Value), track, useInfer, useLevenshtein, albumMode);
}
static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable<(SlResponse, SlFile)> results,
public static IOrderedEnumerable<(SlResponse response, SlFile file)> OrderedResults(IEnumerable<(SlResponse, SlFile)> results,
Track track, bool useInfer = false, bool useLevenshtein = true, bool albumMode = false)
{
bool useBracketCheck = true;
@ -720,7 +724,7 @@ static partial class Program
}
static async Task RunSearches(Track track, SlDictionary results, Func<int, FileConditions, FileConditions, SearchOptions> getSearchOptions,
public static async Task RunSearches(Track track, SlDictionary results, Func<int, FileConditions, FileConditions, SearchOptions> getSearchOptions,
Action<SearchResponse> responseHandler, CancellationToken? ct = null, Action? onSearch = null)
{
bool artist = track.Artist.Length > 0;
@ -731,11 +735,11 @@ static partial class Program
var searchTasks = new List<Task>();
var defaultSearchOpts = getSearchOptions(Config.searchTimeout, Config.necessaryCond, Config.preferredCond);
searchTasks.Add(Search(search, defaultSearchOpts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch(search, defaultSearchOpts, responseHandler, ct, onSearch));
if (search.RemoveDiacriticsIfExist(out string noDiacrSearch) && !track.ArtistMaybeWrong)
{
searchTasks.Add(Search(noDiacrSearch, defaultSearchOpts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch(noDiacrSearch, defaultSearchOpts, responseHandler, ct, onSearch));
}
await Task.WhenAll(searchTasks);
@ -747,7 +751,7 @@ static partial class Program
cond.StrictTitle = infTrack.Title == track.Title;
cond.StrictArtist = false;
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{infTrack.Artist} {infTrack.Title}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{infTrack.Artist} {infTrack.Title}", opts, responseHandler, ct, onSearch));
}
if (Config.desperateSearch)
@ -764,7 +768,7 @@ static partial class Program
StrictAlbum = true
};
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{track.Artist} {track.Album}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{track.Artist} {track.Album}", opts, responseHandler, ct, onSearch));
}
if (artist && title && track.Length != -1 && Config.necessaryCond.LengthTolerance != -1)
{
@ -775,7 +779,7 @@ static partial class Program
StrictArtist = true
};
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{track.Artist} {track.Title}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{track.Artist} {track.Title}", opts, responseHandler, ct, onSearch));
}
}
@ -795,7 +799,7 @@ static partial class Program
LengthTolerance = -1
};
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{track.Album}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{track.Album}", opts, responseHandler, ct, onSearch));
}
if (track2.Title.Length > 3 && artist)
{
@ -806,7 +810,7 @@ static partial class Program
LengthTolerance = -1
};
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{track2.Title}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{track2.Title}", opts, responseHandler, ct, onSearch));
}
if (track2.Artist.Length > 3 && title)
{
@ -817,7 +821,7 @@ static partial class Program
LengthTolerance = -1
};
var opts = getSearchOptions(Math.Min(Config.searchTimeout, 5000), cond, Config.preferredCond);
searchTasks.Add(Search($"{track2.Artist}", opts, responseHandler, ct, onSearch));
searchTasks.Add(DoSearch($"{track2.Artist}", opts, responseHandler, ct, onSearch));
}
}
}
@ -826,7 +830,7 @@ static partial class Program
}
static async Task Search(string search, SearchOptions opts, Action<SearchResponse> rHandler, CancellationToken? ct = null, Action? onSearch = null)
static async Task DoSearch(string search, SearchOptions opts, Action<SearchResponse> rHandler, CancellationToken? ct = null, Action? onSearch = null)
{
await searchSemaphore.WaitAsync();
try
@ -840,7 +844,7 @@ static partial class Program
}
static async Task SearchAndPrintResults(List<Track> tracks)
public static async Task SearchAndPrintResults(List<Track> tracks)
{
foreach (var track in tracks)
{
@ -878,7 +882,7 @@ static partial class Program
if (Config.DoNotDownload && results.IsEmpty)
{
WriteLine($"No results", ConsoleColor.Yellow);
Printing.WriteLine($"No results", ConsoleColor.Yellow);
}
else
{
@ -887,12 +891,12 @@ static partial class Program
Console.WriteLine();
foreach (var (response, file) in orderedResults)
{
Console.WriteLine(DisplayString(track, file, response,
Console.WriteLine(Printing.DisplayString(track, file, response,
Config.PrintResultsFull ? Config.necessaryCond : null, Config.PrintResultsFull ? Config.preferredCond : null,
fullpath: Config.PrintResultsFull, infoFirst: true, showSpeed: Config.PrintResultsFull));
count += 1;
}
WriteLine($"Total: {count}\n", ConsoleColor.Yellow);
Printing.WriteLine($"Total: {count}\n", ConsoleColor.Yellow);
}
Console.WriteLine();
@ -944,7 +948,7 @@ static partial class Program
}
static Track InferTrack(string filename, Track defaultTrack, TrackType type = TrackType.Normal)
public static Track InferTrack(string filename, Track defaultTrack, TrackType type = TrackType.Normal)
{
var t = new Track(defaultTrack);
t.Type = type;
@ -1111,92 +1115,36 @@ static partial class Program
}
static async Task DownloadFile(SearchResponse response, Soulseek.File file, string filePath, Track track, ProgressBar progress, CancellationTokenSource? searchCts = null)
public static bool AlbumsAreSimilar(List<Track> album1, List<Track> album2, int[]? album1SortedLengths = null, int tolerance = 3)
{
if (Config.DoNotDownload)
throw new Exception();
if (album1SortedLengths != null && album1SortedLengths.Length != album2.Count(t => !t.IsNotAudio))
return false;
else if (album1.Count(t => !t.IsNotAudio) != album2.Count(t => !t.IsNotAudio))
return false;
await WaitForLogin();
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
string origPath = filePath;
filePath += ".incomplete";
if (album1SortedLengths == null)
album1SortedLengths = album1.Where(t => !t.IsNotAudio).Select(t => t.Length).OrderBy(x => x).ToArray();
var transferOptions = new TransferOptions(
stateChanged: (state) =>
var album2SortedLengths = album2.Where(t => !t.IsNotAudio).Select(t => t.Length).OrderBy(x => x).ToArray();
for (int i = 0; i < album1SortedLengths.Length; i++)
{
if (downloads.TryGetValue(file.Filename, out var x))
x.transfer = state.Transfer;
},
progressUpdated: (progress) =>
{
if (downloads.TryGetValue(file.Filename, out var x))
x.bytesTransferred = progress.PreviousBytesTransferred;
}
);
try
{
using var cts = new CancellationTokenSource();
using var outputStream = new FileStream(filePath, FileMode.Create);
var wrapper = new DownloadWrapper(origPath, response, file, track, cts, progress);
downloads.TryAdd(file.Filename, wrapper);
// Attempt to make it resume downloads after a network interruption.
// Does not work: The resumed download will be queued until it goes stale.
// The host (slskd) reports that "Another upload to {user} is already in progress"
// when attempting to resume. Must wait until timeout, which can take minutes.
int maxRetries = 3;
int retryCount = 0;
while (true)
{
try
{
await client.DownloadAsync(response.Username, file.Filename,
() => Task.FromResult((Stream)outputStream),
file.Size, startOffset: outputStream.Position,
options: transferOptions, cancellationToken: cts.Token);
break;
}
catch (SoulseekClientException)
{
retryCount++;
if (retryCount >= maxRetries || IsConnectedAndLoggedIn())
throw;
await WaitForLogin();
}
}
}
catch
{
if (File.Exists(filePath))
try { File.Delete(filePath); } catch { }
downloads.TryRemove(file.Filename, out var d);
if (d != null)
lock (d) { d.UpdateText(); }
throw;
if (Math.Abs(album1SortedLengths[i] - album2SortedLengths[i]) > tolerance)
return false;
}
try { searchCts?.Cancel(); }
catch { }
return true;
}
try { Utils.Move(filePath, origPath); }
catch (IOException) { WriteLine($"Failed to rename .incomplete file", ConsoleColor.DarkYellow, true); }
downloads.TryRemove(file.Filename, out var x);
if (x != null)
static readonly List<string> bannedTerms = new()
{
lock (x)
{
x.success = true;
x.UpdateText();
"depeche mode", "beatles", "prince revolutions", "michael jackson", "coexist", "bob dylan", "enter shikari",
"village people", "lenny kravitz", "beyonce", "beyoncé", "lady gaga", "jay z", "kanye west", "rihanna",
"adele", "kendrick lamar", "bad romance", "born this way", "weeknd", "broken hearted", "highway 61 revisited",
"west gold digger", "west good life"
};
}
}
}
public class SearchAndDownloadException : Exception
{
@ -1204,125 +1152,7 @@ static partial class Program
public SearchAndDownloadException(FailureReason reason, string text = "") : base(text) { this.reason = reason; }
}
class DownloadWrapper
{
public string savePath;
public string displayText = "";
public int downloadRotatingBarState = 0;
public Soulseek.File file;
public Transfer? transfer;
public SearchResponse response;
public ProgressBar progress;
public Track track;
public long bytesTransferred = 0;
public bool stalled = false;
public bool queued = false;
public bool success = false;
public CancellationTokenSource cts;
public DateTime startTime = DateTime.Now;
public DateTime lastChangeTime = DateTime.Now;
TransferStates? prevTransferState = null;
long prevBytesTransferred = 0;
bool updatedTextDownload = false;
bool updatedTextSuccess = false;
readonly char[] bars = { '|', '/', '—', '\\' };
public DownloadWrapper(string savePath, SearchResponse response, Soulseek.File file, Track track, CancellationTokenSource cts, ProgressBar progress)
{
this.savePath = savePath;
this.response = response;
this.file = file;
this.cts = cts;
this.track = track;
this.progress = progress;
this.displayText = DisplayString(track, file, response);
RefreshOrPrint(progress, 0, "Initialize: " + displayText, true);
RefreshOrPrint(progress, 0, displayText, false);
}
public void UpdateText()
{
downloadRotatingBarState++;
downloadRotatingBarState %= bars.Length;
float? percentage = bytesTransferred / (float)file.Size;
queued = (transfer?.State & TransferStates.Queued) != 0;
string bar;
string state;
bool downloading = false;
if (stalled)
{
state = "Stalled";
bar = "";
}
else if (transfer != null)
{
if (queued)
state = "Queued";
else if ((transfer.State & TransferStates.Initializing) != 0)
state = "Initialize";
else if ((transfer.State & TransferStates.Completed) != 0)
{
var flag = transfer.State & (TransferStates.Succeeded | TransferStates.Cancelled
| TransferStates.TimedOut | TransferStates.Errored | TransferStates.Rejected
| TransferStates.Aborted);
state = flag.ToString();
if (flag == TransferStates.Succeeded)
success = true;
}
else
{
state = transfer.State.ToString();
if ((transfer.State & TransferStates.InProgress) != 0)
downloading = true;
}
bar = success ? "" : bars[downloadRotatingBarState] + " ";
}
else
{
state = "NullState";
bar = "";
}
string txt = $"{bar}{state}:".PadRight(14) + $" {displayText}";
bool needSimplePrintUpdate = (downloading && !updatedTextDownload) || (success && !updatedTextSuccess);
updatedTextDownload |= downloading;
updatedTextSuccess |= success;
Console.ResetColor();
RefreshOrPrint(progress, (int)((percentage ?? 0) * 100), txt, needSimplePrintUpdate, needSimplePrintUpdate);
}
public DateTime UpdateLastChangeTime(bool updateAllFromThisUser = true, bool forceChanged = false)
{
bool changed = prevTransferState != transfer?.State || prevBytesTransferred != bytesTransferred;
if (changed || forceChanged)
{
lastChangeTime = DateTime.Now;
stalled = false;
if (updateAllFromThisUser)
{
foreach (var (_, dl) in downloads)
{
if (dl != this && dl.response.Username == response.Username)
dl.UpdateLastChangeTime(updateAllFromThisUser: false, forceChanged: true);
}
}
}
prevTransferState = transfer?.State;
prevBytesTransferred = bytesTransferred;
return lastChangeTime;
}
}
class SearchInfo
public class SearchInfo
{
public ConcurrentDictionary<string, (SearchResponse, Soulseek.File)> results;
public ProgressBar progress;
@ -1334,6 +1164,4 @@ static partial class Program
}
}
}

16
slsk-batchdl/Test.cs
View file

@ -1,6 +1,6 @@
using Data;
using Enums;
using ExistingCheckers;
using FileSkippers;
using System.Diagnostics;
using System.Reflection;
@ -62,12 +62,12 @@ namespace Test
"/home/user/docs/report.pdf",
"/home/user/docs/",
};
Assert(Utils.GreatestCommonPath(paths, dirsep: '/') == "/home/user/docs/");
Assert(Utils.GreatestCommonPath(new string[] { "/path/file", "" }, dirsep: '/') == "");
Assert(Utils.GreatestCommonPath(new string[] { "/path/file", "/" }, dirsep: '/') == "/");
Assert(Utils.GreatestCommonPath(new string[] { "/path/dir1", "/path/dir2" }, dirsep: '/') == "/path/");
Assert(Utils.GreatestCommonPath(new string[] { "/path/dir1", "/path/dir2" }, dirsep: '\\') == "");
Assert(Utils.GreatestCommonPath(new string[] { "dir1", "dir2" }, dirsep: '/') == "");
Assert(Utils.GreatestCommonPath(paths) == "/home/user/docs/");
Assert(Utils.GreatestCommonPath(new string[] { "/path/file", "" }) == "");
Assert(Utils.GreatestCommonPath(new string[] { "/path/file", "/" }) == "/");
Assert(Utils.GreatestCommonPath(new string[] { "/path/dir1", "/path/dir2" }) == "/path/");
Assert(Utils.GreatestCommonPath(new string[] { "/path\\dir1/blah", "/path/dir2\\blah" }) == "/path\\");
Assert(Utils.GreatestCommonPath(new string[] { "dir1", "dir2" }) == "");
// RemoveDiacritics
Assert(" Café Crème à la mode Ü".RemoveDiacritics() == " Cafe Creme a la mode U");
@ -327,7 +327,7 @@ namespace Test
Program.m3uEditor = new M3uEditor(path, trackLists, Config.m3uOption);
Program.outputExistingChecker = new M3uExistingChecker(Program.m3uEditor, false);
Program.outputDirSkipper = new M3uSkipper(Program.m3uEditor, false);
var notFound = (List<Track>)ProgramInvoke("DoSkipNotFound", new object[] { trackLists[0].list[0] });
var existing = (List<Track>)ProgramInvoke("DoSkipExisting", new object[] { trackLists[0].list[0] });

35
slsk-batchdl/Utils.cs
View file

@ -140,14 +140,14 @@ public static class Utils
return ((decimal)value) / 1.000000000000000000000000000000000m;
}
public static int GetRecursiveFileCount(string directory)
public static int FileCountRecursive(string directory)
{
if (!Directory.Exists(directory))
return 0;
int count = Directory.GetFiles(directory).Length;
foreach (string subDirectory in Directory.GetDirectories(directory))
count += GetRecursiveFileCount(subDirectory);
count += FileCountRecursive(subDirectory);
return count;
}
@ -512,23 +512,23 @@ public static class Utils
return distance[source.Length, target.Length];
}
public static string GreatestCommonPath(IEnumerable<string> paths, char dirsep)
public static string GreatestCommonPath(IEnumerable<string> paths)
{
string? path = paths.FirstOrDefault();
if (path == null || path.Length == 0)
return "";
int commonPathIndex(string path1, string path2, int maxIndex)
static int commonPathIndex(string path1, string path2, int maxIndex)
{
var minLength = Math.Min(path1.Length, Math.Min(path2.Length, maxIndex));
var commonPathLength = 0;
for (int i = 0; i < minLength; i++)
{
if (path1[i] != path2[i])
break;
if (path1[i] == dirsep)
if ((path1[i] == '/' || path1[i] == '\\') && (path2[i] == '/' || path2[i] == '\\'))
commonPathLength = i + 1;
else if (path1[i] != path2[i])
break;
}
return commonPathLength;
}
@ -541,6 +541,27 @@ public static class Utils
return path[..index];
}
public static string GreatestCommonDirectory(IEnumerable<string> paths)
{
if (paths.Skip(1).Any())
return NormalizedPath(GreatestCommonPath(paths));
else
return NormalizedPath(Path.GetDirectoryName(paths.First().TrimEnd('/').TrimEnd('\\')) ?? "");
}
public static string GreatestCommonDirectorySlsk(IEnumerable<string> paths)
{
if (paths.Skip(1).Any())
return Utils.GreatestCommonPath(paths).Replace('/', '\\').TrimEnd('\\');
else
return Utils.GetDirectoryNameSlsk(paths.First()).Replace('/', '\\').TrimEnd('\\');
}
public static string NormalizedPath(string path)
{
return path.Replace('\\', '/').TrimEnd('/').Trim();
}
public static bool SequenceEqualUpToPermutation<T>(this IEnumerable<T> list1, IEnumerable<T> list2)
{
var cnt = new Dictionary<T, int>();