1
0
Fork 0
mirror of https://github.com/fiso64/slsk-batchdl.git synced 2024-12-22 06:22:41 +00:00

slow print mode

can now search for tracks that are unavailable in youtube,
better console display,
--skip-notfound added,
--no-diacr-search added
This commit is contained in:
fiso64 2023-04-02 18:09:42 +02:00
parent 5c49877040
commit 43ae7a2754
6 changed files with 1005 additions and 378 deletions

View file

@ -6,7 +6,7 @@ A batch downloader for Soulseek using Soulseek.NET. Accepts CSV files, Spotify &
Usage: slsk-batchdl.exe [OPTIONS]
Options:
-p --parent <path> Downloaded music will be placed here
-n --name <name> Folder / playlist name. If not specified, the name of the csv file / spotify playlist is used.
-n --name <name> Folder / playlist name. If not specified, the name of the csv file / spotify / yt playlist is used.
--username <username> Soulseek username
--password <password> Soulseek password
@ -14,58 +14,75 @@ Options:
--spotify-id <id> Your spotify client id (use if the default fails or if playlist private)
--spotify-secret <sec> Your spotify client secret (use if the default fails or if playlist private)
--youtube <url> Download YouTube playlist
--youtube <url> Get tracks from a YouTube playlist
--youtube-key <key> Provide an API key if you also want to search for unavailable uploads
--no-channel-search Enable to also perform a search without channel name if nothing was found (only for yt).
--csv <path> Use a csv file containing track info to download
--artist-col <column> Specify if the csv file contains an artist name column
--track-col <column> Specify if if the csv file contains an track name column
--album-col <unit> CSV album column name. Optional, may improve searching, slower
--full-title-col <column> Specify only if there are no separate artist and track name columns in the csv
--uploader-col <column> Specify when using full title col if there is also an uploader column in the csv (fallback in case artist name cannot be extracted from title)
--artist-col <column> Artist or uploader name column
--title-col <column> Title or track name column
--album-col <column> CSV album column name. Optional, may improve searching, slower
--length-col <column> CSV duration column name. Recommended, will improve accuracy
--time-unit <unit> Time unit for the track duration column, ms or s (default: s)
--yt-desc-col <column> Description column name. Use with --yt-parse.
--yt-id-col <column> Youtube video ID column (only needed if length-col or yt-desc-col don't exist). Use with --yt-parse.
--yt-parse Enable if you have a csv file of YouTube video titles and channel names; attempt to parse.
--pref-format <format> Preferred file format (default: mp3)
--pref-length-tolerance <tol> Preferred length tolerance (if length col provided) (default: 3)
--pref-length-tol <tol> Preferred length tolerance (if length col provided) (default: 3)
--pref-min-bitrate <rate> Preferred minimum bitrate (default: 200)
--pref-max-bitrate <rate> Preferred maximum bitrate (default: 2200)
--pref-max-sample-rate <rate> Preferred maximum sample rate (default: 96000)
--pref-max-samplerate <rate> Preferred maximum sample rate (default: 96000)
--pref-danger-words <list> Comma separated list of words that must appear in either both search result and track title, or in neither of the two. Case-insensitive. (default: "mix, edit,dj ,cover")
--nec-format <format> Necessary file format
--nec-length-tolerance <tol> Necessary length tolerance (default: 3)
--nec-min-bitrate <rate> Necessary minimum bitrate
--nec-max-bitrate <rate> Necessary maximum bitrate
--nec-max-sample-rate <rate> Necessary maximum sample rate
--nec-max-samplerate <rate> Necessary maximum sample rate
--nec-danger-words <list> Comma separated list of words that must appear in either both search result and track title, or in neither of the two. Case-insensitive. (default: "mix, edit,dj ,cover")
--album-search Also search for "[Album name] [track name]". Occasionally helps to find more
--album-search Also search for "[Album name] [track name]". Occasionally helps to find more, slower.
--no-diacr-search Also perform a search without diacritics
--skip-existing Skip if a track matching the conditions is found in the output folder or your music library (if provided)
--skip-notfound Skip searching for tracks that weren't found in Soulseek last time
--remove-ft Remove "ft." or "feat." and everything after from the track names.
--remove-strings <strings> Comma separated list of strings to remove when searching for tracks. Case insesitive.
--music-dir <path> Specify to also skip downloading tracks which are in your library, use with --skip-existing
--reverse Download tracks in reverse order
--skip-if-pref-failed Skip if preferred versions of a track exist but failed to download. If no pref. versions were found, download as normal.
--create-m3u Create an m3u playlist file
--m3u-only Only create an m3u playlist file with existing tracks and exit
--m3u <path> Where to place created m3u files (--parent by default)
--yt-dlp Use yt-dlp to download tracks that weren't found on Soulseek. yt-dlp must be availble from the command line.
--yt-dlp Use yt-dlp to download tracks that weren't found on Soulseek. yt-dlp must be available from the command line.
--yt-dlp-f <format> yt-dlp audio format (default: "bestaudio/best")
--search-timeout <timeout> Maximal search time (default: 15000)
--download-max-stale-time <time> Maximal download time with no progress (default: 80000)
--max-concurrent-processes <num> Max concurrent searches / downloads (default: 2)
--max-retries-per-file <num> Maximum number of users to try downloading from before skipping track (default: 30)
```
- Files satisfying `pref` conditions will be preferred. Files not satisfying `nec` conditions will not be downloaded.
- When using csv, provide either both a track-col and artist-col (ideally), or full-title-col in case separate artist and track names are unavailable. You can also specify --uploader-col (channel names) in that case to use as artist names whenever full-title-col doesn't contain them. Always provide a length-col or get wrong results
--search-timeout <ms> Maximal search time (default: 10000)
--max-stale-time <ms> Maximal download time with no progress (default: 60000)
--concurrent-processes <num> Max concurrent searches / downloads (default: 2)
--max-retries <num> Maximum number of users to try downloading from before skipping track (default: 30)
--slow-output Enable if the progress bars aren't properly updated (bug)
```
Files satisfying `pref` conditions will be preferred. Files not satisfying `nec` conditions will not be downloaded.
Download tracks from a csv file and create m3u:
```
slsk-batchdl.exe -p "C:\Users\fiso64\Music\Playlists" --csv "C:\Users\fiso64\Downloads\test.csv" --username "fakename" --password "fakepass" --artist-col "Artist Name(s)" --track-col "Track Name" --length-col "Duration (ms)" --time-unit "ms" --skip-existing --create-m3u --pref-format "flac"
```
```
Download spotify playlist with fallback to yt-dlp and create a m3u:
```
slsk-batchdl.exe --spotify <url> -p "C:\Users\fiso64\Music\Playlists" --m3u "C:\Users\fiso64\Documents\MusicBee\Playlists" --music-dir "C:\Users\fiso64\Music" --username "fakename" --password "fakepass" --skip-existing --pref-format "flac" --yt-dlp
```
You might need to provide an id and secret when using spotify (which you can get here https://developer.spotify.com/dashboard/applications, under "Create an app").
You might need to provide an id and secret when using spotify, which you can get here https://developer.spotify.com/dashboard/applications. Create an app, then select it and add `http://localhost:48721/callback` as a redirect url in the settings.
Download youtube playlist:
```
--youtube "https://www.youtube.com/playlist?list=PLI_eFW8NAFzYAXZ5DrU6E6mQ_XfhaLBUX" -p "C:\Users\fiso64\Music\Playlists" --username "fakename" --password "fakepass"
```
To include unavailable videos, you will need to provide an api key with `--youtube-key`. Get it here https://console.cloud.google.com. Create a new project, click "Enable Api" and search for "youtube data", then follow the prompts.
## Notes:
- YouTube playlist downloading is bad: There is no way of reliably parsing video information into artist & track name. Also, many videos may be unavailable (or unavailable specifically through the api, for reasons that are beyond me), in which case they won't be downloaded.
- The console output tends to break after a while
- Much of the code was written by ChatGPT
- YouTube playlist downloading is unreliable since there are no track name / artist tags
- The CSV file must be saved with `,` as field delimiter and `"` as string delimiter, encoded with UTF8
- 40% of the code was written by ChatGPT

File diff suppressed because it is too large Load diff

View file

@ -3,17 +3,17 @@
"slsk-batchdl": {
"commandName": "Project"
},
"Profile 1": {
"YouTube": {
"commandName": "Project",
"commandLineArgs": "--youtube \"https://www.youtube.com/playlist?list=PLI_eFW8NAFzYAXZ5DrU6E6mQ_XfhaLBUX\" -p \"C:\\Users\\fiso64\\Music\\Playlists\" --username \"testing_sorry_12312\" --password \"fakepass123123\" --yt-dlp --skip-existing"
},
"Spotify": {
"commandName": "Project",
"commandLineArgs": "--spotify \"likes\" -p \"C:\\Users\\fiso64\\Music\\Playlists\" --username \"fakenameee2233\" --password \"addafffl;\" --skip-existing"
},
"CSV": {
"commandName": "Project",
"commandLineArgs": "-p \"C:\\Users\\fiso64\\Music\\Playlists\" --csv \"C:\\Users\\fiso64\\Downloads\\test.csv\" --username \"fakename99123\" --password \"fakepass123123\" --artist-col \"Artist Name(s)\" --album-col \"Album Name\" --track-col \"Track Name\" --length-col \"Duration (ms)\" --time-unit \"ms\" --pref-format \"flac\" --yt-dlp"
},
"Profile 3": {
"commandName": "Project",
"commandLineArgs": "--spotify \"likes\" -p \"C:\\Users\\fiso64\\Music\\Playlists\" --username \"fakename99123\" --password \"fakepass123123\" --yt-dlp"
},
"Profile 4": {
"commandName": "Project",
"commandLineArgs": "--youtube \"https://www.youtube.com/playlist?list=PLI_eFW8NAFzYAXZ5DrU6E6mQ_XfhaLBUX\" -p \"C:\\Users\\fiso64\\Music\\Playlists\" --username \"fakename99123\" --password \"fakepass123123\" --yt-dlp"
}
}
}

View file

@ -1,10 +1,6 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using SpotifyAPI.Web;
using SpotifyAPI.Web;
using SpotifyAPI.Web.Auth;
using Swan;
using TagLib.IFD.Tags;
public class Spotify
{
@ -33,7 +29,7 @@ public class Spotify
public async Task AuthorizeLogin()
{
Swan.Logging.Logger.NoLogging();
_server = new EmbedIOAuthServer(new Uri("http://localhost:5000/callback"), 5000);
_server = new EmbedIOAuthServer(new Uri("http://localhost:48721/callback"), 48721);
await _server.Start();
_server.AuthorizationCodeReceived += OnAuthorizationCodeReceived;
@ -43,8 +39,8 @@ public class Spotify
{
Scope = new List<string> { Scopes.UserLibraryRead, Scopes.PlaylistReadPrivate }
};
BrowserUtil.Open(request.ToUri());
loggedIn = true;
}
private async Task OnAuthorizationCodeReceived(object sender, AuthorizationCodeResponse response)
@ -54,16 +50,18 @@ public class Spotify
var config = SpotifyClientConfig.CreateDefault();
var tokenResponse = await new OAuthClient(config).RequestToken(
new AuthorizationCodeTokenRequest(
_clientId, _clientSecret, response.Code, new Uri("http://localhost:5000/callback")
_clientId, _clientSecret, response.Code, new Uri("http://localhost:48721/callback")
)
);
_client = new SpotifyClient(tokenResponse.AccessToken);
loggedIn = true;
}
private async Task OnErrorReceived(object sender, string error, string state)
{
Console.WriteLine($"Aborting authorization, error received: {error}");
await _server.Stop();
throw new Exception($"Aborting authorization, error received: {error}");
}
public async Task<bool> IsClientReady()
@ -73,7 +71,7 @@ public class Spotify
return true;
}
public async Task<List<Track>> GetLikes()
public async Task<List<Track>> GetLikes(StringEdit stringEdit)
{
if (!loggedIn)
throw new Exception("Can't get liked music, not logged in");
@ -90,7 +88,7 @@ public class Spotify
{
string[] artists = ((IEnumerable<object>)track.Track.ReadProperty("artists")).Select(a => (string)a.ReadProperty("name")).ToArray();
string artist = artists[0];
string name = (string)track.Track.ReadProperty("name");
string name = stringEdit.Edit((string)track.Track.ReadProperty("name"));
string album = (string)track.Track.ReadProperty("album").ReadProperty("name");
int duration = (int)track.Track.ReadProperty("durationMs");
res.Add(new Track { Album = album, ArtistName = artist, TrackTitle = name, Length = duration / 1000 });
@ -106,7 +104,7 @@ public class Spotify
}
public async Task<(string?, List<Track>)> GetPlaylist(string url)
public async Task<(string?, List<Track>)> GetPlaylist(string url, StringEdit stringEdit)
{
var playlistId = GetPlaylistIdFromUrl(url);
var p = await _client.Playlists.Get(playlistId);
@ -123,7 +121,7 @@ public class Spotify
{
string[] artists = ((IEnumerable<object>)track.Track.ReadProperty("artists")).Select(a => (string)a.ReadProperty("name")).ToArray();
string artist = artists[0];
string name = (string)track.Track.ReadProperty("name");
string name = stringEdit.Edit((string)track.Track.ReadProperty("name"));
string album = (string)track.Track.ReadProperty("album").ReadProperty("name");
int duration = (int)track.Track.ReadProperty("durationMs");
res.Add(new Track { Album = album, ArtistName = artist, TrackTitle = name, Length = duration / 1000 });

View file

@ -1,90 +1,293 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Google.Apis.YouTube.v3;
using Google.Apis.Services;
using System.Xml;
using YoutubeExplode;
using System.Text.RegularExpressions;
public static class YouTube
{
public static async Task<(string, List<Track>)> GetTracks(string url)
private static YoutubeClient? youtube = new YoutubeClient();
private static YouTubeService? youtubeService = null;
public static string apiKey = "";
public static async Task<(string, List<Track>)> GetTracksApi(string url, StringEdit strEdit)
{
StartService();
string playlistId = await UrlToId(url);
var playlistRequest = youtubeService.Playlists.List("snippet");
playlistRequest.Id = playlistId;
var playlistResponse = playlistRequest.Execute();
string playlistName = playlistResponse.Items[0].Snippet.Title;
var playlistItemsRequest = youtubeService.PlaylistItems.List("snippet,contentDetails");
playlistItemsRequest.PlaylistId = playlistId;
playlistItemsRequest.MaxResults = 100;
var tracksDict = await GetDictYtExplode(url, strEdit);
var tracks = new List<Track>();
while (playlistItemsRequest != null)
{
var playlistItemsResponse = playlistItemsRequest.Execute();
foreach (var playlistItem in playlistItemsResponse.Items)
{
if (tracksDict.ContainsKey(playlistItem.Snippet.ResourceId.VideoId))
{
tracks.Add(tracksDict[playlistItem.Snippet.ResourceId.VideoId]);
}
else
{
var title = "";
var uploader = "";
var length = 0;
var desc = "";
try
{
var video = await youtube.Videos.GetAsync(playlistItem.Snippet.ResourceId.VideoId);
title = video.Title;
uploader = video.Author.Title;
length = (int)video.Duration.Value.TotalSeconds;
desc = video.Description;
}
catch
{
var videoRequest = youtubeService.Videos.List("contentDetails,snippet");
videoRequest.Id = playlistItem.Snippet.ResourceId.VideoId;
var videoResponse = videoRequest.Execute();
title = playlistItem.Snippet.Title;
if (videoResponse.Items.Count == 0)
continue;
uploader = videoResponse.Items[0].Snippet.ChannelTitle;
length = (int)XmlConvert.ToTimeSpan(videoResponse.Items[0].ContentDetails.Duration).TotalSeconds;
desc = videoResponse.Items[0].Snippet.Description;
}
Track track = await ParseTrackInfo(strEdit.Edit(title), uploader, playlistItem.Snippet.ResourceId.VideoId, length, false, desc);
tracks.Add(track);
}
}
if (tracksDict.Count >= 200)
{
Console.SetCursorPosition(0, Console.CursorTop);
Console.Write(tracks.Count);
}
playlistItemsRequest.PageToken = playlistItemsResponse.NextPageToken;
if (playlistItemsRequest.PageToken == null)
{
playlistItemsRequest = null;
}
}
Console.WriteLine();
return (playlistName, tracks);
}
public static async Task<Track> ParseTrackInfo(string title, string uploader, string id, int length, bool requestInfoIfNeeded, string desc = "")
{
(string title, string uploader, int length, string desc) info = ("", "", -1, "");
var track = new Track();
track.YtID = id;
title = title.Replace("", "-");
var stringsToRemove = new string[] { "(Official music video)", "(Official video)", "(Official audio)",
"(Lyrics)", "(Official)", "(Lyric Video)", "(Official Lyric Video)", "(Official HD Video)",
"(Official 4K Video)", "(Video)", "[HD]", "[4K]", "(Original Mix)", "(Lyric)", "(Music Video)",
"(Visualizer)", "(Audio)", "Official Lyrics" };
foreach (string s in stringsToRemove)
{
var t = title;
title = Regex.Replace(title, Regex.Escape(s), "", RegexOptions.IgnoreCase);
if (t == title)
{
if (s.Contains("["))
{
string s2 = s.Replace("[", "(").Replace("]", ")");
title = Regex.Replace(title, Regex.Escape(s2), "", RegexOptions.IgnoreCase);
}
else if (s.Contains("("))
{
string s2 = s.Replace("(", "[").Replace(")", "]");
title = Regex.Replace(title, Regex.Escape(s2), "", RegexOptions.IgnoreCase);
}
}
}
var trackTitle = title.Trim();
trackTitle = Regex.Replace(trackTitle, @"\s+", " ");
var artist = uploader.Trim();
if (artist.EndsWith("- Topic"))
{
artist = artist.Substring(0, artist.Length - 7).Trim();
trackTitle = title;
if (artist == "Various Artists")
{
if (desc == "" && requestInfoIfNeeded && id != "")
{
info = await GetVideoInfo(id);
desc = info.desc;
}
if (desc != "")
{
var lines = desc.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
var dotLine = lines.FirstOrDefault(line => line.Contains(" · "));
if (dotLine != null)
artist = dotLine.Split(new[] { " · " }, StringSplitOptions.None)[1];
}
}
}
else
{
int idx = title.IndexOf('-');
var split = title.Split(new[] { '-' }, 2);
if (idx > 0 && idx < title.Length - 1 && (title[idx - 1] == ' ' || title[idx + 1] == ' ') && split[0].Trim() != "" && split[1].Trim() != "")
{
artist = title.Split(new[] { '-' }, 2)[0].Trim();
trackTitle = title.Split(new[] { '-' }, 2)[1].Trim();
}
else
{
track.ArtistMaybeWrong = true;
}
}
if (length <= 0 && id != "" && requestInfoIfNeeded)
{
if (info.length > 0)
length = info.length;
else
{
info = await GetVideoInfo(id);
length = info.length;
}
}
track.Length = length;
track.ArtistName = artist;
track.TrackTitle = trackTitle;
return track;
}
public static async Task<(string title, string uploader, int length, string desc)> GetVideoInfo(string id)
{
(string title, string uploader, int length, string desc) o = ("", "", -1, "");
try
{
var vid = await youtube.Videos.GetAsync(id);
o.title = vid.Title;
o.uploader = vid.Author.ChannelTitle;
o.desc = vid.Description;
o.length = (int)vid.Duration.Value.TotalSeconds;
}
catch
{
if (apiKey != "")
{
try
{
StartService();
var videoRequest = youtubeService.Videos.List("contentDetails,snippet");
videoRequest.Id = id;
var videoResponse = videoRequest.Execute();
o.title = videoResponse.Items[0].Snippet.Title;
o.uploader = videoResponse.Items[0].Snippet.ChannelTitle;
o.length = (int)XmlConvert.ToTimeSpan(videoResponse.Items[0].ContentDetails.Duration).TotalSeconds;
o.desc = videoResponse.Items[0].Snippet.Description;
}
catch { }
}
}
return o;
}
public static void StartService()
{
if (youtubeService == null)
{
if (apiKey == "")
throw new Exception("No API key");
youtubeService = new YouTubeService(new BaseClientService.Initializer()
{
ApiKey = apiKey,
ApplicationName = "slsk-batchdl"
});
}
}
public static void StopService()
{
//try { youtubeService.Dispose(); }
//catch { }
youtubeService = null;
}
public static async Task<Dictionary<string, Track>> GetDictYtExplode(string url, StringEdit strEdit)
{
var youtube = new YoutubeClient();
var playlist = await youtube.Playlists.GetAsync(url);
var playlistTitle = playlist.Title;
var tracks = new List<Track>();
var videoTasks = new List<(ValueTask<YoutubeExplode.Videos.Video>, int)>();
var tracks = new Dictionary<string, Track>();
await foreach (var video in youtube.Playlists.GetVideosAsync(playlist.Id))
{
var title = video.Title;
var title = strEdit.Edit(video.Title);
var uploader = video.Author.Title;
var ytId = video.Id.Value;
var length = (int)video.Duration.Value.TotalSeconds;
title = title.Replace("", "-");
var track = await ParseTrackInfo(title, uploader, ytId, length, true);
var trackTitle = title.Trim();
var artist = uploader.Trim();
tracks[ytId] = track;
}
if (artist.EndsWith("- Topic"))
{
artist = artist.Substring(0, artist.Length - 7).Trim();
trackTitle = title;
return tracks;
}
if (artist == "Various Artists")
{
//var vid = await youtube.Videos.GetAsync(video.Id);
videoTasks.Add((youtube.Videos.GetAsync(video.Id), tracks.Count));
//Thread.Sleep(20);
}
}
else
{
int idx = title.IndexOf('-');
var split = title.Split(new[] { '-' }, 2);
if (idx > 0 && idx < title.Length - 1 && (title[idx - 1] == ' ' || title[idx + 1] == ' ') && split[0].Trim() != "" && split[1].Trim() != "")
public static async Task<(string, List<Track>)> GetTracksYtExplode(string url, StringEdit strEdit)
{
var playlist = await youtube.Playlists.GetAsync(url);
{
artist = title.Split(new[] { '-' }, 2)[0].Trim();
trackTitle = title.Split(new[] { '-' }, 2)[1].Trim();
}
else
{
artist = uploader;
trackTitle = title;
}
}
var playlistTitle = playlist.Title;
var tracks = new List<Track>();
var track = new Track
{
UnparsedTitle = video.Title,
Uploader = uploader,
TrackTitle = trackTitle,
ArtistName = artist,
YtID = ytId,
Length = length
};
await foreach (var video in youtube.Playlists.GetVideosAsync(playlist.Id))
{
var title = strEdit.Edit(video.Title);
var uploader = video.Author.Title;
var ytId = video.Id.Value;
var length = (int)video.Duration.Value.TotalSeconds;
var track = await ParseTrackInfo(title, uploader, ytId, length, true);
tracks.Add(track);
}
foreach ((var vidTask, int idx) in videoTasks)
{
var vid = await vidTask;
var lines = vid.Description.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
var dotLine = lines.FirstOrDefault(line => line.Contains(" · "));
if (dotLine != null)
{
var t = tracks[idx];
t.ArtistName = dotLine.Split(new[] { " · " }, StringSplitOptions.None)[1]; // can't be asked to do it properly
tracks[idx] = t;
}
}
return (playlistTitle, tracks);
}
public static async Task<string> UrlToId(string url)
{
var playlist = await youtube.Playlists.GetAsync(url);
return playlist.Id.ToString();
}
}

View file

@ -8,11 +8,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Goblinfactory.Konsole" Version="6.2.2" />
<PackageReference Include="Google.Apis.YouTube.v3" Version="1.60.0.2945" />
<PackageReference Include="Soulseek" Version="6.1.1" />
<PackageReference Include="SpotifyAPI.Web" Version="7.0.0" />
<PackageReference Include="SpotifyAPI.Web.Auth" Version="7.0.0" />
<PackageReference Include="TagLibSharp" Version="2.3.0" />
<PackageReference Include="YoutubeExplode" Version="6.2.11" />
<PackageReference Include="YoutubeExplode" Version="6.2.12" />
</ItemGroup>
</Project>