From 4f366364367ff733306f15cdfa8461c90c853a90 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 13:09:41 +0200 Subject: [PATCH 01/44] Add SoundCloudSoftException so that already downloaded tracks are not treated as errors --- scdl/scdl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 8eac798e..af89535e 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -112,6 +112,9 @@ class SoundCloudException(Exception): pass +class SoundCloudSoftException(Exception): + pass + def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): logger.error("\nGoodbye!") From c557cc6003e4253f519dcaac046981bfe6f18c88 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 13:13:29 +0200 Subject: [PATCH 02/44] Employ SoundCloudSoftException within download_track so that already downloaded tracks will not trigger errors. Implement Boolean return values. --- scdl/scdl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index af89535e..fbc4259f 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -726,7 +726,7 @@ def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, ex # Skip if file ID or filename already exists if is_already_downloaded and not kwargs.get("force_metadata"): - raise SoundCloudException(f"{filename} already downloaded.") + raise SoundCloudSoftException(f"{filename} already downloaded.") # If file does not exist an error occurred if not os.path.isfile(filename): @@ -753,11 +753,17 @@ def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, ex try_utime(filename, filetime) logger.info(f"{filename} Downloaded.\n") + return True + except SoundCloudException as err: logger.error(err) if exit_on_fail: sys.exit(1) + return False + except SoundCloudSoftException as err: + logger.error(err) + return False def can_convert(filename): ext = os.path.splitext(filename)[1] From 1ed74bfc0c274e38cb15719357cab116f153123d Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 16:53:00 +0200 Subject: [PATCH 03/44] download_url: Permit multiple actions on user profile --- scdl/scdl.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index fbc4259f..498b6c49 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -318,6 +318,9 @@ def download_url(client: SoundCloud, **kwargs): elif item.kind == "user": user = item logger.info("Found a user profile") + if not kwargs.get("f") and not kwargs.get("C") and not kwargs.get("t") and not kwargs.get("a") and not kwargs.get("p") and not kwargs.get("r"): + logger.error("Please provide a download type...") + sys.exit(1) if kwargs.get("f"): logger.info(f"Retrieving all likes of user {user.username}...") resources = client.get_user_likes(user.id, limit=1000) @@ -332,21 +335,21 @@ def download_url(client: SoundCloud, **kwargs): if kwargs.get("strict_playlist"): sys.exit(1) logger.info(f"Downloaded all likes of user {user.username}!") - elif kwargs.get("C"): + if kwargs.get("C"): logger.info(f"Retrieving all commented tracks of user {user.username}...") resources = client.get_user_comments(user.id, limit=1000) for i, comment in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"comment n°{i} of {user.comments_count}") download_track(client, client.get_track(comment.track.id), exit_on_fail=kwargs.get("strict_playlist"), **kwargs) logger.info(f"Downloaded all commented tracks of user {user.username}!") - elif kwargs.get("t"): + if kwargs.get("t"): logger.info(f"Retrieving all tracks of user {user.username}...") resources = client.get_user_tracks(user.id, limit=1000) for i, track in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"track n°{i} of {user.track_count}") download_track(client, track, exit_on_fail=kwargs.get("strict_playlist"), **kwargs) logger.info(f"Downloaded all tracks of user {user.username}!") - elif kwargs.get("a"): + if kwargs.get("a"): logger.info(f"Retrieving all tracks & reposts of user {user.username}...") resources = client.get_user_stream(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): @@ -360,14 +363,14 @@ def download_url(client: SoundCloud, **kwargs): if kwargs.get("strict_playlist"): sys.exit(1) logger.info(f"Downloaded all tracks & reposts of user {user.username}!") - elif kwargs.get("p"): + if kwargs.get("p"): logger.info(f"Retrieving all playlists of user {user.username}...") resources = client.get_user_playlists(user.id, limit=1000) for i, playlist in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"playlist n°{i} of {user.playlist_count}") download_playlist(client, playlist, **kwargs) logger.info(f"Downloaded all playlists of user {user.username}!") - elif kwargs.get("r"): + if kwargs.get("r"): logger.info(f"Retrieving all reposts of user {user.username}...") resources = client.get_user_reposts(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): @@ -381,9 +384,7 @@ def download_url(client: SoundCloud, **kwargs): if kwargs.get("strict_playlist"): sys.exit(1) logger.info(f"Downloaded all reposts of user {user.username}!") - else: - logger.error("Please provide a download type...") - sys.exit(1) + else: logger.error(f"Unknown item type {item.kind}") sys.exit(1) From aae5e33a1d432763f6e422a9f3044de7cd9abd1a Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 17:37:30 +0200 Subject: [PATCH 04/44] Adjust arguments to allow multiple actions on user accounts --- scdl/scdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 498b6c49..a39c41b7 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -4,7 +4,7 @@ """scdl allows you to download music from Soundcloud Usage: - scdl (-l | me) [-a | -f | -C | -t | -p | -r][-c | --force-metadata] + scdl (-l | me) [-a][-r][-t][-f][-C][-p][-c][--force-metadata] [-n ][-o ][--hidewarnings][--debug | --error][--path ] [--addtofile][--addtimestamp][--onlymp3][--hide-progress][--min-size ] [--max-size ][--remove][--no-album-tag][--no-playlist-folder] From 3233747be583fd38529cb98271ae13a329f23739 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 16:15:10 +0200 Subject: [PATCH 05/44] Add --playlist-file, --playlist-file-retain, --playlist-file-name, --playlist-file-extension arguments --- scdl/scdl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index a39c41b7..872f9364 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -11,7 +11,8 @@ [--download-archive ][--sync ][--extract-artist][--flac][--original-art] [--original-name][--no-original][--only-original][--name-format ] [--strict-playlist][--playlist-name-format ][--client-id ] - [--auth-token ][--overwrite][--no-playlist] + [--auth-token ][--overwrite][--no-playlist][--playlist-file] + [--playlist-file-retain][--playlist-file-name][--playlist-file-extension] scdl -h | --help scdl --version @@ -64,6 +65,10 @@ --overwrite Overwrite file if it already exists --strict-playlist Abort playlist downloading if one track fails to download --no-playlist Skip downloading playlists + --playlist-file Generate m3u playlist files (and additionally check them when used with --remove) + --playlist-file-retain Retain corrupted items + --playlist-file-name Specify playlist file name without extension + --playlist-file-extension Specify extension to playlist file """ import cgi From 68915084060e0fad632306fa987926cee68bb9ee Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 14:24:27 +0200 Subject: [PATCH 06/44] Add kwdefget as argument getter with default values --- scdl/scdl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 872f9364..3f71cda4 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -301,6 +301,9 @@ def get_config(config_file: pathlib.Path) -> configparser.ConfigParser: return config +def kwdefget(findkey, defaultvalue, **kwargs): + if not kwargs.get(findkey): return defaultvalue + return kwargs.get(findkey) def download_url(client: SoundCloud, **kwargs): """ From 06a1afa7216646d63d541e39a0f19b95c460136d Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 13:24:25 +0200 Subject: [PATCH 07/44] download_track: Implement playlist_buffer --- dl/Likesm3u8.map.map | 0 scdl/scdl.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 dl/Likesm3u8.map.map diff --git a/dl/Likesm3u8.map.map b/dl/Likesm3u8.map.map new file mode 100644 index 00000000..e69de29b diff --git a/scdl/scdl.py b/scdl/scdl.py index 3f71cda4..4e43211a 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -696,7 +696,7 @@ def download_hls(client: SoundCloud, track: BasicTrack, title: str, playlist_inf return (filename, False) -def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, exit_on_fail=True, **kwargs): +def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, exit_on_fail=True, playlist_buffer=None, **kwargs): """ Downloads a track """ @@ -735,6 +735,7 @@ def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, ex # Skip if file ID or filename already exists if is_already_downloaded and not kwargs.get("force_metadata"): + if playlist_buffer is not None: playlist_buffer.append({ "id": track.id, "path": filename, "uri": track.uri }) raise SoundCloudSoftException(f"{filename} already downloaded.") # If file does not exist an error occurred @@ -761,6 +762,7 @@ def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, ex filetime = int(time.mktime(track.created_at.timetuple())) try_utime(filename, filetime) + if playlist_buffer is not None: playlist_buffer.append({ "id": track.id, "path": filename, "uri": track.uri }) logger.info(f"{filename} Downloaded.\n") return True From 02ca05c70f0851dc5046015bcb154175f85b2a7c Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:24:12 +0200 Subject: [PATCH 08/44] download_playlist: Skip on empty playlist --- scdl/scdl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 4e43211a..91aefca6 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -482,6 +482,9 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, **kwargs ) playlist.tracks = playlist.tracks[: int(kwargs.get("n"))] kwargs["playlist_offset"] = 0 + + if not playlist.tracks or len(playlist.tracks) == 0: return + if kwargs.get("sync"): if os.path.isfile(kwargs.get("sync")): playlist.tracks = sync(client, playlist, playlist_info, **kwargs) From b7623d32bb30bfff2d515ff3b8835c6ed08625c9 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:28:44 +0200 Subject: [PATCH 09/44] download_playlist: Implement playlist file generation and playlist meta information feedback --- scdl/scdl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 91aefca6..3bdc39a7 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -454,7 +454,7 @@ def sync(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist_info, **kwar logger.info('No tracks to download. Exiting...') sys.exit(0) -def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, **kwargs): +def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist_filename_prefix="", subplaylist_buffer=None, **kwargs): """ Downloads a playlist """ @@ -493,6 +493,7 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, **kwargs sys.exit(1) tracknumber_digits = len(str(len(playlist.tracks))) + playlistbuffer=None if not kwargs.get("playlist_file") else [] for counter, track in itertools.islice(enumerate(playlist.tracks, 1), kwargs.get("playlist_offset", 0), None): logger.debug(track) logger.info(f"Track n°{counter}") @@ -503,7 +504,11 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, **kwargs else: track = client.get_track(track.id) - download_track(client, track, playlist_info, kwargs.get("strict_playlist"), **kwargs) + download_track(client, track, playlist_info, kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + if kwargs.get("playlist_file"): + playlist_filename=playlist_filename_prefix + playlist_name + ".m3u8" + playlist_process(client, playlistbuffer, playlist_filename, **kwargs) + if subplaylist_buffer: subplaylist_buffer.append({ "id": playlist.id, "path": playlist_filename, "uri": playlist.uri }) finally: if not kwargs.get("no_playlist_folder"): os.chdir("..") From a3eecc17955106e4109e391a20f2dcac9a420452 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:37:05 +0200 Subject: [PATCH 10/44] download_url: Implement playlist generation for user likes --- dl/Likesm3u8.map.map | 0 scdl/scdl.py | 9 +++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 dl/Likesm3u8.map.map diff --git a/dl/Likesm3u8.map.map b/dl/Likesm3u8.map.map deleted file mode 100644 index e69de29b..00000000 diff --git a/scdl/scdl.py b/scdl/scdl.py index 3bdc39a7..7c4b5d87 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -331,17 +331,22 @@ def download_url(client: SoundCloud, **kwargs): sys.exit(1) if kwargs.get("f"): logger.info(f"Retrieving all likes of user {user.username}...") + playlistbuffer=None if not kwargs.get("playlist_file") else [] + subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_likes(user.id, limit=1000) for i, like in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"like n°{i} of {user.likes_count}") if hasattr(like, "track"): - download_track(client, like.track, exit_on_fail=kwargs.get("strict_playlist"), **kwargs) + download_track(client, like.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) elif hasattr(like, "playlist"): - download_playlist(client, client.get_playlist(like.playlist.id), **kwargs) + download_playlist(client, client.get_playlist(like.playlist.id), playlist_filename_prefix=kwdefget("playlist_file_name", "Likes", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: logger.error(f"Unknown like type {like}") if kwargs.get("strict_playlist"): sys.exit(1) + if kwargs.get("playlist_file"): + playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Likes", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) + playlist_process(client, subplaylistbuffer, kwdefget("playlist_file_name", "Likes Playlists", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), no_export=True, **kwargs) logger.info(f"Downloaded all likes of user {user.username}!") if kwargs.get("C"): logger.info(f"Retrieving all commented tracks of user {user.username}...") From c981e0c6cda782ce9e34c47fb120f5d0ea7895fa Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:44:23 +0200 Subject: [PATCH 11/44] download_url: Implement playlist generation for commented tracks --- scdl/scdl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 7c4b5d87..1777bc66 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -350,10 +350,12 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all likes of user {user.username}!") if kwargs.get("C"): logger.info(f"Retrieving all commented tracks of user {user.username}...") + playlistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_comments(user.id, limit=1000) for i, comment in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"comment n°{i} of {user.comments_count}") - download_track(client, client.get_track(comment.track.id), exit_on_fail=kwargs.get("strict_playlist"), **kwargs) + download_track(client, client.get_track(comment.track.id), exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + if kwargs.get("playlist_file"): playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Commented", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) logger.info(f"Downloaded all commented tracks of user {user.username}!") if kwargs.get("t"): logger.info(f"Retrieving all tracks of user {user.username}...") From e572a0cb99fd2f980ec458e8e0768ca9ffc4dfdd Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:46:37 +0200 Subject: [PATCH 12/44] download_url: Implement playlist generation for posted tracks --- scdl/scdl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 1777bc66..8d6e99d2 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -359,10 +359,12 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all commented tracks of user {user.username}!") if kwargs.get("t"): logger.info(f"Retrieving all tracks of user {user.username}...") + playlistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_tracks(user.id, limit=1000) for i, track in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"track n°{i} of {user.track_count}") - download_track(client, track, exit_on_fail=kwargs.get("strict_playlist"), **kwargs) + download_track(client, track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + if kwargs.get("playlist_file"): playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Tracks", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) logger.info(f"Downloaded all tracks of user {user.username}!") if kwargs.get("a"): logger.info(f"Retrieving all tracks & reposts of user {user.username}...") From 3fc38e6628aec740facf60d72a16deac5bc2e1dc Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:49:50 +0200 Subject: [PATCH 13/44] download_url: Implement playlist generation for joined posts and reposts (also known as stream) --- scdl/scdl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 8d6e99d2..b735412b 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -368,17 +368,22 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all tracks of user {user.username}!") if kwargs.get("a"): logger.info(f"Retrieving all tracks & reposts of user {user.username}...") + playlistbuffer=None if not kwargs.get("playlist_file") else [] + subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_stream(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"item n°{i} of {user.track_count + user.reposts_count if user.reposts_count else '?'}") if item.type in ("track", "track-repost"): - download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), **kwargs) + download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) elif item.type in ("playlist", "playlist-repost"): - download_playlist(client, item.playlist, **kwargs) + download_playlist(client, item.playlist, kwdefget("playlist_file_name", "Stream", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: logger.error(f"Unknown item type {item.type}") if kwargs.get("strict_playlist"): sys.exit(1) + if kwargs.get("playlist_file"): + playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Stream", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) + playlist_process(client, subplaylistbuffer, kwdefget("playlist_file_name", "Stream Playlists", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), no_export=True, **kwargs) logger.info(f"Downloaded all tracks & reposts of user {user.username}!") if kwargs.get("p"): logger.info(f"Retrieving all playlists of user {user.username}...") From d0b2beff4ecb28c4e30277728ec11ed2477c8c7b Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:51:44 +0200 Subject: [PATCH 14/44] download_url: Implement playlist generation for playlists --- scdl/scdl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index b735412b..1240c581 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -387,10 +387,13 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all tracks & reposts of user {user.username}!") if kwargs.get("p"): logger.info(f"Retrieving all playlists of user {user.username}...") + #subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_playlists(user.id, limit=1000) for i, playlist in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"playlist n°{i} of {user.playlist_count}") - download_playlist(client, playlist, **kwargs) + download_playlist(client, playlist, **kwargs) # subplaylist_buffer=subplaylistbuffer, + #if kwargs.get("playlist_file"): + # playlist_process(client, subplaylistbuffer, kwdefget("playlist_file_name", "Playlists", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), no_export=True, **kwargs) logger.info(f"Downloaded all playlists of user {user.username}!") if kwargs.get("r"): logger.info(f"Retrieving all reposts of user {user.username}...") From ae4ef2657aa29d02f91b7d5204e5d45a61240569 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:52:52 +0200 Subject: [PATCH 15/44] download_url: Implement playlist generation for reposts --- scdl/scdl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 1240c581..3e4cff75 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -397,17 +397,22 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all playlists of user {user.username}!") if kwargs.get("r"): logger.info(f"Retrieving all reposts of user {user.username}...") + playlistbuffer=None if not kwargs.get("playlist_file") else [] + subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_reposts(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"item n°{i} of {user.reposts_count or '?'}") if item.type == "track-repost": - download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), **kwargs) + download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) elif item.type == "playlist-repost": - download_playlist(client, item.playlist, **kwargs) + download_playlist(client, item.playlist, kwdefget("playlist_file_name", "Reposts", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: logger.error(f"Unknown item type {item.type}") if kwargs.get("strict_playlist"): sys.exit(1) + if kwargs.get("playlist_file"): + playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Reposts", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) + playlist_process(client, subplaylistbuffer, kwdefget("playlist_file_name", "Reposts Playlists", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), no_export=True, **kwargs) logger.info(f"Downloaded all reposts of user {user.username}!") else: From 412baaceba7c5b17f2c5115f9d90f2587d9abe5f Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 16:29:58 +0200 Subject: [PATCH 16/44] download_hls: Use temporary file so interruptions will not cause inconsistency --- scdl/scdl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 3e4cff75..3914b3ef 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -713,16 +713,18 @@ def download_hls(client: SoundCloud, track: BasicTrack, title: str, playlist_inf # Get the requests stream url = get_transcoding_m3u8(client, transcoding, **kwargs) - filename_path = os.path.abspath(filename) + temp = tempfile.NamedTemporaryFile(delete=False) + temp_filename = temp.name + os.path.splitext(filename)[1] p = subprocess.Popen( - ["ffmpeg", "-i", url, "-c", "copy", filename_path, "-loglevel", "error"], + ["ffmpeg", "-i", url, "-c", "copy", temp_filename, "-loglevel", "error"], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout, stderr = p.communicate() if stderr: logger.error(stderr.decode("utf-8")) + shutil.move(temp_filename, os.path.join(os.getcwd(), filename)) return (filename, False) From 23252112f620638fff4acdf50126b6137513f730 Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 12:36:18 +0200 Subject: [PATCH 17/44] download_url: Implement limit (-n) for user likes --- scdl/scdl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 3914b3ef..32b94ece 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -333,7 +333,8 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Retrieving all likes of user {user.username}...") playlistbuffer=None if not kwargs.get("playlist_file") else [] subplaylistbuffer=None if not kwargs.get("playlist_file") else [] - resources = client.get_user_likes(user.id, limit=1000) + resources = client.get_user_likes(user.id, limit=int(kwdefget("n", "1000", **kwargs))) + ilim=int(kwdefget("n", "-1", **kwargs)) for i, like in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"like n°{i} of {user.likes_count}") if hasattr(like, "track"): @@ -344,6 +345,8 @@ def download_url(client: SoundCloud, **kwargs): logger.error(f"Unknown like type {like}") if kwargs.get("strict_playlist"): sys.exit(1) + ilim=ilim-1 + if ilim == 0: break if kwargs.get("playlist_file"): playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Likes", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) playlist_process(client, subplaylistbuffer, kwdefget("playlist_file_name", "Likes Playlists", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), no_export=True, **kwargs) From 4d2bdddef4ec2417d0cd856c5bdd83c4e6d1a106 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 15:22:44 +0200 Subject: [PATCH 18/44] Add playlist_process to export m3u playlists as well as to retain corrupted tracks in a user-serviceable manner --- scdl/scdl.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 32b94ece..e2b25ab5 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1010,5 +1010,53 @@ def is_ffmpeg_available(): """ return shutil.which("ffmpeg") is not None +def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_export=False, no_retain=False, **kwargs): + if kwargs.get("playlist_file_retain") and no_retain == False: + oldint = playlist_map_read(playlist_filename) + oldext = None if no_export == True else playlist_import(playlist_filename) + oldindex=-1 + + logger.debug(f"Old Map: {oldint}") + logger.debug("Old Playlist: {oldext}") + + for oldel in reversed(oldint): + tpath=oldel["path"] + oldindex=oldindex+1 + if oldel["id"] == "-1": + # Stop retaining track if deleted from playlist by the user + if no_export == False and oldext is not None and oldel["path"] not in oldext: + logger.debug(f"Not retaining {tpath} ({oldel['uri']})") + continue + # Stop retaining playlist if according file deleted by the user + elif no_export == True and not os.path.isfile(oldel["path"]): + logger.debug(f"Not retaining {tpath} ({oldel['uri']})") + continue + # Stop retaining if track or playlist has been restored + if next((newel for newel in playlist_buffer if newel["uri"] == oldel["uri"]), None) is not None: + logger.debug(f"Stopping to retain restored {tpath} ({oldel['uri']})") + continue + + else: + # Check if item removed + if next((newel for newel in playlist_buffer if newel["uri"] == oldel["uri"]), None) is not None: continue + # Check if item removed because it is corrupted + if check_item(client, oldel["uri"]) == True: + # Item not corrupted + logger.debug(f"Not retaining {tpath} ({oldel['uri']})") + if no_export == True and os.path.isfile(oldel["path"]): os.remove(oldel["path"]) + continue + oldel["id"] = "-1" + + # Temporarily retain item due to corrupted file on server + logger.debug(f"Retaining {tpath} ({oldel['uri']})") + newindex = 0 if len(playlist_buffer) - oldindex < 0 else len(playlist_buffer) - oldindex + playlist_buffer.insert(newindex, oldel) + + logger.debug(f"New Map: {playlist_buffer}") + playlist_map_write(playlist_buffer, playlist_filename) + + if no_export == False: + playlist_export(playlist_buffer, playlist_filename) + if __name__ == "__main__": main() From c9d2451da0f6597d3eaa2587e0898bdb01ade7e8 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 18:34:34 +0200 Subject: [PATCH 19/44] Add playlist_export function to generate m3u playlist files --- scdl/scdl.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index e2b25ab5..b983cac4 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1058,5 +1058,13 @@ def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_ if no_export == False: playlist_export(playlist_buffer, playlist_filename) + + +def playlist_export(playlist_buffer, playlist_filename): + with open(playlist_filename, "w") as fout: + for playlist_item in playlist_buffer: + fout.write(playlist_item["path"] + "\n") + + if __name__ == "__main__": main() From f3d4fad39aa4ce8b0443836a96148de393e59c1c Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 12:49:53 +0200 Subject: [PATCH 20/44] Add playlist_import function to import m3u playlists --- scdl/scdl.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index b983cac4..2896ee6f 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1060,6 +1060,18 @@ def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_ +def playlist_import(playlist_filename): + try: + res = [] + if not os.path.isfile(playlist_filename): return None + with open(playlist_filename, "r") as fin: + for fline in fin.read().splitlines(): + if not fline.startswith("#"): res.append(fline) + + return res + except Exception: + return None + def playlist_export(playlist_buffer, playlist_filename): with open(playlist_filename, "w") as fout: for playlist_item in playlist_buffer: From 296b3b400e385de99b3df250e273cd0bd17a68fd Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 18:35:01 +0200 Subject: [PATCH 21/44] Add playlist_map_write function to store internal playlist information not suitable for export --- scdl/scdl.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 2896ee6f..1f0e6165 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1058,7 +1058,11 @@ def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_ if no_export == False: playlist_export(playlist_buffer, playlist_filename) - +def playlist_map_write(playlist_buffer, playlist_filename): + if playlist_buffer is None or len(playlist_buffer) == 0: return + with open(playlist_filename + ".map", "w") as fout: + for playlist_item in playlist_buffer: + fout.write(str(playlist_item["id"]) + ":" + playlist_item["path"] + ":" + playlist_item["uri"] + "\n") def playlist_import(playlist_filename): try: From 51a4c3fbf34b1695a77848ac8446e2912913b4c3 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 7 Sep 2022 18:12:13 +0200 Subject: [PATCH 22/44] Add playlist_map_read to read internal playlist information --- scdl/scdl.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 1f0e6165..eab6fa88 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1058,6 +1058,18 @@ def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_ if no_export == False: playlist_export(playlist_buffer, playlist_filename) +def playlist_map_read(playlist_filename): + try: + res = [] + if not os.path.isfile(playlist_filename + ".map"): return res + with open(playlist_filename + ".map", "r") as fin: + for fline in fin.read().splitlines(): + ffields = fline.split(":", 2) + if len(ffields) == 3: res.append({ "id": ffields[0], "path": ffields[1], "uri": ffields[2] }) + return res + except Exception: + return [] + def playlist_map_write(playlist_buffer, playlist_filename): if playlist_buffer is None or len(playlist_buffer) == 0: return with open(playlist_filename + ".map", "w") as fout: From c01b410b2641086df83175fccd79d8485775c0eb Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 14:49:26 +0200 Subject: [PATCH 23/44] Add check_item function to check item remote state --- scdl/scdl.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index eab6fa88..81101fa3 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1093,6 +1093,24 @@ def playlist_export(playlist_buffer, playlist_filename): for playlist_item in playlist_buffer: fout.write(playlist_item["path"] + "\n") +def check_item(client: SoundCloud, itemuri): + item = client.resolve(itemuri) + if not item: + return False + elif item.kind == "track": + if item.policy == "BLOCK": return False + if item.downloadable and client.get_track_original_download(item.id, item.secret_token): return True + if not item.media.transcodings: return False + for t in item.media.transcodings: + if t.format.protocol == "hls" and "aac" in t.preset: return True + elif t.format.protocol == "hls" and "mp3" in t.preset: return True + return False + elif item.kind == "playlist": + return item.tracks is not None and len(item.tracks) > 0 + elif item.kind == "user": + return True + else: + return False if __name__ == "__main__": main() From 4bb43be5e34bfa939af0ba3d71a3914a0a912627 Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 16:29:26 +0200 Subject: [PATCH 24/44] main: Adjust arguments to remove_files call to also consider playlist files --- scdl/scdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 81101fa3..4e12bd35 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -247,7 +247,7 @@ def main(): download_url(client, **python_args) if arguments["--remove"]: - remove_files() + remove_files(arguments["--playlist-file"] is not None, kwdefget("playlist_file_extension", "m3u8", **python_args)) def validate_url(client: SoundCloud, url: str): From 38397987f16c886ee6fe765fba5d0d7124bccbfc Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 16:33:16 +0200 Subject: [PATCH 25/44] remove_files: Implement additional check against playlist files and skip folder if none can be found --- scdl/scdl.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 4e12bd35..e9ed8259 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -422,15 +422,24 @@ def download_url(client: SoundCloud, **kwargs): logger.error(f"Unknown item type {item.kind}") sys.exit(1) -def remove_files(): +def remove_files(check_playlist_file, playlist_file_extension): """ Removes any pre-existing tracks that were not just downloaded """ logger.info("Removing local track files that were not downloaded...") - files = [f for f in os.listdir(".") if os.path.isfile(f)] - for f in files: - if f not in fileToKeep: - os.remove(f) + dirs = [d for d in os.listdir(".") if os.path.isdir(d)] + dirs.insert(0, ".") + for d in dirs: + files = [f for f in os.listdir(d) if os.path.isfile(f)] + playlist_file = next((f for f in files if f.endswith(playlist_file_extension)), None) + if playlist_file is None: continue + logger.debug(f"Removing from {d}") + playlist_data = playlist_import(playlist_file) + if playlist_data is None: continue + for f in files: + if not f.endswith(playlist_file_extension) and not f.endswith(playlist_file_extension + ".map") and f not in fileToKeep and f not in playlist_data: + logger.info(f"Deleting {f}") + os.remove(os.path.join(d, f)) def sync(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist_info, **kwargs): """ From 0471b70d3ff2ba9be471b3e8b4c0b731853390db Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 17:32:33 +0200 Subject: [PATCH 26/44] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 659b0174..5de945bf 100755 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ scdl -l https://soundcloud.com/pandadub/sets/the-lost-ship --sync archive.txt # Download your likes (with authentification token) scdl me -f + +# Download your likes and tracks, generate playlist files +scdl me -f -t --playlist-file ``` ## Options: @@ -87,6 +90,10 @@ scdl me -f --auth-token [token] Specify the auth token to use --overwrite Overwrite file if it already exists --strict-playlist Abort playlist downloading if one track fails to download +--playlist-file Generate m3u playlist files (and additionally check them when used with --remove) +--playlist-file-retain Retain corrupted items +--playlist-file-name Specify playlist file name without extension +--playlist-file-extension Specify extension to playlist file ``` From f0ef4189a4b00b5fd70cd1a27858881bdee3a462 Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 23:08:51 +0200 Subject: [PATCH 27/44] set_metadata: Add error handling to fallback artwork download --- scdl/scdl.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index e9ed8259..2bb01a86 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -920,16 +920,19 @@ def set_metadata(track: BasicTrack, filename: str, playlist_info=None, **kwargs) ): response = None except Exception: - pass - if response is None: - new_artwork_url = artwork_url.replace("large", "t500x500") - response = requests.get(new_artwork_url, stream=True) - if response.headers["Content-Type"] not in ( - "image/png", - "image/jpeg", - "image/jpg", - ): response = None + try: + if response is None: + new_artwork_url = artwork_url.replace("large", "t500x500") + response = requests.get(new_artwork_url, stream=True) + if response.headers["Content-Type"] not in ( + "image/png", + "image/jpeg", + "image/jpg", + ): + response = None + except Exception: + response = None if response is None: logger.error(f"Could not get cover art at {new_artwork_url}") with tempfile.NamedTemporaryFile() as out_file: @@ -958,7 +961,7 @@ def set_metadata(track: BasicTrack, filename: str, playlist_info=None, **kwargs) ) elif mutagen_file.__class__ == mutagen.mp4.MP4: mutagen_file["\xa9cmt"] = track.description - if response: + if response is not None: if mutagen_file.__class__ == mutagen.flac.FLAC: p = mutagen.flac.Picture() p.data = out_file.read() From eba1eea81e4189efb6cabaacf3576582cc10f34b Mon Sep 17 00:00:00 2001 From: resmh Date: Thu, 8 Sep 2022 23:57:45 +0200 Subject: [PATCH 28/44] download_track: Add BaseException handler --- scdl/scdl.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 2bb01a86..d2cf5178 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -820,6 +820,10 @@ def download_track(client: SoundCloud, track: BasicTrack, playlist_info=None, ex logger.error(err) return False + except BaseException as err: + logger.error(err) + return False + def can_convert(filename): ext = os.path.splitext(filename)[1] return "wav" in ext or "aif" in ext From b26911c7644fac43f91c9e8d80d80b33ac6f16bb Mon Sep 17 00:00:00 2001 From: resmh Date: Fri, 9 Sep 2022 12:00:30 +0200 Subject: [PATCH 29/44] download_playlist: Add BaseException handler --- scdl/scdl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index d2cf5178..a1386adb 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -543,6 +543,9 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist playlist_filename=playlist_filename_prefix + playlist_name + ".m3u8" playlist_process(client, playlistbuffer, playlist_filename, **kwargs) if subplaylist_buffer: subplaylist_buffer.append({ "id": playlist.id, "path": playlist_filename, "uri": playlist.uri }) + except BaseException as err: + logger.error(err) + return False finally: if not kwargs.get("no_playlist_folder"): os.chdir("..") From 457819e05919ca347caba6e8a1bba2a736cfc1e4 Mon Sep 17 00:00:00 2001 From: resmh Date: Fri, 9 Sep 2022 13:00:41 +0200 Subject: [PATCH 30/44] Update scdl.cfg for consistent track names --- scdl/scdl.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scdl/scdl.cfg b/scdl/scdl.cfg index fc693206..ed451a30 100644 --- a/scdl/scdl.cfg +++ b/scdl/scdl.cfg @@ -3,7 +3,7 @@ client_id = a3e059563d7fd3372b49b37f00a00bcf auth_token = path = . name_format = {title} -playlist_name_format = {playlist[title]}_{title} +playlist_name_format = {title} # example name_format values: # {timestamp}_{user[username]}_{title} @@ -13,4 +13,4 @@ playlist_name_format = {playlist[title]}_{title} # playlist_name_format playlist attributes: # playlist[author] - username of playlist author -# playlist[title] - name of playlist \ No newline at end of file +# playlist[title] - name of playlist From e25080d68edc5be909dc9aaa5a703367783d3765 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:52:15 +0200 Subject: [PATCH 31/44] remove_files: Consider all playlists within the folder --- scdl/scdl.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index a1386adb..15bd7bf2 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -431,10 +431,12 @@ def remove_files(check_playlist_file, playlist_file_extension): dirs.insert(0, ".") for d in dirs: files = [f for f in os.listdir(d) if os.path.isfile(f)] - playlist_file = next((f for f in files if f.endswith(playlist_file_extension)), None) - if playlist_file is None: continue + playlist_data = [] + for plfile in files: + if plfile.endswith(playlist_file_extension): playlist_data += playlist_import(plfile) + if len(playlist_data) == 0: continue logger.debug(f"Removing from {d}") - playlist_data = playlist_import(playlist_file) + if playlist_data is None: continue for f in files: if not f.endswith(playlist_file_extension) and not f.endswith(playlist_file_extension + ".map") and f not in fileToKeep and f not in playlist_data: From b2ee1076434a92fbea99dfa90579b4d8be89b6a2 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 13:28:50 +0200 Subject: [PATCH 32/44] Add --playlist-file-cache parameter to skip updates and metadata retrieval of present files --- scdl/scdl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 15bd7bf2..97cf4495 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -13,6 +13,7 @@ [--strict-playlist][--playlist-name-format ][--client-id ] [--auth-token ][--overwrite][--no-playlist][--playlist-file] [--playlist-file-retain][--playlist-file-name][--playlist-file-extension] + [--playlist-file-cache] scdl -h | --help scdl --version @@ -69,6 +70,7 @@ --playlist-file-retain Retain corrupted items --playlist-file-name Specify playlist file name without extension --playlist-file-extension Specify extension to playlist file + --playlist-file-cache Skip updates for present files """ import cgi From 787b2073a094bc0bf15dddf2a52604912b9cc651 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:18:01 +0200 Subject: [PATCH 33/44] playlist_process: Also generate map if playlist-file-cache set --- scdl/scdl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index 97cf4495..f6a0a632 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1076,6 +1076,8 @@ def playlist_process(client: SoundCloud, playlist_buffer, playlist_filename, no_ playlist_buffer.insert(newindex, oldel) logger.debug(f"New Map: {playlist_buffer}") + + if kwargs.get("playlist_file_retain") or (kwargs.get("playlist_file_cache") and no_retain == False): playlist_map_write(playlist_buffer, playlist_filename) if no_export == False: From 5220be6c02d3756caf18fd29d572f17a53088c61 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:15:05 +0200 Subject: [PATCH 34/44] download_track_cached: Lookup item path by item id and check file presence before calling download_track --- scdl/scdl.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scdl/scdl.py b/scdl/scdl.py index f6a0a632..c3ef1a6d 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1137,5 +1137,14 @@ def check_item(client: SoundCloud, itemuri): else: return False +def download_track_cached(client: SoundCloud, track: BasicTrack, playlist_cache=None, playlist_info=None, exit_on_fail=True, playlist_buffer=None, **kwargs): + if playlist_cache is not None and playlist_buffer is not None: + cacheres = next((cached for cached in playlist_cache if cached["id"] == str(track.id)), None) + if cacheres is None or cacheres["path"] is None: return False + if os.path.isfile(cacheres["path"]): + playlist_buffer.append({ "id": track.id, "path": cacheres["path"], "uri": track.uri }) + return True + return download_track(client, track, playlist_info=playlist_info, exit_on_fail=exit_on_fail, playlist_buffer=playlist_buffer, **kwargs) + if __name__ == "__main__": main() From ba446679eb7252cf3c43f981e237eb6c6b562033 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:11:30 +0200 Subject: [PATCH 35/44] download_url: Call download_track_cached for user likes --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index c3ef1a6d..56e6c465 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -333,6 +333,7 @@ def download_url(client: SoundCloud, **kwargs): sys.exit(1) if kwargs.get("f"): logger.info(f"Retrieving all likes of user {user.username}...") + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Likes", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) playlistbuffer=None if not kwargs.get("playlist_file") else [] subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_likes(user.id, limit=int(kwdefget("n", "1000", **kwargs))) @@ -340,7 +341,7 @@ def download_url(client: SoundCloud, **kwargs): for i, like in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"like n°{i} of {user.likes_count}") if hasattr(like, "track"): - download_track(client, like.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, like.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) elif hasattr(like, "playlist"): download_playlist(client, client.get_playlist(like.playlist.id), playlist_filename_prefix=kwdefget("playlist_file_name", "Likes", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: From 5842903155c5bee00bfceb519218200b66b5f0b8 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:53:53 +0200 Subject: [PATCH 36/44] download_url: Call download_track_cached for user commented tracks --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 56e6c465..66ddc42a 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -356,11 +356,12 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all likes of user {user.username}!") if kwargs.get("C"): logger.info(f"Retrieving all commented tracks of user {user.username}...") + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Commented", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) playlistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_comments(user.id, limit=1000) for i, comment in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"comment n°{i} of {user.comments_count}") - download_track(client, client.get_track(comment.track.id), exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, client.get_track(comment.track.id), exit_on_fail=kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) if kwargs.get("playlist_file"): playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Commented", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) logger.info(f"Downloaded all commented tracks of user {user.username}!") if kwargs.get("t"): From b52d8dbb6943354b154185050a9ecfc95cfb9d67 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:55:11 +0200 Subject: [PATCH 37/44] download_url: Call download_track_cached for user tracks --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 66ddc42a..69f6fac0 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -367,10 +367,11 @@ def download_url(client: SoundCloud, **kwargs): if kwargs.get("t"): logger.info(f"Retrieving all tracks of user {user.username}...") playlistbuffer=None if not kwargs.get("playlist_file") else [] + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Tracks", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) resources = client.get_user_tracks(user.id, limit=1000) for i, track in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"track n°{i} of {user.track_count}") - download_track(client, track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, track, exit_on_fail=kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) if kwargs.get("playlist_file"): playlist_process(client, playlistbuffer, kwdefget("playlist_file_name", "Tracks", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs), **kwargs) logger.info(f"Downloaded all tracks of user {user.username}!") if kwargs.get("a"): From 69485af8b5ba2a79287ff4a7862c9ae2b786f89d Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:55:48 +0200 Subject: [PATCH 38/44] download_url: Call download_track_cached for user stream --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 69f6fac0..7d108b81 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -376,13 +376,14 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all tracks of user {user.username}!") if kwargs.get("a"): logger.info(f"Retrieving all tracks & reposts of user {user.username}...") + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Stream", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) playlistbuffer=None if not kwargs.get("playlist_file") else [] subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_stream(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"item n°{i} of {user.track_count + user.reposts_count if user.reposts_count else '?'}") if item.type in ("track", "track-repost"): - download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) elif item.type in ("playlist", "playlist-repost"): download_playlist(client, item.playlist, kwdefget("playlist_file_name", "Stream", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: From bfc8d8d1d250eece88241411ce5ca5ff4c9eddfb Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:56:29 +0200 Subject: [PATCH 39/44] download_url: Call download_track_cached for user reposts --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 7d108b81..9757886a 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -406,13 +406,14 @@ def download_url(client: SoundCloud, **kwargs): logger.info(f"Downloaded all playlists of user {user.username}!") if kwargs.get("r"): logger.info(f"Retrieving all reposts of user {user.username}...") + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Reposts", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) playlistbuffer=None if not kwargs.get("playlist_file") else [] subplaylistbuffer=None if not kwargs.get("playlist_file") else [] resources = client.get_user_reposts(user.id, limit=1000) for i, item in itertools.islice(enumerate(resources, 1), offset, None): logger.info(f"item n°{i} of {user.reposts_count or '?'}") if item.type == "track-repost": - download_track(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, item.track, exit_on_fail=kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) elif item.type == "playlist-repost": download_playlist(client, item.playlist, kwdefget("playlist_file_name", "Reposts", **kwargs) + " - ", subplaylist_buffer=subplaylistbuffer, **kwargs) else: From e9b746026efc13e64f24cd031691acab4e9252ba Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 14:57:08 +0200 Subject: [PATCH 40/44] download_url: Call download_track_cached for playlists --- scdl/scdl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 9757886a..53664932 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -537,6 +537,7 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist tracknumber_digits = len(str(len(playlist.tracks))) playlistbuffer=None if not kwargs.get("playlist_file") else [] + playlistcache=None if not kwargs.get("playlist_file_cache") else playlist_map_read(kwdefget("playlist_file_name", "Likes", **kwargs) + "." + kwdefget("playlist_file_extension", "m3u8", **kwargs)) for counter, track in itertools.islice(enumerate(playlist.tracks, 1), kwargs.get("playlist_offset", 0), None): logger.debug(track) logger.info(f"Track n°{counter}") @@ -547,7 +548,7 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist else: track = client.get_track(track.id) - download_track(client, track, playlist_info, kwargs.get("strict_playlist"), playlist_buffer=playlistbuffer, **kwargs) + download_track_cached(client, track, playlist_info, kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) if kwargs.get("playlist_file"): playlist_filename=playlist_filename_prefix + playlist_name + ".m3u8" playlist_process(client, playlistbuffer, playlist_filename, **kwargs) From c1fa44e10a042516b4edde47699d2dfbf3ca1015 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 15:04:24 +0200 Subject: [PATCH 41/44] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5de945bf..3a217f4f 100755 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ scdl -l https://soundcloud.com/pandadub/sets/the-lost-ship --sync archive.txt # Download your likes (with authentification token) scdl me -f -# Download your likes and tracks, generate playlist files -scdl me -f -t --playlist-file +# Download your likes and tracks, generate playlist files and use the cache for speedup +scdl me -f -t --playlist-file --playlist-file-cache ``` ## Options: @@ -94,6 +94,7 @@ scdl me -f -t --playlist-file --playlist-file-retain Retain corrupted items --playlist-file-name Specify playlist file name without extension --playlist-file-extension Specify extension to playlist file +--playlist-file-cache Skip updates for present files ``` From d9797de40fb04a8051f70ef52518bc36a5407f50 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 15:47:35 +0200 Subject: [PATCH 42/44] playlist_import: Adjust filter for extended m3u attributes --- scdl/scdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 53664932..62c9167c 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1113,7 +1113,7 @@ def playlist_import(playlist_filename): if not os.path.isfile(playlist_filename): return None with open(playlist_filename, "r") as fin: for fline in fin.read().splitlines(): - if not fline.startswith("#"): res.append(fline) + if not fline.startswith("#EXT") or fline == sanitize_filename(fline): res.append(fline) return res except Exception: From 6ebf3cef359d2d989bc0171662a3bca1499968f7 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 15:56:26 +0200 Subject: [PATCH 43/44] download_track_cached: Adjust signature and logic --- scdl/scdl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 62c9167c..25824aef 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -1143,11 +1143,10 @@ def check_item(client: SoundCloud, itemuri): else: return False -def download_track_cached(client: SoundCloud, track: BasicTrack, playlist_cache=None, playlist_info=None, exit_on_fail=True, playlist_buffer=None, **kwargs): +def download_track_cached(client: SoundCloud, track: BasicTrack, playlist_info=None, exit_on_fail=True, playlist_cache=None, playlist_buffer=None, **kwargs): if playlist_cache is not None and playlist_buffer is not None: cacheres = next((cached for cached in playlist_cache if cached["id"] == str(track.id)), None) - if cacheres is None or cacheres["path"] is None: return False - if os.path.isfile(cacheres["path"]): + if cacheres is not None and cacheres["path"] is not None and os.path.isfile(cacheres["path"]): playlist_buffer.append({ "id": track.id, "path": cacheres["path"], "uri": track.uri }) return True return download_track(client, track, playlist_info=playlist_info, exit_on_fail=exit_on_fail, playlist_buffer=playlist_buffer, **kwargs) From f8ec83582363d82b6221c39ee8f00b34eddbcd92 Mon Sep 17 00:00:00 2001 From: resmh Date: Wed, 14 Sep 2022 16:29:34 +0200 Subject: [PATCH 44/44] download_playlist: Adjust playlist generator indentation --- scdl/scdl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scdl/scdl.py b/scdl/scdl.py index 25824aef..25a7751b 100755 --- a/scdl/scdl.py +++ b/scdl/scdl.py @@ -549,10 +549,10 @@ def download_playlist(client: SoundCloud, playlist: BasicAlbumPlaylist, playlist track = client.get_track(track.id) download_track_cached(client, track, playlist_info, kwargs.get("strict_playlist"), playlist_cache=playlistcache, playlist_buffer=playlistbuffer, **kwargs) - if kwargs.get("playlist_file"): - playlist_filename=playlist_filename_prefix + playlist_name + ".m3u8" - playlist_process(client, playlistbuffer, playlist_filename, **kwargs) - if subplaylist_buffer: subplaylist_buffer.append({ "id": playlist.id, "path": playlist_filename, "uri": playlist.uri }) + if kwargs.get("playlist_file"): + playlist_filename=playlist_filename_prefix + playlist_name + ".m3u8" + playlist_process(client, playlistbuffer, playlist_filename, **kwargs) + if subplaylist_buffer: subplaylist_buffer.append({ "id": playlist.id, "path": playlist_filename, "uri": playlist.uri }) except BaseException as err: logger.error(err) return False