From 7f642263698a923d3e48b4f933bda743ab80c9ce Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 7 Feb 2025 14:58:43 -0500 Subject: [PATCH 1/6] Add more keys to filter - `heatmap`: is a huge list of tiny "chapters" used to display popularity of sections of the video - `requested_subtitles`: also has a URL that expires --- tubesync/sync/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 2f9e6593..791bfe6d 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -282,12 +282,18 @@ def filter_response(arg_dict, copy_arg=False): ) ) - for key in frozenset(('subtitles', 'automatic_captions',)): + for key in frozenset(('subtitles', 'requested_subtitles', 'automatic_captions',)): if key in response_dict.keys(): key_dict = response_dict[key] for lang_code in key_dict: _drop_url_keys(key_dict, lang_code, drop_subtitles_url) # end of subtitles cleanup }}} + + # beginning of heatmap cleanup {{{ + for key in frozenset(('heatmap',)): + if key in response_dict.keys(): + del response_dict[key] + # end of heatmap cleanup }}} return response_dict From cfec87679600283e7626ee5c3292d2dede952a40 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Feb 2025 14:14:56 -0500 Subject: [PATCH 2/6] Adjust for `requested_subtitles` layout --- tubesync/sync/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 791bfe6d..e536f57d 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -282,11 +282,15 @@ def filter_response(arg_dict, copy_arg=False): ) ) - for key in frozenset(('subtitles', 'requested_subtitles', 'automatic_captions',)): + for key in frozenset(('subtitles', 'automatic_captions',)): if key in response_dict.keys(): key_dict = response_dict[key] for lang_code in key_dict: _drop_url_keys(key_dict, lang_code, drop_subtitles_url) + + for key in frozenset(('requested_subtitles',)): + if key in response_dict.keys(): + _drop_url_keys(response_dict, key, drop_subtitles_url) # end of subtitles cleanup }}} # beginning of heatmap cleanup {{{ From 8bff3b6c24fa10fe860e4979ab5aa013dde5ddf7 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Feb 2025 16:02:38 -0500 Subject: [PATCH 3/6] More stringent checking of types --- tubesync/sync/utils.py | 55 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index e536f57d..6a745c45 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -205,20 +205,26 @@ def normalize_codec(codec_str): def _url_keys(arg_dict, filter_func): result = {} - for key in arg_dict.keys(): - if 'url' in key: - result.update( - {key: filter_func(key=key, url=arg_dict[key])} - ) + if isinstance(arg_dict, dict): + for key, value in arg_dict.items(): + if 'url' in key: + result.update( + {key: filter_func(key=key, url=value)} + ) return result def _drop_url_keys(arg_dict, key, filter_func): + assert isinstance(arg_dict, dict) if key in arg_dict.keys(): - for val_dict in arg_dict[key]: - for url_key, remove in _url_keys(val_dict, filter_func).items(): - if remove is True: - del val_dict[url_key] + key_list = arg_dict[key] + assert isinstance(key_list, list) + if isinstance(key_list, list): + for val_dict in key_list: + assert isinstance(val_dict, dict) + for url_key, remove in _url_keys(val_dict, filter_func).items(): + if remove is True: + del val_dict[url_key] def filter_response(arg_dict, copy_arg=False): @@ -260,13 +266,16 @@ def filter_response(arg_dict, copy_arg=False): '__needs_testing', '__working', )) - for key in frozenset(('formats', 'requested_formats',)): - _drop_url_keys(response_dict, key, drop_format_url) + for key in ('formats', 'requested_formats',): if key in response_dict.keys(): - for format in response_dict[key]: - for drop_key in drop_keys: - if drop_key in format.keys(): - del format[drop_key] + _drop_url_keys(response_dict, key, drop_format_url) + formats = response_dict[key] + assert isinstance(formats, list) + if isinstance(formats, list): + for format in formats: + for drop_key in drop_keys: + if drop_key in format.keys(): + del format[drop_key] # end of formats cleanup }}} # beginning of subtitles cleanup {{{ @@ -282,19 +291,17 @@ def filter_response(arg_dict, copy_arg=False): ) ) - for key in frozenset(('subtitles', 'automatic_captions',)): + for key in ('subtitles', 'requested_subtitles', 'automatic_captions',): if key in response_dict.keys(): - key_dict = response_dict[key] - for lang_code in key_dict: - _drop_url_keys(key_dict, lang_code, drop_subtitles_url) - - for key in frozenset(('requested_subtitles',)): - if key in response_dict.keys(): - _drop_url_keys(response_dict, key, drop_subtitles_url) + lang_codes = response_dict[key] + assert isinstance(lang_codes, dict) + if isinstance(lang_codes, dict): + for lang_code in lang_codes.keys(): + _drop_url_keys(lang_codes, lang_code, drop_subtitles_url) # end of subtitles cleanup }}} # beginning of heatmap cleanup {{{ - for key in frozenset(('heatmap',)): + for key in ('heatmap',): if key in response_dict.keys(): del response_dict[key] # end of heatmap cleanup }}} From 2205f08124344d64f0702d7f5efc310aae54a9be Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Feb 2025 16:11:45 -0500 Subject: [PATCH 4/6] Remove the assert for `lang_codes` The test data doesn't conform to this expectation. --- tubesync/sync/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 6a745c45..87d95285 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -294,7 +294,6 @@ def filter_response(arg_dict, copy_arg=False): for key in ('subtitles', 'requested_subtitles', 'automatic_captions',): if key in response_dict.keys(): lang_codes = response_dict[key] - assert isinstance(lang_codes, dict) if isinstance(lang_codes, dict): for lang_code in lang_codes.keys(): _drop_url_keys(lang_codes, lang_code, drop_subtitles_url) From 208a540bc95d0afd49b8e334fe0bac800917c082 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Feb 2025 19:59:00 -0500 Subject: [PATCH 5/6] Add and use `list_of_dictionaries` function --- tubesync/sync/utils.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 87d95285..7afb8067 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -203,6 +203,17 @@ def normalize_codec(codec_str): return result +def list_of_dictionaries(arg_list, arg_function=lambda x: x): + assert callable(arg_function) + if isinstance(arg_list, list): + def _assert_and_call(arg_dict): + assert isinstance(arg_dict, dict) + if isinstance(arg_dict, dict): + return arg_function(arg_dict) + return (True, list(map(_assert_and_call, arg_list)),) + return (False, arg_list,) + + def _url_keys(arg_dict, filter_func): result = {} if isinstance(arg_dict, dict): @@ -214,17 +225,17 @@ def _url_keys(arg_dict, filter_func): return result +# expects a dictionary where the value at key is a: +# list of dictionaries def _drop_url_keys(arg_dict, key, filter_func): + def _del_url_keys(_arg_dict): + for url_key, remove in _url_keys(_arg_dict, filter_func).items(): + if remove is True: + del _arg_dict[url_key] + assert isinstance(arg_dict, dict) if key in arg_dict.keys(): - key_list = arg_dict[key] - assert isinstance(key_list, list) - if isinstance(key_list, list): - for val_dict in key_list: - assert isinstance(val_dict, dict) - for url_key, remove in _url_keys(val_dict, filter_func).items(): - if remove is True: - del val_dict[url_key] + list_of_dictionaries(arg_dict[key], _del_url_keys) def filter_response(arg_dict, copy_arg=False): @@ -266,16 +277,15 @@ def filter_response(arg_dict, copy_arg=False): '__needs_testing', '__working', )) + def del_drop_keys(arg_dict): + for drop_key in drop_keys: + if drop_key in arg_dict.keys(): + del arg_dict[drop_key] + for key in ('formats', 'requested_formats',): if key in response_dict.keys(): _drop_url_keys(response_dict, key, drop_format_url) - formats = response_dict[key] - assert isinstance(formats, list) - if isinstance(formats, list): - for format in formats: - for drop_key in drop_keys: - if drop_key in format.keys(): - del format[drop_key] + list_of_dictionaries(response_dict[key], del_drop_keys) # end of formats cleanup }}} # beginning of subtitles cleanup {{{ From e58fae1f9693eba4d43062fdd85088c6ad2e0f17 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 8 Feb 2025 20:17:53 -0500 Subject: [PATCH 6/6] Skip over items that are not dictionaries --- tubesync/sync/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index 7afb8067..fa904c5f 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -207,9 +207,9 @@ def list_of_dictionaries(arg_list, arg_function=lambda x: x): assert callable(arg_function) if isinstance(arg_list, list): def _assert_and_call(arg_dict): - assert isinstance(arg_dict, dict) if isinstance(arg_dict, dict): return arg_function(arg_dict) + return arg_dict return (True, list(map(_assert_and_call, arg_list)),) return (False, arg_list,)