From 2c4765c7f1c808c7af92020be7a86222ab0be5be Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 11:07:46 -0400 Subject: [PATCH 01/56] Replace `download_media_metadata` in tasks.py --- tubesync/sync/tasks.py | 160 ++++++++--------------------------------- 1 file changed, 31 insertions(+), 129 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 4cc81d6c..3e1b2262 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -46,46 +46,33 @@ db_vendor = db.connection.vendor register_huey_signals() -def get_hash(task_name, pk): - ''' - Create a background_task compatible hash for a Task or CompletedTask. - ''' - task_params = json.dumps(((str(pk),), {}), sort_keys=True) - return sha1(f'{task_name}{task_params}'.encode('utf-8')).hexdigest() - - -def map_task_to_instance(task, using_history=True): +def map_task_to_instance(task): ''' Reverse-maps a scheduled backgrond task to an instance. Requires the task name to be a known task function and the first argument to be a UUID. This is used because UUID's are incompatible with background_task's "creator" feature. ''' TASK_MAP = { - 'sync.tasks.index_source_task': Source, + 'sync.tasks.index_source': Source, 'sync.tasks.download_media_image': Media, 'sync.tasks.download_media_file': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, 'sync.tasks.rename_all_media_for_source': Source, - 'sync.tasks.wait_for_media_premiere': Media, } MODEL_URL_MAP = { Source: 'sync:source', Media: 'sync:media-item', } # Unpack - task_func = task.name if using_history else task.task_name + task_func = task.name model = TASK_MAP.get(task_func, None) if not model: return None, None url = MODEL_URL_MAP.get(model, None) if not url: return None, None - task_args = task.task_params if using_history else None - try: - task_args = task_args or json.loads(task.task_params) - except (TypeError, ValueError, AttributeError): - return None, None + task_args = task.task_params if len(task_args) != 2: return None, None args, kwargs = task_args @@ -142,86 +129,58 @@ def update_task_status(task, status): def get_source_completed_tasks(source_id, only_errors=False): ''' - Returns a queryset of CompletedTask objects for a source by source ID. + Returns a queryset of TaskHistory objects for a source by source ID. ''' - q = {'task_params__istartswith': f'[["{source_id}"'} + qs = get_model_tasks(source_id) if only_errors: - q['failed_at__isnull'] = False - return CompletedTask.objects.filter(**q).order_by('-failed_at') + qs = qs.filter(failed_at__isnull=False) + return qs.order_by('-failed_at') -def get_model_task(model_pk, /, name=None, qs=None): +def get_model_tasks(model_pk, /, name=None, qs=None): if qs is None: qs = TaskHistory.objects.all() if name is not None: qs = qs.filter(name__endswith=name) - params_prefix = f'[["{model_pk}"' - qs = qs.filter(task_params__istartswith=params_prefix) - return qs[0] if qs.count() else False + #return qs.filter(task_params__0__0=model_pk) + return qs.filter(task_params__istartswith=f'[["{model_pk}"') def get_running_tasks(arg_dt=None, /): + max_run_time = getattr(settings, 'MAX_RUN_TIME', 3600) return TaskHistory.objects.running( now=arg_dt, - within=timezone.timedelta(seconds=settings.MAX_RUN_TIME), + within=timezone.timedelta(seconds=max_run_time), ) -def get_running_task_by_name(arg_str, media_id, /): +def get_running_task_by_name(arg_str, instance_id, /): name = arg_str if '.' not in name: name = f'sync.tasks.{name}' - tqs = get_running_tasks().filter(name=name, task_params__0__0=media_id) + tqs = get_model_tasks(instance_id, qs=get_running_tasks()) + tqs = tqs.filter(name=name) return tqs[0] if tqs.count() else False def get_media_download_task(media_id): - #return get_running_task_by_name('download_media_file', media_id) - return get_model_task( - media_id, - name='download_media_file', - qs=get_running_tasks(), - ) + return get_running_task_by_name('download_media_file', media_id) def get_media_thumbnail_task(media_id): - #return get_running_task_by_name('download_media_image', media_id) - return get_model_task( - media_id, - name='download_media_image', - qs=get_running_tasks(), - ) + return get_running_task_by_name('download_media_image', media_id) def get_tasks(task_name, id=None, /, instance=None): assert not (id is None and instance is None) arg = str(id or instance.pk) - return Task.objects.get_task(str(task_name), args=(arg,),) + return get_running_task_by_name(str(task_name), arg) def get_first_task(task_name, id=None, /, *, instance=None): - tqs = get_tasks(task_name, id, instance).order_by('run_at') + tqs = get_tasks(task_name, id, instance).order_by('scheduled_at') return tqs[0] if tqs.count() else False def get_media_metadata_task(media_id): return get_first_task('sync.tasks.download_media_metadata', media_id) def get_source_index_task(source_id): - return get_first_task('sync.tasks.index_source_task', source_id) - - -def delete_task_by_source(task_name, source_id): - now = timezone.now() - unlocked = Task.objects.unlocked(now) - qs = unlocked.filter( - task_name=task_name, - task_params__istartswith=f'[["{source_id}"', - ) - return qs.delete() - - -def delete_task_by_media(task_name, args): - max_run_time = getattr(settings, 'MAX_RUN_TIME', 3600) - now = timezone.now() - expires_at = now - timedelta(seconds=max_run_time) - task_qs = Task.objects.get_task(task_name, args=args) - unlocked = task_qs.filter(locked_by=None) | task_qs.filter(locked_at__lt=expires_at) - return unlocked.delete() + return get_first_task('sync.tasks.index_source', source_id) def cleanup_completed_tasks(): @@ -229,19 +188,9 @@ def cleanup_completed_tasks(): delta = timezone.now() - timedelta(days=days_to_keep) log.info(f'Deleting completed tasks older than {days_to_keep} days ' f'(run_at before {delta})') - CompletedTask.objects.filter(run_at__lt=delta).delete() TaskHistory.objects.filter(end_at__lt=delta).delete() -@atomic(durable=False) -def migrate_queues(): - return Task.objects.exclude( - queue=Val(TaskQueue.NET) - ).update( - queue=Val(TaskQueue.NET) - ) - - def save_model(instance): with atomic(durable=False): instance.save() @@ -274,19 +223,6 @@ def upcoming_media(): valid, hours = media.wait_for_premiere() if valid: save_model(media) - task = get_first_task('sync.tasks.wait_for_media_premiere', media_id) - if not task: - # create a task to update - when = media.published + timezone.timedelta(minutes=1) - vn_fmt = _('Waiting for the premiere of "{}" at: {}') - vn = vn_fmt.format( - media.key, - media.published.isoformat(' ', 'seconds'), - ) - wait_for_media_premiere(media_id, run_at=when, verbose_name=vn) - task = get_first_task('sync.tasks.wait_for_media_premiere', media_id) - if hours: - update_task_status(task, f'available in {hours} hours') log.debug(f'upcoming_media: wait_for_premiere: {media.key}: {valid=} {hours=}') @@ -352,7 +288,7 @@ def contains_http429(q, task_id, /): def wait_for_errors(model, /, *, queue_name=None, task_name=None): if task_name is None: task_name=tuple(( - 'sync.tasks.download_media', + 'sync.tasks.download_media_file', 'sync.tasks.download_media_metadata', )) elif isinstance(task_name, str): @@ -362,18 +298,10 @@ def wait_for_errors(model, /, *, queue_name=None, task_name=None): ft = get_first_task(tn, instance=model) if ft: tasks.append(ft) - window = timezone.timedelta(hours=3) + timezone.now() - tqs = Task.objects.filter( - task_name__in=task_name, - attempts__gt=0, - locked_at__isnull=True, - run_at__lte=window, - last_error__contains='HTTPError 429: Too Many Requests', - ) for task in tasks: update_task_status(task, 'paused (429)') - total_count = tqs.count() + total_count = int() if queue_name: from django_huey import get_queue q = get_queue(queue_name) @@ -644,11 +572,13 @@ def index_source(source_id): delay=65-(30*num), ) log.info(f'Scheduling task to download metadata for: {media.url}') - verbose_name = _('Downloading metadata for: "{}": {}') - download_media_metadata( + TaskHistory.schedule( + download_media_metadata, str(media.pk), - schedule=dict(priority=35), - verbose_name=verbose_name.format(media.key, media.name), + priority=65, + remove_duplicates=True, + vn_fmt=_('Downloading metadata for: "{}": {}'), + vn_args=(media.key, media.name,), ) # Reset task.verbose_name to the saved value update_task_status(task, None) @@ -801,7 +731,7 @@ def save_media(media_id): @db_task(delay=60, priority=60, retries=3, retry_delay=600, queue=Val(TaskQueue.LIMIT)) -def download_metadata(media_id): +def download_media_metadata(media_id): ''' Downloads the metadata for a media item. ''' @@ -814,7 +744,7 @@ def download_metadata(media_id): raise CancelExecution(_('no such media'), retry=False) from e if media.manual_skip: log.info(f'Task for ID: {media_id} / {media} skipped, due to task being manually skipped.') - return False + return source = media.source wait_for_errors( media, @@ -864,7 +794,6 @@ def download_metadata(media_id): if raise_exception: raise log.debug(str(e)) - return False else: keep_metadata_lock = True finally: @@ -907,7 +836,6 @@ def download_metadata(media_id): else: log.info(f'Saved {len(media.metadata_dumps())} bytes of metadata for: ' f'{source} / {media}: {media_id}') - return True finally: metadata_lock.acquired = False @@ -1261,29 +1189,3 @@ def delete_all_media_for_source(source_id, source_name, source_directory): rmtree(directory_path, True) -# Old tasks system -from background_task import background # noqa: E402 -from background_task.exceptions import InvalidTaskError # noqa: E402 -from background_task.models import Task, CompletedTask # noqa: E402 - - -@background(schedule=dict(priority=0, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def wait_for_media_premiere(media_id): - try: - media = Media.objects.get(pk=media_id) - except Media.DoesNotExist as e: - raise InvalidTaskError(_('no such media')) from e - else: - t = media.wait_for_premiere() - if t[0]: - save_model(media) - -@background(schedule=dict(priority=40, run_at=60), queue=Val(TaskQueue.NET), remove_existing_tasks=True) -def download_media_metadata(media_id): - try: - res = download_metadata(media_id) - return res.get(blocking=True) - except CancelExecution as e: - raise InvalidTaskError(str(e)) from e - - From 353baf4a228e3c38ba50ccba08696b657259e6e1 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 11:13:54 -0400 Subject: [PATCH 02/56] fixup: remove unused imports --- tubesync/sync/tasks.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 3e1b2262..1728e177 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -5,14 +5,12 @@ import os -import json import random import requests import time import uuid from collections import deque as queue from io import BytesIO -from hashlib import sha1 from pathlib import Path from datetime import timedelta from shutil import copyfile, rmtree @@ -219,7 +217,6 @@ def upcoming_media(): ), ) for media in qs_gen(qs): - media_id = str(media.pk) valid, hours = media.wait_for_premiere() if valid: save_model(media) From e1b51edeab62bc5eb3e134ded555fdb70636452b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 11:25:39 -0400 Subject: [PATCH 03/56] Remove outdated function calls in views.py --- tubesync/sync/views.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 26fd1644..daa3da27 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -31,7 +31,7 @@ from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMedia ConfirmDeleteMediaServerForm, SourceForm) from .utils import delete_file, validate_url from .tasks import ( - map_task_to_instance, get_error_message, migrate_queues, delete_task_by_media, + map_task_to_instance, get_error_message, get_running_tasks, get_media_download_task, get_source_completed_tasks, check_source_directory_exists, index_source, download_media_image, ) @@ -662,8 +662,6 @@ class MediaRedownloadView(FormView, SingleObjectMixin): return super().dispatch(request, *args, **kwargs) def form_valid(self, form): - # Delete any active download tasks for the media - delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) # If the thumbnail file exists on disk, delete it if self.object.thumb_file_exists: delete_file(self.object.thumb.path) @@ -714,8 +712,6 @@ class MediaSkipView(FormView, SingleObjectMixin): return super().dispatch(request, *args, **kwargs) def form_valid(self, form): - # Delete any active download tasks for the media - delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) # If the media file exists on disk, delete it if self.object.media_file_exists: # Delete all files which contains filename @@ -886,7 +882,6 @@ class TasksView(ListView): data['total_errors'] = errors_qs.count() data['scheduled'] = list() data['total_scheduled'] = scheduled_qs.count() - data['migrated'] = migrate_queues() data['wait_for_database_queue'] = False def add_to_task(task): From a4f43cdb60c0421a300245bea98a6fbda30bf257 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 11:35:52 -0400 Subject: [PATCH 04/56] Update signals.py --- tubesync/sync/signals.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index b5bd6b3e..046f0724 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -17,7 +17,6 @@ from common.models import TaskHistory from common.utils import glob_quote, mkdir_p from .models import Source, Media, Metadata from .tasks import ( - delete_task_by_media, get_media_download_task, get_media_metadata_task, get_media_thumbnail_task, map_task_to_instance, delete_all_media_for_source, rename_media, save_all_media_for_source, @@ -317,10 +316,12 @@ def media_post_save(sender, instance, created, **kwargs): # If the media is missing metadata schedule it to be downloaded if not (media.skip or media.has_metadata or existing_media_metadata_task): log.info(f'Scheduling task to download metadata for: {media.url}') - verbose_name = _('Downloading metadata for: {}: "{}"') - download_media_metadata( + TaskHistory.schedule( + download_media_metadata, str(media.pk), - verbose_name=verbose_name.format(media.key, media.name), + remove_duplicates=True, + vn_fmt=_('Downloading metadata for: {}: "{}"'), + vn_args=(media.key, media.name,), ) # If the media is missing a thumbnail schedule it to be downloaded (unless we are skipping this media) if not media.thumb_file_exists: @@ -371,9 +372,6 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): - # Triggered before media is deleted, delete any unlocked scheduled tasks - log.info(f'Deleting tasks for media: {instance.name}') - delete_task_by_media('sync.tasks.download_media_metadata', (str(instance.pk),)) # Remove thumbnail file for deleted media if instance.thumb: instance.thumb.delete(save=False) From 838da6278af4cc165a5524cd2282fdf94faa4b5d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 11:47:33 -0400 Subject: [PATCH 05/56] Return a query set --- tubesync/sync/tasks.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1728e177..581e3573 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -150,25 +150,26 @@ def get_running_tasks(arg_dt=None, /): within=timezone.timedelta(seconds=max_run_time), ) -def get_running_task_by_name(arg_str, instance_id, /): +def get_running_tasks_by_name(arg_str, instance_id, /): name = arg_str if '.' not in name: name = f'sync.tasks.{name}' tqs = get_model_tasks(instance_id, qs=get_running_tasks()) - tqs = tqs.filter(name=name) - return tqs[0] if tqs.count() else False + return tqs.filter(name=name) def get_media_download_task(media_id): - return get_running_task_by_name('download_media_file', media_id) + tqs = get_running_tasks_by_name('download_media_file', media_id) + return tqs[0] if tqs.count() else False def get_media_thumbnail_task(media_id): - return get_running_task_by_name('download_media_image', media_id) + tqs = get_running_tasks_by_name('download_media_image', media_id) + return tqs[0] if tqs.count() else False def get_tasks(task_name, id=None, /, instance=None): assert not (id is None and instance is None) arg = str(id or instance.pk) - return get_running_task_by_name(str(task_name), arg) + return get_running_tasks_by_name(str(task_name), arg) def get_first_task(task_name, id=None, /, *, instance=None): tqs = get_tasks(task_name, id, instance).order_by('scheduled_at') From d098bdcf7b432e13e957c05112d4602408dd9297 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:12:42 -0400 Subject: [PATCH 06/56] Delete config/root/etc/s6-overlay/s6-rc.d/user/contents.d/background-task-workers --- .../s6-overlay/s6-rc.d/user/contents.d/background-task-workers | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/user/contents.d/background-task-workers diff --git a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/background-task-workers b/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/background-task-workers deleted file mode 100644 index 8b137891..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/user/contents.d/background-task-workers +++ /dev/null @@ -1 +0,0 @@ - From a1843476f5ec420c2284b31d7eca0035de594ac9 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:16:32 -0400 Subject: [PATCH 07/56] Remove `background_task` from settings.py --- tubesync/tubesync/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index 3360be3a..da2fab5d 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -25,7 +25,6 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django.contrib.humanize', 'sass_processor', - 'background_task', 'django_huey', 'common', 'sync', From 88119c66f641a03fb6dcc757a5b79f8618d9ea84 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:19:18 -0400 Subject: [PATCH 08/56] Delete config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker --- .../background-task-workers/contents.d/tubesync-network-worker | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker deleted file mode 100644 index 8b137891..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/contents.d/tubesync-network-worker +++ /dev/null @@ -1 +0,0 @@ - From d944e4d66bbd77ca23ae79045f754a173ee407d3 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:19:28 -0400 Subject: [PATCH 09/56] Delete config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type --- config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type b/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type deleted file mode 100644 index 757b4221..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/background-task-workers/type +++ /dev/null @@ -1 +0,0 @@ -bundle From 5f65e5b31e67025815286bc88f1fc31901c71e75 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:19:52 -0400 Subject: [PATCH 10/56] Delete config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run --- .../root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run deleted file mode 100755 index 7f7bcd26..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/run +++ /dev/null @@ -1,6 +0,0 @@ -#!/command/with-contenv bash - -exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ - /usr/bin/python3 /app/manage.py process_tasks \ - --queue network --duration 43200 \ - --sleep "10.${RANDOM}" From 4be74ea2ae67918357f489186abb42bf163286c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:20:15 -0400 Subject: [PATCH 11/56] Delete config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies --- .../etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies deleted file mode 100644 index 283e1305..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/dependencies +++ /dev/null @@ -1 +0,0 @@ -gunicorn \ No newline at end of file From 6af77804a8ab56e6bd202bec5fbbd42641d31373 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:20:27 -0400 Subject: [PATCH 12/56] Delete config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal --- .../etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal deleted file mode 100644 index d751378e..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/down-signal +++ /dev/null @@ -1 +0,0 @@ -SIGINT From eaf6b593aabdf6b24e191b164440cbdb4622a8bb Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:20:37 -0400 Subject: [PATCH 13/56] Delete config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type --- config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type b/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type deleted file mode 100644 index 1780f9f4..00000000 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-network-worker/type +++ /dev/null @@ -1 +0,0 @@ -longrun \ No newline at end of file From 67614dc8a7b1b0adef1b570b25b2ff46da7a4790 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:22:47 -0400 Subject: [PATCH 14/56] Update ci.yaml --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c755057..02c10ddd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,7 +119,6 @@ jobs: sudo ln -v -s -f -T ~/.config/TubeSync/config /config sudo ln -v -s -f -T ~/.config/TubeSync/downloads /downloads cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py - cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/* cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/* cd tubesync && python3 -B manage.py collectstatic --no-input --link - name: Check with ruff From a1d929d8c961a2e399be79532b4622ce8f598396 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:29:45 -0400 Subject: [PATCH 15/56] Update Dockerfile --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f509ab8b..32361d10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -514,10 +514,6 @@ RUN --mount=type=tmpfs,target=/cache \ # Copy root COPY config/root / -# patch background_task -COPY patches/background_task/ \ - /usr/local/lib/python3/dist-packages/background_task/ - # patch yt_dlp COPY patches/yt_dlp/ \ /usr/local/lib/python3/dist-packages/yt_dlp/ From 7447513d32af964818769d24091ba2d0d362f5dc Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:31:15 -0400 Subject: [PATCH 16/56] Update Pipfile --- Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Pipfile b/Pipfile index a23c3dbb..b2448a1c 100644 --- a/Pipfile +++ b/Pipfile @@ -14,7 +14,6 @@ pillow = "*" whitenoise = "*" gunicorn = "*" httptools = "*" -django-background-tasks = ">=1.2.8" django-basicauth = "*" psycopg = {extras = ["binary", "pool"], version = "*"} mysqlclient = "*" From bfdb4fe9119677edb4227b9b2db97e9c150e74d6 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:43:01 -0400 Subject: [PATCH 17/56] Remove `background_task` from views.py --- tubesync/sync/views.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index daa3da27..d3123e52 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -22,7 +22,6 @@ from django.utils.translation import gettext_lazy as _ from common.models import TaskHistory from common.timestamp import timestamp_to_datetime from common.utils import append_uri_params, mkdir_p, multi_key_sort -from background_task.models import Task from django_huey import DJANGO_HUEY, get_queue from common.huey import h_q_reset_tasks from .models import Source, Media, MediaServer @@ -43,9 +42,6 @@ from . import youtube def get_waiting_tasks(): - background_task_ids = { - str(t.pk) for t in Task.objects.all() - } huey_queue_names = (DJANGO_HUEY or {}).get('queues', {}) huey_queues = list(map(get_queue, huey_queue_names)) huey_task_ids = { @@ -56,7 +52,7 @@ def get_waiting_tasks(): ) } return TaskHistory.objects.filter( - task_id__in=huey_task_ids.union(background_task_ids), + task_id__in=huey_task_ids, ) @@ -896,37 +892,6 @@ class TasksView(ListView): return 'error' return True and obj - verbose_names = dict() - for task in Task.objects.filter(locked_by__isnull=False): - # There was broken logic in `Task.objects.locked()`, work around it. - # With that broken logic, the tasks never resume properly. - # This check unlocks the tasks without a running process. - # `task.locked_by_pid_running()` returns: - # - `True`: locked and PID exists - # - `False`: locked and PID does not exist - # - `None`: not `locked_by`, so there was no PID to check - locked_by_pid_running = task.locked_by_pid_running() - if locked_by_pid_running is False: - task.locked_by = None - # do not wait for the task to expire - task.locked_at = None - task.save() - task_id = str(task.pk) - verbose_names[task_id] = task.verbose_name - try: - task = TaskHistory.objects.get(task_id=task_id) - except TaskHistory.DoesNotExist: - # possibly create a new instance? - pass - else: - if locked_by_pid_running and add_to_task(task): - # Use the status if it is available - task.verbose_name = verbose_names.get(task_id) or task.verbose_name - data['running'].append(task) - elif locked_by_pid_running and 'wait_for_database_queue' in task.name: - data['wait_for_database_queue'] = True - verbose_names = None - for task in running_qs: if task in data['running']: continue @@ -1031,7 +996,6 @@ class ResetTasks(FormView): def form_valid(self, form): # Delete all tasks - Task.objects.all().delete() huey_queue_names = (DJANGO_HUEY or {}).get('queues', {}) for queue_name in huey_queue_names: h_q_reset_tasks(queue_name) @@ -1054,7 +1018,7 @@ class TaskScheduleView(FormView, SingleObjectMixin): template_name = 'sync/task-schedule.html' form_class = ScheduleTaskForm - model = Task + model = TaskHistory errors = dict( invalid_when=_('The type ({}) was incorrect.'), when_before_now=_('The date and time must be in the future.'), From 342a55b7f9fbf83c8d12f05fc53db8175bf13d0d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:50:20 -0400 Subject: [PATCH 18/56] Remove `background_task` from signals.py --- tubesync/sync/signals.py | 112 --------------------------------------- 1 file changed, 112 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 046f0724..dea0e1cd 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -8,10 +8,6 @@ from django.db.transaction import atomic, on_commit from django.dispatch import receiver from django.utils import timezone from django.utils.translation import gettext_lazy as _ -from background_task.signals import ( - task_created, task_started, task_successful, task_rescheduled, task_failed, -) -from background_task.models import Task from common.logger import log from common.models import TaskHistory from common.utils import glob_quote, mkdir_p @@ -160,114 +156,6 @@ def source_pre_delete(sender, instance, **kwargs): )) -@receiver(task_created, dispatch_uid='sync.signals.task_task_created') -@atomic(durable=False) -def task_task_created(sender, task=None, **kwargs): - if task is None: - return - task_obj = task - th, created = TaskHistory.objects.get_or_create( - task_id=str(task_obj.pk), - name=task_obj.task_name, - queue=task_obj.queue, - ) - th.scheduled_at = task_obj.run_at - th.priority = (100 - task_obj.priority) - th.repeat = task_obj.repeat - th.repeat_until = task_obj.repeat_until - th.task_params = list(task_obj.params()) - th.verbose_name = task_obj.verbose_name - th.save() - if created: - log.debug(f'Created a new task history record: {th.pk}: {th.verbose_name}') - - -@receiver(task_started, dispatch_uid='sync.signals.task_task_started') -@atomic(durable=False) -def task_task_started(sender, **kwargs): - locked_tasks = Task.objects.locked(timezone.now()) - for task_obj in locked_tasks: - th, created = TaskHistory.objects.get_or_create( - task_id=str(task_obj.pk), - name=task_obj.task_name, - queue=task_obj.queue, - ) - th.attempts += 1 - th.end_at = task_obj.locked_at - th.priority = (100 - task_obj.priority) - th.repeat = task_obj.repeat - th.repeat_until = task_obj.repeat_until - th.start_at = task_obj.locked_at - th.task_params = list(task_obj.params()) - th.verbose_name = task_obj.verbose_name - th.save() - if created: - log.debug(f'Started a new task history record: {th.pk}: {th.verbose_name}') - - -@receiver(task_rescheduled, dispatch_uid='sync.signals.task_task_rescheduled') -@atomic(durable=False) -def task_task_rescheduled(sender, task=None, **kwargs): - if task is None: - return - now_dt = timezone.now() - task_obj = task - th, created = TaskHistory.objects.get_or_create( - task_id=str(task_obj.pk), - name=task_obj.task_name, - queue=task_obj.queue, - ) - th.elapsed += ( - now_dt - task_obj.locked_at - ).total_seconds() - th.end_at = now_dt - th.scheduled_at = task_obj.run_at - th.start_at = task_obj.locked_at - th.save() - if created: - log.debug(f'Rescheduled a new task history record: {th.pk}: {th.verbose_name}') - -def merge_completed_task_into_history(task_id, task_obj): - th, created = TaskHistory.objects.get_or_create( - task_id=str(task_id), - name=task_obj.task_name, - queue=task_obj.queue, - ) - th.elapsed += ( - (task_obj.failed_at or task_obj.run_at) - task_obj.locked_at - ).total_seconds() - th.end_at = task_obj.run_at - th.failed_at = task_obj.failed_at - th.last_error = task_obj.last_error - th.repeat = task_obj.repeat - th.repeat_until = task_obj.repeat_until - th.start_at = task_obj.locked_at - th.verbose_name = task_obj.verbose_name - th.save() - - -@receiver(task_successful, dispatch_uid='sync.signals.task_task_successful') -@atomic(durable=False) -def task_task_successful(sender, task_id, completed_task, **kwargs): - merge_completed_task_into_history(task_id, completed_task) - - -@receiver(task_failed, dispatch_uid='sync.signals.task_task_failed') -@atomic(durable=False) -def task_task_failed(sender, task_id, completed_task, **kwargs): - merge_completed_task_into_history(task_id, completed_task) - # Triggered after a task fails by reaching its max retry attempts - obj, url = map_task_to_instance(completed_task, using_history=False) - if isinstance(obj, Source): - log.error(f'Permanent failure for source: {obj} task: {completed_task}') - obj.has_failed = True - obj.save() - - if isinstance(obj, Media) and completed_task.task_name == "sync.tasks.download_media_metadata": - log.error(f'Permanent failure for media: {obj} task: {completed_task}') - obj.skip = True - obj.save() - @receiver(post_save, sender=Media) def media_post_save(sender, instance, created, **kwargs): media = instance From c4ab3ddda83e0ee9dd9e87d52abb985e9cfa08f3 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 12:53:06 -0400 Subject: [PATCH 19/56] fixup: remove unused imports --- tubesync/sync/signals.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index dea0e1cd..3166f0e3 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -6,7 +6,6 @@ from django.db import IntegrityError from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.transaction import atomic, on_commit from django.dispatch import receiver -from django.utils import timezone from django.utils.translation import gettext_lazy as _ from common.logger import log from common.models import TaskHistory @@ -14,7 +13,6 @@ from common.utils import glob_quote, mkdir_p from .models import Source, Media, Metadata from .tasks import ( get_media_download_task, get_media_metadata_task, get_media_thumbnail_task, - map_task_to_instance, delete_all_media_for_source, rename_media, save_all_media_for_source, check_source_directory_exists, download_source_images, index_source, download_media_file, download_media_metadata, download_media_image, From d1344c40cdf28befaa648ee06ac0126521ee2962 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 16 Jul 2025 19:26:16 -0400 Subject: [PATCH 20/56] Clean up tests after debugging --- tubesync/sync/tests.py | 61 +++++++++--------------------------------- 1 file changed, 13 insertions(+), 48 deletions(-) diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 90d20ca8..fc380a9c 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -15,7 +15,6 @@ from django.test import TestCase, Client, override_settings from django.utils import timezone from django_huey import DJANGO_HUEY, get_queue from common.models import TaskHistory -from huey.consumer_options import ConsumerConfig from .models import Source, Media from .tasks import ( cleanup_old_media, check_source_directory_exists, @@ -29,30 +28,19 @@ from .choices import (Val, Fallback, IndexSchedule, SourceResolution, class FrontEndTestCase(TestCase): + maxDiff = None @classmethod def setUpClass(cls): super().setUpClass() - cls._consumers = dict() - for qn, qc in DJANGO_HUEY.get('queues', dict()).items(): + # Use immediate mode to execute tasks in this process + for qn in DJANGO_HUEY.get('queues', dict()): q = get_queue(qn) - consumer_opts = qc.get('consumer', {}) - config = ConsumerConfig(**consumer_opts) - config.validate() - #consumer = q.create_consumer(**config.values) - #cls._consumers[qn] = consumer - #consumer.start() - q.immediate = True + # Set the storage variable before using the property. q.immediate_use_memory = True - - @classmethod - def tearDownClass(cls): - for qn, consumer in cls._consumers.items(): - consumer.stop(graceful=True) - super().tearDownClass() + q.immediate = False def setUp(self): - self.maxDiff = None # Disable general logging for test case logging.disable(logging.CRITICAL) @@ -189,13 +177,6 @@ class FrontEndTestCase(TestCase): def test_source(self): #logging.disable(logging.NOTSET) - def get_model_task(model_pk, /, name=None): - qs = TaskHistory.objects.all() - if name is not None: - qs = qs.filter(name__endswith=name) - params_prefix = f'[["{model_pk}"' - qs = qs.filter(task_params__istartswith=params_prefix) - return qs[0] if qs.count() else False # Sources overview page c = Client() response = c.get('/sources') @@ -257,15 +238,8 @@ class FrontEndTestCase(TestCase): name='sync.tasks.index_source', task_params__0__0=source_uuid, ).order_by('end_at') - self.assertNotEqual(list(), list(index_task_qs)) - self.assertNotEqual( - list(), - [ - th.__dict__ for th in TaskHistory.objects.all() - ] - ) - task = get_model_task(source_uuid, name='index_source') - self.assertNotEqual(False, task) + self.assertTrue(index_task_qs) + task = index_task_qs.last() self.assertEqual(task.queue, get_queue(Val(TaskQueue.LIMIT)).name) # save and refresh the Source source.refresh_from_db() @@ -466,27 +440,18 @@ class FrontEndTestCase(TestCase): end_at=now_dt, ) # Check the tasks to fetch the media thumbnails have been scheduled - def get_model_task(model_pk, /, name=None): - qs = TaskHistory.objects.all() - if name is not None: - qs = qs.filter(name__endswith=name) - params_prefix = f'[["{model_pk}"' - qs = qs.filter(task_params__istartswith=params_prefix) - return 1 == qs.count() - name_suffix = 'download_media_file' - found_download_task1 = get_model_task(test_media1_pk, name_suffix) - found_download_task2 = get_model_task(test_media2_pk, name_suffix) + found_download_task1 = get_media_download_task(test_media1_pk) + found_download_task2 = get_media_download_task(test_media2_pk) found_download_task3 = get_media_download_task(test_media3_pk) - name_suffix = 'download_media_image' - found_thumbnail_task1 = get_model_task(test_media1_pk, name_suffix) - found_thumbnail_task2 = get_model_task(test_media2_pk, name_suffix) + found_thumbnail_task1 = get_media_thumbnail_task(test_media1_pk) + found_thumbnail_task2 = get_media_thumbnail_task(test_media2_pk) found_thumbnail_task3 = get_media_thumbnail_task(test_media3_pk) self.assertTrue(found_download_task1) self.assertTrue(found_download_task2) - self.assertTrue(not not found_download_task3) + self.assertTrue(found_download_task3) self.assertTrue(found_thumbnail_task1) self.assertTrue(found_thumbnail_task2) - self.assertTrue(not not found_thumbnail_task3) + self.assertTrue(found_thumbnail_task3) # Check the media is listed on the media overview page response = c.get('/media') self.assertEqual(response.status_code, 200) From 129925af5880520eff66507abd7f2733b08a7a20 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Jul 2025 19:18:01 -0400 Subject: [PATCH 21/56] Use the `TaskHistory.id` column --- tubesync/sync/templates/sync/tasks.html | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/tasks.html b/tubesync/sync/templates/sync/tasks.html index 85f38c20..cdf84cc7 100644 --- a/tubesync/sync/templates/sync/tasks.html +++ b/tubesync/sync/templates/sync/tasks.html @@ -55,11 +55,9 @@ Error: "{{ task.error_message }}"
Task will be retried at {{ task.scheduled_at|date:'Y-m-d H:i:s' }} - {% if '-' not in task.task_id %} - + - {% endif %} {% empty %} There are no tasks with errors on this page. @@ -87,11 +85,10 @@ Priority: {{ task.priority }} Queue: {{ task.queue }}
Task will run {% if task.run_now %}immediately{% else %}at {{ task.scheduled_at|date:'Y-m-d H:i:s' }} - {% if '-' not in task.task_id %} - + - {% endif %}{% endif %} + {% endif %} {% empty %} From 75e083f9874b01fd801dd69225fe00b128153d0a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Jul 2025 20:10:10 -0400 Subject: [PATCH 22/56] Reschedule the actual task using its `TaskHistory` instance --- tubesync/sync/views.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 26fd1644..d3bd2d27 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1059,7 +1059,7 @@ class TaskScheduleView(FormView, SingleObjectMixin): template_name = 'sync/task-schedule.html' form_class = ScheduleTaskForm - model = Task + model = TaskHistory errors = dict( invalid_when=_('The type ({}) was incorrect.'), when_before_now=_('The date and time must be in the future.'), @@ -1109,7 +1109,6 @@ class TaskScheduleView(FormView, SingleObjectMixin): ) def form_valid(self, form): - max_attempts = getattr(settings, 'MAX_ATTEMPTS', 15) when = form.cleaned_data.get('when') if not isinstance(when, self.now.__class__): @@ -1130,14 +1129,20 @@ class TaskScheduleView(FormView, SingleObjectMixin): if form.errors: return super().form_invalid(form) - self.object.attempts = max_attempts // 2 - self.object.run_at = max(self.now, when) - self.object.save() - TaskHistory.objects.filter( - task_id=str(self.object.pk), - ).update( - scheduled_at=self.object.run_at, - ) + huey_queue_names = (DJANGO_HUEY or {}).get('queues', {}) + huey_queues = list(map(get_queue, huey_queue_names)) + matching = { q for q in huey_queues if q.name == self.object.queue } + try: + q = matching.pop() + except KeyError as e: + pk = self.object.pk + queue = self.object.queue + msg = f'TaskScheduleView: queue not found: {pk=} {queue=}' + log.exception(msg, exc_info=e) + else: + self.object.scheduled_at = max(self.now, when) + if q and q.reschedule(self.object.task_id, self.object.scheduled_at): + self.object.save() return super().form_valid(form) From dd03023e99b62dc68d9cdb6eea3320e4681b7a42 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Jul 2025 20:12:34 -0400 Subject: [PATCH 23/56] fixup: add a missing import --- tubesync/sync/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index d3bd2d27..89fcb7ef 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -25,6 +25,7 @@ from common.utils import append_uri_params, mkdir_p, multi_key_sort from background_task.models import Task from django_huey import DJANGO_HUEY, get_queue from common.huey import h_q_reset_tasks +from common.logger import log from .models import Source, Media, MediaServer from .forms import (ValidateSourceForm, ConfirmDeleteSourceForm, RedownloadMediaForm, SkipMediaForm, EnableMediaForm, ResetTasksForm, ScheduleTaskForm, From 87459af9e2f9a72625dd7155148726816f898489 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 17 Jul 2025 20:31:20 -0400 Subject: [PATCH 24/56] Log the failure when the task was already removed --- tubesync/sync/views.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 89fcb7ef..31e332f9 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -1132,18 +1132,22 @@ class TaskScheduleView(FormView, SingleObjectMixin): huey_queue_names = (DJANGO_HUEY or {}).get('queues', {}) huey_queues = list(map(get_queue, huey_queue_names)) - matching = { q for q in huey_queues if q.name == self.object.queue } + pk = self.object.pk + queue = self.object.queue + task_id = self.object.task_id + matching = { q for q in huey_queues if q.name == queue } try: q = matching.pop() except KeyError as e: - pk = self.object.pk - queue = self.object.queue msg = f'TaskScheduleView: queue not found: {pk=} {queue=}' log.exception(msg, exc_info=e) else: self.object.scheduled_at = max(self.now, when) - if q and q.reschedule(self.object.task_id, self.object.scheduled_at): + if q and q.reschedule(task_id, self.object.scheduled_at): self.object.save() + else: + msg = f'TaskScheduleView: task not found: {pk=} {task_id=}' + log.warning(msg) return super().form_valid(form) From 106da4616feecf514b382acf04f155253a555dca Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 05:35:34 -0400 Subject: [PATCH 25/56] Store the metadata from channels or playlists --- tubesync/sync/youtube.py | 52 +++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index c0e55f6d..1b7122a5 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -81,7 +81,10 @@ def get_channel_id(url): else: return channel_id -def get_channel_image_info(url): +def get_image_info(url): + avatar_url = None + banner_url = None + thumbnail_url = None opts = get_yt_opts() opts.update({ 'skip_download': True, @@ -94,20 +97,47 @@ def get_channel_image_info(url): with yt_dlp.YoutubeDL(opts) as y: try: response = y.extract_info(url, download=False) - - avatar_url = None - banner_url = None + except yt_dlp.utils.DownloadError as e: + raise YouTubeError(f'Failed to extract info for "{url}": {e}') from e + else: + height = 0 for thumbnail in response['thumbnails']: if thumbnail['id'] == 'avatar_uncropped': avatar_url = thumbnail['url'] - if thumbnail['id'] == 'banner_uncropped': + elif thumbnail['id'] == 'banner_uncropped': banner_url = thumbnail['url'] - if banner_url is not None and avatar_url is not None: - break - - return avatar_url, banner_url - except yt_dlp.utils.DownloadError as e: - raise YouTubeError(f'Failed to extract channel info for "{url}": {e}') from e + elif (thumbnail.get('height') or 0) > height: + thumbnail_url = thumbnail['url'] + try: + entry_type = response['entries'][0].get('_type') + except IndexError: + # an empty entries list + pass + else: + if 'url' == entry_type: + del response['entries'] + elif 'playlist' == entry_type: + for playlist in response['entries']: + del playlist['entries'] + from .models import Metadata + t = Metadata.objects.defer('value').filter( + source__isnull=True, + media__isnull=True, + ).get_or_create( + key=response['id'], + site=response['extractor_key'], + ) + md = t[0] + field_defaults = { + f.attname: f.get_default() + for f in md._meta.fields + if f.has_default() + } + if 'retrieved' in field_defaults: + md.retrieved = field_defaults['retrieved'] + md.value = response + md.save() + return avatar_url, banner_url, thumbnail_url def _subscriber_only(msg='', response=None): From ae75302da4f07dfd8dbd55085510636d6141bba4 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 06:01:56 -0400 Subject: [PATCH 26/56] Update source.py --- tubesync/sync/models/source.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 42afd2ca..7930a214 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -5,7 +5,6 @@ from collections import deque as queue from pathlib import Path from django import db from django.conf import settings -from django.core.exceptions import SuspiciousOperation from django.core.validators import RegexValidator from django.utils import timezone from django.utils.text import slugify @@ -20,7 +19,7 @@ from ..choices import (Val, from ..fields import CommaSepChoiceField from ..youtube import ( get_media_info as get_youtube_media_info, - get_channel_image_info as get_youtube_channel_image_info, + get_image_info as get_youtube_image_info, ) from ._migrations import media_file_storage from ._private import _srctype_dict @@ -484,10 +483,7 @@ class Source(db.models.Model): @property def get_image_url(self): - if self.is_playlist: - raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') - - return get_youtube_channel_image_info(self.url) + return get_youtube_image_info(self.url) def directory_exists(self): From f66809113a3743ee731471a1761ca7fe1cc9dafe Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 06:09:52 -0400 Subject: [PATCH 27/56] Update tasks.py --- tubesync/sync/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 00f38a20..170be1b9 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -720,10 +720,11 @@ def download_source_images(source_id): log.error(f'Task download_source_images(pk={source_id}) called but no ' f'source exists with ID: {source_id}') raise CancelExecution(_('no such source'), retry=False) from e - avatar, banner = source.get_image_url + avatar, banner, thumbnail = source.get_image_url log.info(f'Thumbnail URL for source with ID: {source_id} / {source} ' f'Avatar: {avatar} ' - f'Banner: {banner}') + f'Banner: {banner} ' + f'Thumbnail: {thumbnail}') if banner is not None: url = banner i = get_remote_image(url) From efe0d5a09c9cb5c3dc28536f128ef3c5c755b99c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 06:20:32 -0400 Subject: [PATCH 28/56] Normalize `height` before comparing to `max_height` --- tubesync/sync/youtube.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 1b7122a5..7bfda388 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -100,13 +100,19 @@ def get_image_info(url): except yt_dlp.utils.DownloadError as e: raise YouTubeError(f'Failed to extract info for "{url}": {e}') from e else: - height = 0 + max_height = 0 for thumbnail in response['thumbnails']: + thumbnail_height = thumbnail.get('height') + try: + thumbnail_height = int(thumbnail_height) + except (TypeError, ValueError,): + thumbnail_height = int() if thumbnail['id'] == 'avatar_uncropped': avatar_url = thumbnail['url'] elif thumbnail['id'] == 'banner_uncropped': banner_url = thumbnail['url'] - elif (thumbnail.get('height') or 0) > height: + elif thumbnail_height > max_height: + max_height = thumbnail_height thumbnail_url = thumbnail['url'] try: entry_type = response['entries'][0].get('_type') From 1136f20128738961138ac54a6ffcabad7900fc08 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 06:30:49 -0400 Subject: [PATCH 29/56] Accept playlists also --- tubesync/sync/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index fdd65981..21dd5afb 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -43,7 +43,7 @@ def source_pre_save(sender, instance, **kwargs): check_source_directory_exists.call_local(*args) existing_copy_channel_images = existing_source.copy_channel_images new_copy_channel_images = instance.copy_channel_images - if new_copy_channel_images and not (existing_copy_channel_images or instance.is_playlist): + if new_copy_channel_images and existing_copy_channel_images: download_source_images(str(instance.pk)) existing_dirpath = existing_source.directory_path.resolve(strict=True) new_dirpath = instance.directory_path.resolve(strict=False) @@ -117,7 +117,7 @@ def source_post_save(sender, instance, created, **kwargs): # Check directory exists and create an indexing task for newly created sources if created: check_source_directory_exists(str(source.pk)) - if source.copy_channel_images and not source.is_playlist: + if source.copy_channel_images: download_source_images(str(source.pk)) if source.is_active: log.info(f'Scheduling first media indexing for source: {source.name}') From d619aca56f55ba069ac932f3ce37b7f996632825 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 06:32:38 -0400 Subject: [PATCH 30/56] fixup: restore the `not` --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 21dd5afb..13e1084d 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -43,7 +43,7 @@ def source_pre_save(sender, instance, **kwargs): check_source_directory_exists.call_local(*args) existing_copy_channel_images = existing_source.copy_channel_images new_copy_channel_images = instance.copy_channel_images - if new_copy_channel_images and existing_copy_channel_images: + if new_copy_channel_images and not existing_copy_channel_images: download_source_images(str(instance.pk)) existing_dirpath = existing_source.directory_path.resolve(strict=True) new_dirpath = instance.directory_path.resolve(strict=False) From 70a26314e51237c5fb2d1ffefc7bf4b8ac6f8b2d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 07:12:25 -0400 Subject: [PATCH 31/56] Set `locking_mode` to `NORMAL` explicitly --- tubesync/tubesync/local_settings.py.container | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 566cfb24..f4205772 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -51,8 +51,13 @@ else: "OPTIONS": { "timeout": 10, "transaction_mode": "IMMEDIATE", + # PRAGMA locking_mode = NORMAL | EXCLUSIVE # PRAGMA journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF + # DO NOT change locking_mode to EXCLUSIVE! + # This is a foot-gun, and invalidates a behavior the code relies on. + # journal_mode WAL offers increased concurrency, the default is DELETE. "init_command": """ + PRAGMA locking_mode = NORMAL; PRAGMA journal_mode = TRUNCATE; PRAGMA journal_size_limit = 67108864; PRAGMA legacy_alter_table = OFF; From d4f3b6addf16327e4c86fc8493a96e2176d6f941 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 07:38:29 -0400 Subject: [PATCH 32/56] Save the `thumbnail` URL to a file --- tubesync/sync/tasks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 170be1b9..ede85d2e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -725,6 +725,22 @@ def download_source_images(source_id): f'Avatar: {avatar} ' f'Banner: {banner} ' f'Thumbnail: {thumbnail}') + if thumbnail is not None: + url = thumbnail + i = get_remote_image(url) + image_file = BytesIO() + i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True) + + for file_name in ["thumbnail.jpg",]: + # Reset file pointer to the beginning for the next save + image_file.seek(0) + # Create a Django ContentFile from BytesIO stream + django_file = ContentFile(image_file.read()) + file_path = source.directory_path / file_name + with open(file_path, 'wb') as f: + f.write(django_file.read()) + i = image_file = None + if banner is not None: url = banner i = get_remote_image(url) From 75dd32f76089e88f9fbf67174b2246cd59568086 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 09:16:36 -0400 Subject: [PATCH 33/56] Add basic search query support --- tubesync/sync/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 26fd1644..dc9d5647 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -503,6 +503,7 @@ class MediaView(ListView): self.filter_source = None self.show_skipped = False self.only_skipped = False + self.query = None super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): @@ -519,6 +520,7 @@ class MediaView(ListView): only_skipped = request.GET.get('only_skipped', '').strip() if only_skipped == 'yes': self.only_skipped = True + self.query = request.GET.get('query') or request.POST.get('query') or None return super().dispatch(request, *args, **kwargs) def get_queryset(self): @@ -526,6 +528,9 @@ class MediaView(ListView): if self.filter_source: q = q.filter(source=self.filter_source) + if self.query: + needle = self.query + q = q.filter(Q(title__icontains=needle) | Q(key__contains=needle)) if self.only_skipped: q = q.filter(Q(can_download=False) | Q(skip=True) | Q(manual_skip=True)) elif not self.show_skipped: @@ -543,6 +548,7 @@ class MediaView(ListView): data['source'] = self.filter_source data['show_skipped'] = self.show_skipped data['only_skipped'] = self.only_skipped + data['query'] = self.query or str() return data From e1a486115ea32a12756770c80cd491cf66e0c78d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 09:28:10 -0400 Subject: [PATCH 34/56] Add a basic search box --- tubesync/sync/templates/sync/media.html | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index d2d4e639..0baf28ba 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -22,6 +22,17 @@ {% endif %} +
+
+ {% csrf_token %} +
+ Search: +
+
+ +
+
+
{% include 'infobox.html' with message=message %}
{% for m in media %} From ee247060ab2c2e06a559e2234787b494a75e52ce Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 09:54:33 -0400 Subject: [PATCH 35/56] Use GET for now --- tubesync/sync/templates/sync/media.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 0baf28ba..2d067355 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -23,13 +23,18 @@
-
- {% csrf_token %} -
- Search: + + +
+
-
- +
+ +
+
+
From 30cfb5dc692a1e4698d377b612232b9999e6e6ae Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 09:57:31 -0400 Subject: [PATCH 36/56] fixup: set value for submit --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 2d067355..dbc0d91d 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -34,7 +34,7 @@
- +
From a71e40f5091f5e30f944c7c5600315e3247b8b00 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 12:14:51 -0400 Subject: [PATCH 37/56] Arrange the box and button better --- tubesync/sync/templates/sync/media.html | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index dbc0d91d..a625b982 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -22,18 +22,16 @@ {% endif %} -
-
+
+ -
+
-
-
-
+
From 25f5af466ae1c8a8053d4b3b3b3df57f0e9c524c Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 12:22:02 -0400 Subject: [PATCH 38/56] Preserve the previous search query --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index a625b982..8f55b637 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -29,7 +29,7 @@ - +
From ed91a2bc2c593eaeddfcc5e93760f84efdb50195 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 12:26:48 -0400 Subject: [PATCH 39/56] Do not set set value when query wasn't provided --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 8f55b637..3a53cf9d 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -29,7 +29,7 @@ - +
From 51c26365381a4b5410119f91a1b8b77e95becd6d Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 12:36:00 -0400 Subject: [PATCH 40/56] Show more of the placeholder by default --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 3a53cf9d..be0d12e6 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -29,7 +29,7 @@ - +
From e5422c284e1f5ab5fe3d719d712a7a54326c1206 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 15:54:59 -0400 Subject: [PATCH 41/56] Stop creating the query string manually --- tubesync/sync/templates/sync/media.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index be0d12e6..cd054f8b 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -9,16 +9,16 @@
{% if show_skipped %} - Hide skipped media + Hide skipped media {% else %} - Show skipped media + Show skipped media {% endif %}
{% if only_skipped %} - Only skipped media + Only skipped media {% else %} - Only skipped media + Only skipped media {% endif %}
From 0a0612f77bacde6dcfd7c3ae8d275a8657dbd5c6 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 17:08:33 -0400 Subject: [PATCH 42/56] Use flex box for positioning --- tubesync/sync/templates/sync/media.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index cd054f8b..5340df08 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -25,14 +25,14 @@
-
+
-
- +
+
From c307f5081facd4010c6398a373ba00dffb8d7f99 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 17:12:47 -0400 Subject: [PATCH 43/56] fixup: use `btn` class for submit input --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 5340df08..54e5defc 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -32,7 +32,7 @@
- +
From 8c30d19718dea73bb86713bf878351adeac95399 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 17:16:22 -0400 Subject: [PATCH 44/56] fixup: oddly the larger display was worse --- tubesync/sync/templates/sync/media.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 54e5defc..daff8de0 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -25,13 +25,13 @@
-
+
-
+
From 18594ac5add8da4b8380ed9a4903c1c928d3f927 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 17:30:42 -0400 Subject: [PATCH 45/56] Search in `fulltitle` and `description` from metadata --- tubesync/sync/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index dc9d5647..e6fd73b9 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -530,7 +530,12 @@ class MediaView(ListView): q = q.filter(source=self.filter_source) if self.query: needle = self.query - q = q.filter(Q(title__icontains=needle) | Q(key__contains=needle)) + q = q.filter( + Q(new_metadata__value__fulltitle__icontains=needle) | + Q(new_metadata__value__description__icontains=needle) | + Q(title__icontains=needle) | + Q(key__contains=needle) + ) if self.only_skipped: q = q.filter(Q(can_download=False) | Q(skip=True) | Q(manual_skip=True)) elif not self.show_skipped: From cd419673bde34474d8d2cfd55c285b5e73ceb300 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 19:25:26 -0400 Subject: [PATCH 46/56] Add `remove_duplicated_rows` function before `task_id` unique change --- .../0004_alter_taskhistory_task_id.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tubesync/common/migrations/0004_alter_taskhistory_task_id.py b/tubesync/common/migrations/0004_alter_taskhistory_task_id.py index 9d2dc45f..60f7ca07 100644 --- a/tubesync/common/migrations/0004_alter_taskhistory_task_id.py +++ b/tubesync/common/migrations/0004_alter_taskhistory_task_id.py @@ -1,6 +1,39 @@ # Generated by Django 5.2.4 on 2025-07-15 21:40 from django.db import migrations, models +from common.logger import log + + +def remove_duplicated_rows(apps, schema_editor): + def keep_which(task_id): + th_qs = TaskHistory.objects.all().order_by('-id') + tqs = th_qs.filter(task_id=task_id) + return tqs[0].id + + TaskHistory = apps.get_model("common", 'TaskHistory') + duplicates = set( + TaskHistory.objects.values( + 'task_id', + ).alias( + count=models.Count('id'), + ).filter( + count__gt=1, + ).values_list( + 'task_id', + flat=True, + ), + ) + + log.info(f'TaskHistory rows: {len(duplicates)=}') + for task_id, n in enumerate(duplicates, start=1): + keeping = keep_which(task_id) + log.debug(f'{n=}: {task_id=}: {keeping=}') + TaskHistory.objects.filter( + task_id=task_id, + ).exclude( + id=keeping, + ).delete() + log.info(f'TaskHistory rows: finished removing duplicates.') class Migration(migrations.Migration): @@ -10,6 +43,10 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython( + remove_duplicated_rows, + migrations.RunPython.noop, + ), migrations.AlterField( model_name='taskhistory', name='task_id', From d3a13e6b1cfe4a0bafcb218445a8197da5b801e7 Mon Sep 17 00:00:00 2001 From: tcely Date: Fri, 18 Jul 2025 19:28:58 -0400 Subject: [PATCH 47/56] fixup: remove the unnecessary f-string --- tubesync/common/migrations/0004_alter_taskhistory_task_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/migrations/0004_alter_taskhistory_task_id.py b/tubesync/common/migrations/0004_alter_taskhistory_task_id.py index 60f7ca07..9b251e4e 100644 --- a/tubesync/common/migrations/0004_alter_taskhistory_task_id.py +++ b/tubesync/common/migrations/0004_alter_taskhistory_task_id.py @@ -33,7 +33,7 @@ def remove_duplicated_rows(apps, schema_editor): ).exclude( id=keeping, ).delete() - log.info(f'TaskHistory rows: finished removing duplicates.') + log.info('TaskHistory rows: finished removing duplicates.') class Migration(migrations.Migration): From f74aae55c2eb4f64997ae61e0c3a0896d6dcb338 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 03:46:20 -0400 Subject: [PATCH 48/56] Do not always search in descriptions --- tubesync/sync/views.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index e6fd73b9..18b3ded7 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -504,23 +504,30 @@ class MediaView(ListView): self.show_skipped = False self.only_skipped = False self.query = None + self.search_description = False super().__init__(*args, **kwargs) def dispatch(self, request, *args, **kwargs): - filter_by = request.GET.get('filter', '') + def post_or_get(request, /, key, default=None): + return request.POST.get(key) or request.GET.get(key) or default + + filter_by = post_or_get(request, 'filter', '') if filter_by: try: self.filter_source = Source.objects.get(pk=filter_by) except Source.DoesNotExist: self.filter_source = None - show_skipped = request.GET.get('show_skipped', '').strip() + show_skipped = post_or_get(request, 'show_skipped', '').strip() if show_skipped == 'yes': self.show_skipped = True if not self.show_skipped: - only_skipped = request.GET.get('only_skipped', '').strip() + only_skipped = post_or_get(request, 'only_skipped', '').strip() if only_skipped == 'yes': self.only_skipped = True - self.query = request.GET.get('query') or request.POST.get('query') or None + self.query = post_or_get(request, 'query') + self.search_description = str(post_or_get(request, 'search_description')).strip().lower() in ( + 'enable', 'on', 'true', 'yes', '1', + ) return super().dispatch(request, *args, **kwargs) def get_queryset(self): @@ -530,12 +537,19 @@ class MediaView(ListView): q = q.filter(source=self.filter_source) if self.query: needle = self.query - q = q.filter( - Q(new_metadata__value__fulltitle__icontains=needle) | - Q(new_metadata__value__description__icontains=needle) | - Q(title__icontains=needle) | - Q(key__contains=needle) - ) + if self.search_description: + q = q.filter( + Q(new_metadata__value__fulltitle__icontains=needle) | + Q(new_metadata__value__description__icontains=needle) | + Q(title__icontains=needle) | + Q(key__contains=needle) + ) + else: + q = q.filter( + Q(new_metadata__value__fulltitle__icontains=needle) | + Q(title__icontains=needle) | + Q(key__contains=needle) + ) if self.only_skipped: q = q.filter(Q(can_download=False) | Q(skip=True) | Q(manual_skip=True)) elif not self.show_skipped: @@ -554,6 +568,7 @@ class MediaView(ListView): data['show_skipped'] = self.show_skipped data['only_skipped'] = self.only_skipped data['query'] = self.query or str() + data['search_description'] = self.search_description return data From 4bbcd6068205beef91eb7172ed1669669129a2af Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 07:34:52 -0400 Subject: [PATCH 49/56] Allow searching in the description as an option --- tubesync/sync/templates/sync/media.html | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index daff8de0..e69e4fc9 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -22,16 +22,20 @@ {% endif %}
-
+
-
-
-{% include 'pagination.html' with pagination=sources.paginator filter=source.pk show_skipped=show_skipped only_skipped=only_skipped%} +{% include 'pagination.html' with filter=source.pk show_skipped=show_skipped only_skipped=only_skipped search_descriptionsearch_description query=query %} {% endblock %} From f129ac2fa9e87dd32e08e9a700539b82a26b65ed Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 08:31:37 -0400 Subject: [PATCH 53/56] fixup: add the missing '=' --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index ebf36780..162b1ccb 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -82,5 +82,5 @@
{% endfor %}
-{% include 'pagination.html' with filter=source.pk show_skipped=show_skipped only_skipped=only_skipped search_descriptionsearch_description query=query %} +{% include 'pagination.html' with filter=source.pk show_skipped=show_skipped only_skipped=only_skipped search_description=search_description query=query %} {% endblock %} From 65f025eabe1e47a05d8cd39783ace1994e54d916 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 08:41:45 -0400 Subject: [PATCH 54/56] Display the `search_description` status --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index 162b1ccb..f52491de 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -33,7 +33,7 @@
From 50dff35f639a02e417aaa36196dbc3db14c76d0c Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 08:46:41 -0400 Subject: [PATCH 55/56] Stop creating the query string manually --- tubesync/common/templates/pagination.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/templates/pagination.html b/tubesync/common/templates/pagination.html index e48b24d8..1bb0e2b5 100644 --- a/tubesync/common/templates/pagination.html +++ b/tubesync/common/templates/pagination.html @@ -3,7 +3,7 @@
From 7edf90ba22dfe8360d7a333090beeed81d944898 Mon Sep 17 00:00:00 2001 From: tcely Date: Sat, 19 Jul 2025 08:48:51 -0400 Subject: [PATCH 56/56] With variables are no longer needed --- tubesync/sync/templates/sync/media.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/templates/sync/media.html b/tubesync/sync/templates/sync/media.html index f52491de..0ba43b04 100644 --- a/tubesync/sync/templates/sync/media.html +++ b/tubesync/sync/templates/sync/media.html @@ -82,5 +82,5 @@
{% endfor %}
-{% include 'pagination.html' with filter=source.pk show_skipped=show_skipped only_skipped=only_skipped search_description=search_description query=query %} +{% include 'pagination.html' %} {% endblock %}