From e93abfc6f28e149d90e84f396e3d04f2bec391c8 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 11 Aug 2025 11:40:23 -0400 Subject: [PATCH 1/4] Add `QuerySetEmptyError` --- tubesync/common/errors.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tubesync/common/errors.py b/tubesync/common/errors.py index b097df28..f2e65f90 100644 --- a/tubesync/common/errors.py +++ b/tubesync/common/errors.py @@ -60,3 +60,11 @@ class FormatUnavailableError(Exception): self.format = format super().__init__(*args, **kwargs) + +class QuerySetEmptyError(Exception): + # Raised when a primary key was missing when iterating a query set. + def __init__(self, *args, exc=None, key=None, **kwargs): + self.exc = exc + self.key = key + super().__init__(*args, **kwargs) + From afac67544e8160ef30ff3715bf9b00c117e7f2db Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 11 Aug 2025 11:49:39 -0400 Subject: [PATCH 2/4] Raise the new `QuerySetEmptyError` exception --- tubesync/common/utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index d017cf59..aa2aa563 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -11,7 +11,7 @@ from functools import partial from operator import attrgetter, itemgetter from pathlib import Path from urllib.parse import urlunsplit, urlencode, urlparse -from .errors import DatabaseConnectionError +from .errors import DatabaseConnectionError, QuerySetEmptyError def directory_and_stem(arg_path, /, all_suffixes=False): filepath = Path(arg_path) @@ -304,14 +304,22 @@ def django_queryset_generator(query_set, /, *, gc.disable() if use_chunked_fetch: for key in qs._iterator(use_chunked_fetch, chunk_size): - yield query_set.filter(pk=key)[0] + try: + yield query_set.filter(pk=key)[0] + except IndexError as exc: + msg = f'missing primary key: {key}' + raise QuerySetEmptyError(msg, exc=exc, key=key) from exc key = None gc.collect(generation=1) key = None else: for page in iter(Paginator(qs, page_size)): for key in page.object_list: - yield query_set.filter(pk=key)[0] + try: + yield query_set.filter(pk=key)[0] + except IndexError as exc: + msg = f'missing primary key: {key}' + raise QuerySetEmptyError(msg, exc=exc, key=key) from exc key = None gc.collect(generation=1) key = None From f3fa19d2f075e986ae1bf46bdc222a97cd1a6491 Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 11 Aug 2025 11:58:19 -0400 Subject: [PATCH 3/4] Log the details from the schedule indexing task --- tubesync/sync/tasks.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index db56ea35..361b959a 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -255,13 +255,18 @@ def schedule_indexing(): ) if skip_source: continue - # clear all existing media locks - media_qs = Media.objects.filter(source=source).only('uuid') - for media in qs_gen(media_qs): - huey_lock_task( - f'media:{media.uuid}', - queue=Val(TaskQueue.DB), - ).clear() + try: + # clear all existing media locks + media_qs = Media.objects.filter(source=source).only('uuid') + for media in qs_gen(media_qs): + huey_lock_task( + f'media:{media.uuid}', + queue=Val(TaskQueue.DB), + ).clear() + except QuerySetEmptyError as e: + msg = f'missing media from "{source.name}": {source.pk}: {e.key}' + log.exception(msg, exc_info=e) + pass # schedule a new indexing task log.info(f'Scheduling an indexing task for source "{source.name}": {source.pk}') TaskHistory.schedule( From b0127fafec2296e7a9184d99ce2b6085777ed14e Mon Sep 17 00:00:00 2001 From: tcely Date: Mon, 11 Aug 2025 12:00:27 -0400 Subject: [PATCH 4/4] fixup: add a missing import --- tubesync/sync/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 361b959a..9a425a9e 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -31,6 +31,7 @@ from common.errors import ( HueyConsumerError, DownloadFailedException, FormatUnavailableError, NoFormatException, NoMediaException, NoThumbnailException, + QuerySetEmptyError, ) from common.utils import ( django_queryset_generator as qs_gen, remove_enclosed, seconds_to_timestr, )