Merge pull request #1274 from tcely/patch-11

Include more information when a key is missing

tubesync/common/errors.py

@@ -60,3 +60,11 @@ class FormatUnavailableError(Exception):
         self.format = format
         super().__init__(*args, **kwargs)
 
+
+class QuerySetEmptyError(Exception):
+    # Raised when a primary key was missing when iterating a query set.
+    def __init__(self, *args, exc=None, key=None, **kwargs):
+        self.exc = exc
+        self.key = key
+        super().__init__(*args, **kwargs)
+

tubesync/common/utils.py

@@ -11,7 +11,7 @@ from functools import partial
 from operator import attrgetter, itemgetter
 from pathlib import Path
 from urllib.parse import urlunsplit, urlencode, urlparse
-from .errors import DatabaseConnectionError
+from .errors import DatabaseConnectionError, QuerySetEmptyError
 
 def directory_and_stem(arg_path, /, all_suffixes=False):
     filepath = Path(arg_path)
@@ -307,14 +307,22 @@ def django_queryset_generator(query_set, /, *,
     gc.disable()
     if use_chunked_fetch:
         for key in qs._iterator(use_chunked_fetch, chunk_size):
-            yield query_set.filter(pk=key)[0]
+            try:
+                yield query_set.filter(pk=key)[0]
+            except IndexError as exc:
+                msg = f'missing primary key: {key}'
+                raise QuerySetEmptyError(msg, exc=exc, key=key) from exc
             key = None
             gc.collect(generation=1)
         key = None
     else:
         for page in iter(Paginator(qs, page_size)):
             for key in page.object_list:
-                yield query_set.filter(pk=key)[0]
+                try:
+                    yield query_set.filter(pk=key)[0]
+                except IndexError as exc:
+                    msg = f'missing primary key: {key}'
+                    raise QuerySetEmptyError(msg, exc=exc, key=key) from exc
                 key = None
                 gc.collect(generation=1)
             key = None

tubesync/sync/tasks.py

@@ -31,6 +31,7 @@ from common.errors import (
     HueyConsumerError,
     DownloadFailedException, FormatUnavailableError,
     NoFormatException, NoMediaException, NoThumbnailException,
+    QuerySetEmptyError,
 )
 from common.utils import (  django_queryset_generator as qs_gen,
                             remove_enclosed, seconds_to_timestr, )
@@ -255,13 +256,18 @@ def schedule_indexing():
         )
         if skip_source:
             continue
-        # clear all existing media locks
-        media_qs = Media.objects.filter(source=source).only('uuid')
-        for media in qs_gen(media_qs):
-            huey_lock_task(
-                f'media:{media.uuid}',
-                queue=Val(TaskQueue.DB),
-            ).clear()
+        try:
+            # clear all existing media locks
+            media_qs = Media.objects.filter(source=source).only('uuid')
+            for media in qs_gen(media_qs):
+                huey_lock_task(
+                    f'media:{media.uuid}',
+                    queue=Val(TaskQueue.DB),
+                ).clear()
+        except QuerySetEmptyError as e:
+            msg = f'missing media from "{source.name}": {source.pk}: {e.key}'
+            log.exception(msg, exc_info=e)
+            pass
         # schedule a new indexing task
         log.info(f'Scheduling an indexing task for source "{source.name}": {source.pk}')
         TaskHistory.schedule(