Merge branch 'main' into tcely-curl-thumbnails

tubesync/install_bun.sh

@@ -72,7 +72,7 @@ extract_bun() {
     unzip -u -o -d './.bun' "${fn}" &&
         install -v -T ./.bun/bun-linux-*/bun "${_staged}" &&
         { # bun spawning unzip hangs for an unknown reason fairly often
-            local _attempt ; for _attempt in {1..3} ; do
+            local _attempt ; for _attempt in {1..5} ; do
                 "${_staged}" run "${HERE}/verify_bun.ts" --release 'bun-v1.3.14' --asset "${fn}" --install-dir "${dest_dir}" &&
                     break || sleep "${_attempt}"
             done ;

tubesync/sync/signals.py

@@ -148,9 +148,8 @@ def source_pre_delete(sender, instance, **kwargs):
     instance.deactivate()
 
     # Fetch the media source
-    sqs = Source.objects.filter(filter_text=str(source.pk))
-    if sqs.count():
-        media_source = sqs[0]
+    media_source = Source.objects.filter(filter_text=str(source.pk)).first()
+    if media_source:
         # Schedule deletion of media
         on_commit(partial(
             TaskHistory.schedule,
@@ -260,7 +259,7 @@ def media_post_save(sender, instance, created, **kwargs):
 @receiver(pre_delete, sender=Media)
 def media_pre_delete(sender, instance, **kwargs):
     # Remove thumbnail file for deleted media
-    if instance.thumb:
+    if instance.thumb_file_exists:
         instance.thumb.delete(save=False)
     # Save the metadata site & thumbnail URL to the metadata column
     existing_metadata = instance.loaded_metadata
@@ -352,9 +351,14 @@ def media_post_delete(sender, instance, **kwargs):
     created = False
     create_for_indexing_task = (
         not (
-            #not instance.downloaded and
-            instance.skip and
-            instance.manual_skip
+            (
+                instance.source and
+                instance.source.key.endswith('/deleted')
+            ) or
+            (
+                instance.skip and
+                instance.manual_skip
+            )
         )
     )
     if create_for_indexing_task:
@@ -391,7 +395,7 @@ def media_post_delete(sender, instance, **kwargs):
             key=skipped_media.key,
         )
         try:
-            if instance_qs.count():
+            if instance_qs.exists():
                 with atomic(durable=False):
                     # clear the link to a media instance
                     Metadata.objects.filter(media=skipped_media).update(media=None)

tubesync/sync/tasks.py

@@ -45,7 +45,6 @@ from .utils import get_remote_image, resize_image_to_height, filter_response
 from .youtube import YouTubeError
 
 atomic = db.transaction.atomic
-db_vendor = db.connection.vendor
 register_huey_signals()
 
 
@@ -128,11 +127,9 @@ def update_task_status(task, status):
     else:
         task.verbose_name = f'[{status}] {task._verbose_name}'
     try:
-        task.save(update_fields={'verbose_name'})
+        retry_django_db(3)(task.save)(update_fields={'verbose_name'})
     except db.DatabaseError as e:
-        if 'Save with update_fields did not affect any rows.' == str(e):
-            pass
-        else:
+        if 'Save with update_fields did not affect any rows.' != str(e):
             raise
     return True
 
@@ -207,7 +204,7 @@ def cleanup_completed_tasks():
 def save_model(instance):
     with atomic(durable=False):
         instance.save()
-    if 'sqlite' != db_vendor:
+    if 'sqlite' != db.connection.vendor:
         return
 
     # work around for SQLite and its many
@@ -785,6 +782,13 @@ def delete_media(media_id):
             f'media:{media.uuid}',
             queue=Val(TaskQueue.DB),
         ):
+            # remove thumbnail from storage
+            if media.thumb_file_exists:
+                # copy it before its gone
+                if media.media_file_exists:
+                    media.copy_thumbnail()
+                media.thumb.delete(save=False)
+            # intentionally not removing downloaded file here
             media.delete()
 
 
@@ -911,7 +915,7 @@ def download_media_metadata(media_id):
                     log.debug(number(parts))
                     media.published = published_datetime
                     media.manual_skip = True
-                    media.save()
+                    save_model(media)
                     raise_exception = False
             except (ValueError, IndexError, OverflowError):
                 log.exception('could not assign published_datetime')
@@ -958,7 +962,7 @@ def download_media_metadata(media_id):
 
     # Don't filter media here, the post_save signal will handle that
     try:
-        media.save()
+        save_model(media)
     # ruff: ignore[TRY203]
     except Exception:
         raise
@@ -1004,18 +1008,20 @@ def download_media_image(media_id, url):
     image_file = BytesIO()
     i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True)
     image_file.seek(0)
+    thumbnail_bytes = image_file.read()
+    i = image_file = None
     if media.thumb_file_exists:
         media.thumb.delete(save=False)
-    media.thumb.save(
+    retry_django_db(3)(media.thumb.save)(
         'thumb',
         SimpleUploadedFile(
             'thumb',
-            image_file.read(),
+            thumbnail_bytes,
             'image/jpeg',
         ),
         save=True
     )
-    i = image_file = None
+    thumbnail_bytes = None
     log.info(f'Saved thumbnail for: {media} from: {url}')
     # After media is downloaded, copy the updated thumbnail.
     if media.downloaded and media.thumb_file_exists:
@@ -1114,7 +1120,7 @@ def download_media_file(media_id, override=False, *, task=None):
 
             # Media has been downloaded successfully
             media.download_finished(format_str, container, filepath)
-            media.save()
+            save_model(media)
             media.rename_files()
             media.copy_thumbnail()
             media.write_nfo_file()
@@ -1128,6 +1134,17 @@ def download_media_file(media_id, override=False, *, task=None):
             schedule_media_servers_update()
         finally:
             watchdog_result.revoke()
+            try:
+                task = TaskHistory.objects.get(task_id=watchdog_result.id)
+            except TaskHistory.DoesNotExist:
+                pass
+            else:
+                if watchdog_result.is_revoked():
+                    default_verbose_name = f'{task.name}: {task.task_id}'
+                    update_model(
+                        task,
+                        verbose_name=f'[revoked] {task.verbose_name or default_verbose_name}',
+                    )
 
 
 @db_task(priority=90, retries=5, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.NET))
@@ -1240,7 +1257,7 @@ def refresh_formats(media_id):
             raise exc
         # the metadata has already been saved, trigger the post_save signal
         log.info(f'Saving refreshed formats for "{media.key}": {msg}')
-        media.save()
+        save_model(media)
 
 
 @db_task(delay=300, priority=80, retries=5, retry_delay=600, queue=Val(TaskQueue.FS))
@@ -1382,29 +1399,61 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
         log.warning(f'Task delete_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
         # this task can run after a source was deleted
-    mqs = Media.objects.all().defer(
-        'metadata',
-    ).filter(
-        source=source or source_id,
-    )
-    # no delay for these tasks
-    delete_media.map({
-        str(media.pk)
-        for media in qs_gen(mqs)
-    })
-    with atomic(durable=True):
-        mqs.update(manual_skip=True, skip=True)
-        log.info(f'Deleting media for source: {source_name}')
-        mqs.delete()
-    # Remove the directory, if the user requested that
+    else:
+        source_lock = huey_lock_task(
+            f'source:{source.uuid}',
+            queue=Val(TaskQueue.FS),
+        )
+        if source_lock.acquired:
+            raise CancelExecution(_('already locked'), retry=True)
+        source_lock.acquired = True
+        update_model(
+            source,
+            key = source.key + '/deleted',
+            name = f'[Deleting] {source.name}',
+        )
     directory_path = Path(source_directory)
     remove = (
         (source and source.delete_removed_media) or
         (directory_path / '.to_be_removed').is_file()
     )
-    if source:
+    mqs = Media.objects.all().defer(
+        'metadata',
+    ).filter(
+        source=source or source_id,
+    )
+
+    for media in qs_gen(mqs):
+        # remove thumbnail from storage
+        if media.thumb_file_exists:
+            # copy it before its gone
+            if media.media_file_exists:
+                media.copy_thumbnail()
+            media.thumb.delete(save=False)
+        # if requested, remove download from storage
+        if remove and media.media_file_exists:
+            media.media_file.delete(save=False)
+        # no delay for these tasks
+        delete_media(str(media.pk))
+
+    try:
         with atomic(durable=True):
-            source.delete()
+            mqs.update(manual_skip=True, skip=True)
+            log.info(f'Deleting media for source: {source_name}')
+            mqs.delete()
+        if source:
+            try:
+                source = Source.objects.get(uuid=source.uuid)
+            except Source.DoesNotExist:
+                source = False
+            else:
+                with atomic(durable=True):
+                    source.delete()
+                    source = False
+    finally:
+        if source is not None:
+            source_lock.acquired = False
+    # Remove the directory, if the user requested that
     if remove:
         log.info(f'Deleting directory for: {source_name}: {directory_path}')
         rmtree(directory_path, True)

tubesync/tubesync/settings.py

@@ -376,6 +376,9 @@ YOUTUBE_DEFAULTS = {
         'youtubepot-bgutilhttp': {
             'base_url': ['http://127.0.0.1:4416'],
         },
+        'youtubepot-bgutilscript': {
+            'server_home': ['/app/bgutil-ytdlp-pot-provider/server'],
+        },
     },
     'postprocessor_args': {
         'videoremuxer+ffmpeg': ['-bsf:v', 'setts=pts=DTS'],
@@ -383,6 +386,7 @@ YOUTUBE_DEFAULTS = {
     },
     'js_runtimes': {
         'deno': {'path': None,},
+        'node': {'path': '/usr/bin',},
         'quickjs': {'path': None,},
     },
 }