From bcb331429ef93550df7dce8661ff928204216a1d Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 09:38:43 -0400 Subject: [PATCH 1/8] fix: various adjustments in tasks --- tubesync/sync/tasks.py | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ae759ccf..1021c9e3 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -14,7 +14,7 @@ from collections import deque as queue from io import BytesIO from pathlib import Path from datetime import timedelta -from shutil import copyfile, rmtree +from shutil import rmtree from django import db from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile @@ -45,7 +45,6 @@ from .utils import get_remote_image, resize_image_to_height, filter_response from .youtube import YouTubeError atomic = db.transaction.atomic -db_vendor = db.connection.vendor register_huey_signals() @@ -128,11 +127,9 @@ def update_task_status(task, status): else: task.verbose_name = f'[{status}] {task._verbose_name}' try: - task.save(update_fields={'verbose_name'}) + retry_django_db(3)(task.save)(update_fields={'verbose_name'}) except db.DatabaseError as e: - if 'Save with update_fields did not affect any rows.' == str(e): - pass - else: + if 'Save with update_fields did not affect any rows.' != str(e): raise return True @@ -207,7 +204,7 @@ def cleanup_completed_tasks(): def save_model(instance): with atomic(durable=False): instance.save() - if 'sqlite' != db_vendor: + if 'sqlite' != db.connection.vendor: return # work around for SQLite and its many @@ -917,7 +914,7 @@ def download_media_metadata(media_id): log.debug(number(parts)) media.published = published_datetime media.manual_skip = True - media.save() + save_model(media) raise_exception = False except (ValueError, IndexError, OverflowError): log.exception('could not assign published_datetime') @@ -964,7 +961,7 @@ def download_media_metadata(media_id): # Don't filter media here, the post_save signal will handle that try: - media.save() + save_model(media) # ruff: ignore[TRY203] except Exception: raise @@ -1010,27 +1007,22 @@ def download_media_image(media_id, url): image_file = BytesIO() i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True) image_file.seek(0) - media.thumb.save( + thumbnail_bytes = image_file.read() + i = image_file = None + retry_django_db(3)(media.thumb.save)( 'thumb', SimpleUploadedFile( 'thumb', - image_file.read(), + thumbnail_bytes, 'image/jpeg', ), save=True ) - i = image_file = None + thumbnail_bytes = None log.info(f'Saved thumbnail for: {media} from: {url}') # After media is downloaded, copy the updated thumbnail. - copy_thumbnail = ( - media.downloaded and - media.source.copy_thumbnails and - media.thumb_file_exists - ) - if copy_thumbnail: - log.info(f'Copying media thumbnail from: {media.thumb.path} ' - f'to: {media.thumbpath}') - copyfile(media.thumb.path, media.thumbpath) + if media.downloaded and media.thumb_file_exists: + media.copy_thumbnail() return True @huey_signal(huey_signals.SIGNAL_COMPLETE, queue=Val(TaskQueue.NET)) @@ -1124,7 +1116,7 @@ def download_media_file(media_id, override=False, *, task=None): # Media has been downloaded successfully media.download_finished(format_str, container, filepath) - media.save() + save_model(media) media.rename_files() media.copy_thumbnail() media.write_nfo_file() @@ -1138,6 +1130,17 @@ def download_media_file(media_id, override=False, *, task=None): schedule_media_servers_update() finally: watchdog_result.revoke() + try: + task = TaskHistory.objects.get(task_id=watchdog_result.id) + except TaskHistory.DoesNotExist: + pass + else: + if watchdog_result.is_revoked(): + default_verbose_name = f'{task.name}: {task.task_id}' + update_model( + task, + verbose_name=f'[revoked] {task.verbose_name or default_verbose_name}', + ) @db_task(delay=30, expires=210, priority=100, queue=Val(TaskQueue.NET)) @@ -1234,7 +1237,7 @@ def refresh_formats(media_id): raise exc # the metadata has already been saved, trigger the post_save signal log.info(f'Saving refreshed formats for "{media.key}": {msg}') - media.save() + save_model(media) @db_task(delay=300, priority=80, retries=5, retry_delay=600, queue=Val(TaskQueue.FS)) From 4655ec140e84623d855d89339fc71ccc421aa9d1 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 10:52:23 -0400 Subject: [PATCH 2/8] fix: lock the source while deleting This prevents `save_all_media_for_source` from running while we are trying to delete the media instances. --- tubesync/sync/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1021c9e3..1ff4d7c3 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -1379,6 +1379,12 @@ def delete_all_media_for_source(source_id, source_name, source_directory): log.warning(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') # this task can run after a source was deleted + if source: + source_lock = huey_lock_task( + f'source:{source.uuid}', + queue=Val(TaskQueue.FS), + ) + source_lock.acquired = True mqs = Media.objects.all().defer( 'metadata', ).filter( @@ -1402,6 +1408,7 @@ def delete_all_media_for_source(source_id, source_name, source_directory): if source: with atomic(durable=True): source.delete() + source_lock.acquired = False if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From 47af4beaf69295ef94a54a4ffbe0f8ad1f931f45 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 15:04:48 -0400 Subject: [PATCH 3/8] fix: enhance source deletion Refactor media deletion logic to handle thumbnails and media files more effectively, ensuring proper cleanup and locking mechanisms are in place. --- tubesync/sync/tasks.py | 70 ++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 1ff4d7c3..0b4cfa14 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -788,6 +788,13 @@ def delete_media(media_id): f'media:{media.uuid}', queue=Val(TaskQueue.DB), ): + # remove thumbnail from storage + if media.thumb_file_exists: + # copy it before its gone + if media.media_file_exists: + media.copy_thumbnail() + media.thumb.delete(save=False) + # intentionally not removing downloaded file here media.delete() @@ -1379,36 +1386,61 @@ def delete_all_media_for_source(source_id, source_name, source_directory): log.warning(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') # this task can run after a source was deleted - if source: + else: source_lock = huey_lock_task( f'source:{source.uuid}', queue=Val(TaskQueue.FS), ) + if source_lock.acquired: + raise CancelExecution(_('already locked'), retry=True) source_lock.acquired = True - mqs = Media.objects.all().defer( - 'metadata', - ).filter( - source=source or source_id, - ) - # no delay for these tasks - delete_media.map({ - str(media.pk) - for media in qs_gen(mqs) - }) - with atomic(durable=True): - mqs.update(manual_skip=True, skip=True) - log.info(f'Deleting media for source: {source_name}') - mqs.delete() - # Remove the directory, if the user requested that + update_model( + source, + key = source.key + '/deleted', + name = f'[Deleting] {source.name}', + ) directory_path = Path(source_directory) remove = ( (source and source.delete_removed_media) or (directory_path / '.to_be_removed').is_file() ) - if source: + mqs = Media.objects.all().defer( + 'metadata', + ).filter( + source=source or source_id, + ) + + for media in qs_gen(mqs): + # remove thumbnail from storage + if media.thumb_file_exists: + # copy it before its gone + if media.media_file_exists: + media.copy_thumbnail() + media.thumb.delete(save=False) + # if requested, remove download from storage + if remove and media.media_file_exists: + media.media_file.delete(save=False) + # no delay for these tasks + delete_media(str(media.pk)) + + try: with atomic(durable=True): - source.delete() - source_lock.acquired = False + mqs.update(manual_skip=True, skip=True) + log.info(f'Deleting media for source: {source_name}') + mqs.delete() + if source: + try: + source = Source.objects.get(uuid=source.uuid) + except Source.DoesNotExist: + source = False + else: + with atomic(durable=True): + source.delete() + source = False + finally: + if source is not None: + source_lock.acquired = False + # Remove the directory, if the user requested that if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) From acc94d1eb22748e0d177e03217dd8e1be15f89de Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 15:16:25 -0400 Subject: [PATCH 4/8] chore(scripts): increase retry attempts for bun --- tubesync/install_bun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/install_bun.sh b/tubesync/install_bun.sh index 5ee97e29..3f402919 100755 --- a/tubesync/install_bun.sh +++ b/tubesync/install_bun.sh @@ -72,7 +72,7 @@ extract_bun() { unzip -u -o -d './.bun' "${fn}" && install -v -T ./.bun/bun-linux-*/bun "${_staged}" && { # bun spawning unzip hangs for an unknown reason fairly often - local _attempt ; for _attempt in {1..3} ; do + local _attempt ; for _attempt in {1..5} ; do "${_staged}" run "${HERE}/verify_bun.ts" --release 'bun-v1.3.14' --asset "${fn}" --install-dir "${dest_dir}" && break || sleep "${_attempt}" done ; From 4d37bc9c8371c13f05e014aa19da3b8d925a9d16 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 15:39:23 -0400 Subject: [PATCH 5/8] fix: use the storage as the source of truth --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 435d9173..bdd1cde8 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -260,7 +260,7 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): # Remove thumbnail file for deleted media - if instance.thumb: + if instance.thumb_file_exists: instance.thumb.delete(save=False) # Save the metadata site & thumbnail URL to the metadata column existing_metadata = instance.loaded_metadata From cbc43222a5982f5c7e464396904b11fcdb55fe0e Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 16:29:05 -0400 Subject: [PATCH 6/8] fix: do not recreate `Media` instances when deleting the source --- tubesync/sync/signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index bdd1cde8..8ae4ade0 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -352,7 +352,7 @@ def media_post_delete(sender, instance, **kwargs): created = False create_for_indexing_task = ( not ( - #not instance.downloaded and + instance.source.key.endswith('/deleted') or instance.skip and instance.manual_skip ) From 9027b3af8f365e5cd932d30f6f13b4f25b7a685b Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 19:38:49 -0400 Subject: [PATCH 7/8] chore: tweak signals --- tubesync/sync/signals.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 8ae4ade0..9b38d831 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -148,9 +148,8 @@ def source_pre_delete(sender, instance, **kwargs): instance.deactivate() # Fetch the media source - sqs = Source.objects.filter(filter_text=str(source.pk)) - if sqs.count(): - media_source = sqs[0] + media_source = Source.objects.filter(filter_text=str(source.pk)).first() + if media_source: # Schedule deletion of media on_commit(partial( TaskHistory.schedule, @@ -352,9 +351,14 @@ def media_post_delete(sender, instance, **kwargs): created = False create_for_indexing_task = ( not ( - instance.source.key.endswith('/deleted') or - instance.skip and - instance.manual_skip + ( + instance.source and + instance.source.key.endswith('/deleted') + ) or + ( + instance.skip and + instance.manual_skip + ) ) ) if create_for_indexing_task: @@ -391,7 +395,7 @@ def media_post_delete(sender, instance, **kwargs): key=skipped_media.key, ) try: - if instance_qs.count(): + if instance_qs.exists(): with atomic(durable=False): # clear the link to a media instance Metadata.objects.filter(media=skipped_media).update(media=None) From b5bb2863a177bc35b5f8664ea438f0d9a0296a61 Mon Sep 17 00:00:00 2001 From: tcely Date: Wed, 23 Sep 2026 23:39:56 -0400 Subject: [PATCH 8/8] chore: add youtubepot-bgutilscript configuration This makes the script logging slightly less concerning and shorter. --- tubesync/tubesync/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index c505c67b..f7dcab0f 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -376,6 +376,9 @@ YOUTUBE_DEFAULTS = { 'youtubepot-bgutilhttp': { 'base_url': ['http://127.0.0.1:4416'], }, + 'youtubepot-bgutilscript': { + 'server_home': ['/app/bgutil-ytdlp-pot-provider/server'], + }, }, 'postprocessor_args': { 'videoremuxer+ffmpeg': ['-bsf:v', 'setts=pts=DTS'], @@ -383,6 +386,7 @@ YOUTUBE_DEFAULTS = { }, 'js_runtimes': { 'deno': {'path': None,}, + 'node': {'path': '/usr/bin',}, 'quickjs': {'path': None,}, }, }