diff --git a/tubesync/install_bun.sh b/tubesync/install_bun.sh index 5ee97e29..3f402919 100755 --- a/tubesync/install_bun.sh +++ b/tubesync/install_bun.sh @@ -72,7 +72,7 @@ extract_bun() { unzip -u -o -d './.bun' "${fn}" && install -v -T ./.bun/bun-linux-*/bun "${_staged}" && { # bun spawning unzip hangs for an unknown reason fairly often - local _attempt ; for _attempt in {1..3} ; do + local _attempt ; for _attempt in {1..5} ; do "${_staged}" run "${HERE}/verify_bun.ts" --release 'bun-v1.3.14' --asset "${fn}" --install-dir "${dest_dir}" && break || sleep "${_attempt}" done ; diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 435d9173..9b38d831 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -148,9 +148,8 @@ def source_pre_delete(sender, instance, **kwargs): instance.deactivate() # Fetch the media source - sqs = Source.objects.filter(filter_text=str(source.pk)) - if sqs.count(): - media_source = sqs[0] + media_source = Source.objects.filter(filter_text=str(source.pk)).first() + if media_source: # Schedule deletion of media on_commit(partial( TaskHistory.schedule, @@ -260,7 +259,7 @@ def media_post_save(sender, instance, created, **kwargs): @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): # Remove thumbnail file for deleted media - if instance.thumb: + if instance.thumb_file_exists: instance.thumb.delete(save=False) # Save the metadata site & thumbnail URL to the metadata column existing_metadata = instance.loaded_metadata @@ -352,9 +351,14 @@ def media_post_delete(sender, instance, **kwargs): created = False create_for_indexing_task = ( not ( - #not instance.downloaded and - instance.skip and - instance.manual_skip + ( + instance.source and + instance.source.key.endswith('/deleted') + ) or + ( + instance.skip and + instance.manual_skip + ) ) ) if create_for_indexing_task: @@ -391,7 +395,7 @@ def media_post_delete(sender, instance, **kwargs): key=skipped_media.key, ) try: - if instance_qs.count(): + if instance_qs.exists(): with atomic(durable=False): # clear the link to a media instance Metadata.objects.filter(media=skipped_media).update(media=None) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 8fdfda96..750fc906 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -45,7 +45,6 @@ from .utils import get_remote_image, resize_image_to_height, filter_response from .youtube import YouTubeError atomic = db.transaction.atomic -db_vendor = db.connection.vendor register_huey_signals() @@ -128,11 +127,9 @@ def update_task_status(task, status): else: task.verbose_name = f'[{status}] {task._verbose_name}' try: - task.save(update_fields={'verbose_name'}) + retry_django_db(3)(task.save)(update_fields={'verbose_name'}) except db.DatabaseError as e: - if 'Save with update_fields did not affect any rows.' == str(e): - pass - else: + if 'Save with update_fields did not affect any rows.' != str(e): raise return True @@ -207,7 +204,7 @@ def cleanup_completed_tasks(): def save_model(instance): with atomic(durable=False): instance.save() - if 'sqlite' != db_vendor: + if 'sqlite' != db.connection.vendor: return # work around for SQLite and its many @@ -785,6 +782,13 @@ def delete_media(media_id): f'media:{media.uuid}', queue=Val(TaskQueue.DB), ): + # remove thumbnail from storage + if media.thumb_file_exists: + # copy it before its gone + if media.media_file_exists: + media.copy_thumbnail() + media.thumb.delete(save=False) + # intentionally not removing downloaded file here media.delete() @@ -911,7 +915,7 @@ def download_media_metadata(media_id): log.debug(number(parts)) media.published = published_datetime media.manual_skip = True - media.save() + save_model(media) raise_exception = False except (ValueError, IndexError, OverflowError): log.exception('could not assign published_datetime') @@ -958,7 +962,7 @@ def download_media_metadata(media_id): # Don't filter media here, the post_save signal will handle that try: - media.save() + save_model(media) # ruff: ignore[TRY203] except Exception: raise @@ -1004,18 +1008,20 @@ def download_media_image(media_id, url): image_file = BytesIO() i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True) image_file.seek(0) + thumbnail_bytes = image_file.read() + i = image_file = None if media.thumb_file_exists: media.thumb.delete(save=False) - media.thumb.save( + retry_django_db(3)(media.thumb.save)( 'thumb', SimpleUploadedFile( 'thumb', - image_file.read(), + thumbnail_bytes, 'image/jpeg', ), save=True ) - i = image_file = None + thumbnail_bytes = None log.info(f'Saved thumbnail for: {media} from: {url}') # After media is downloaded, copy the updated thumbnail. if media.downloaded and media.thumb_file_exists: @@ -1114,7 +1120,7 @@ def download_media_file(media_id, override=False, *, task=None): # Media has been downloaded successfully media.download_finished(format_str, container, filepath) - media.save() + save_model(media) media.rename_files() media.copy_thumbnail() media.write_nfo_file() @@ -1128,6 +1134,17 @@ def download_media_file(media_id, override=False, *, task=None): schedule_media_servers_update() finally: watchdog_result.revoke() + try: + task = TaskHistory.objects.get(task_id=watchdog_result.id) + except TaskHistory.DoesNotExist: + pass + else: + if watchdog_result.is_revoked(): + default_verbose_name = f'{task.name}: {task.task_id}' + update_model( + task, + verbose_name=f'[revoked] {task.verbose_name or default_verbose_name}', + ) @db_task(priority=90, retries=5, backoff_class=DjangoBackgroundTasksBackoff, task_base=AttemptsTask, queue=Val(TaskQueue.NET)) @@ -1240,7 +1257,7 @@ def refresh_formats(media_id): raise exc # the metadata has already been saved, trigger the post_save signal log.info(f'Saving refreshed formats for "{media.key}": {msg}') - media.save() + save_model(media) @db_task(delay=300, priority=80, retries=5, retry_delay=600, queue=Val(TaskQueue.FS)) @@ -1382,29 +1399,61 @@ def delete_all_media_for_source(source_id, source_name, source_directory): log.warning(f'Task delete_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') # this task can run after a source was deleted - mqs = Media.objects.all().defer( - 'metadata', - ).filter( - source=source or source_id, - ) - # no delay for these tasks - delete_media.map({ - str(media.pk) - for media in qs_gen(mqs) - }) - with atomic(durable=True): - mqs.update(manual_skip=True, skip=True) - log.info(f'Deleting media for source: {source_name}') - mqs.delete() - # Remove the directory, if the user requested that + else: + source_lock = huey_lock_task( + f'source:{source.uuid}', + queue=Val(TaskQueue.FS), + ) + if source_lock.acquired: + raise CancelExecution(_('already locked'), retry=True) + source_lock.acquired = True + update_model( + source, + key = source.key + '/deleted', + name = f'[Deleting] {source.name}', + ) directory_path = Path(source_directory) remove = ( (source and source.delete_removed_media) or (directory_path / '.to_be_removed').is_file() ) - if source: + mqs = Media.objects.all().defer( + 'metadata', + ).filter( + source=source or source_id, + ) + + for media in qs_gen(mqs): + # remove thumbnail from storage + if media.thumb_file_exists: + # copy it before its gone + if media.media_file_exists: + media.copy_thumbnail() + media.thumb.delete(save=False) + # if requested, remove download from storage + if remove and media.media_file_exists: + media.media_file.delete(save=False) + # no delay for these tasks + delete_media(str(media.pk)) + + try: with atomic(durable=True): - source.delete() + mqs.update(manual_skip=True, skip=True) + log.info(f'Deleting media for source: {source_name}') + mqs.delete() + if source: + try: + source = Source.objects.get(uuid=source.uuid) + except Source.DoesNotExist: + source = False + else: + with atomic(durable=True): + source.delete() + source = False + finally: + if source is not None: + source_lock.acquired = False + # Remove the directory, if the user requested that if remove: log.info(f'Deleting directory for: {source_name}: {directory_path}') rmtree(directory_path, True) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index c505c67b..f7dcab0f 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -376,6 +376,9 @@ YOUTUBE_DEFAULTS = { 'youtubepot-bgutilhttp': { 'base_url': ['http://127.0.0.1:4416'], }, + 'youtubepot-bgutilscript': { + 'server_home': ['/app/bgutil-ytdlp-pot-provider/server'], + }, }, 'postprocessor_args': { 'videoremuxer+ffmpeg': ['-bsf:v', 'setts=pts=DTS'], @@ -383,6 +386,7 @@ YOUTUBE_DEFAULTS = { }, 'js_runtimes': { 'deno': {'path': None,}, + 'node': {'path': '/usr/bin',}, 'quickjs': {'path': None,}, }, }