wimby/tubesync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1604 from meeb/test-tasks-adjustments

Browse Source 
fix: improve source deletion
This commit is contained in:
tcely
2026-09-24 04:21:46 -04:00
committed by GitHub
parent b5bb2863a1 9027b3af8f
commit 49ee012bbb
3 changed files with 95 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tubesync/install_bun.sh
View File

@@ -72,7 +72,7 @@ extract_bun() {
unzip -u -o -d './.bun' "${fn}" &&
install -v -T ./.bun/bun-linux-*/bun "${_staged}" &&
{ # bun spawning unzip hangs for an unknown reason fairly often
local _attempt ; for _attempt in {1..3} ; do
local _attempt ; for _attempt in {1..5} ; do
"${_staged}" run "${HERE}/verify_bun.ts" --release 'bun-v1.3.14' --asset "${fn}" --install-dir "${dest_dir}" &&
break || sleep "${_attempt}"
done ;

20
tubesync/sync/signals.py
View File

@@ -148,9 +148,8 @@ def source_pre_delete(sender, instance, **kwargs):
instance.deactivate()
# Fetch the media source
sqs = Source.objects.filter(filter_text=str(source.pk))
if sqs.count():
media_source = sqs[0]
media_source = Source.objects.filter(filter_text=str(source.pk)).first()
if media_source:
# Schedule deletion of media
on_commit(partial(
TaskHistory.schedule,
@@ -260,7 +259,7 @@ def media_post_save(sender, instance, created, **kwargs):
@receiver(pre_delete, sender=Media)
def media_pre_delete(sender, instance, **kwargs):
# Remove thumbnail file for deleted media
if instance.thumb:
if instance.thumb_file_exists:
instance.thumb.delete(save=False)
# Save the metadata site & thumbnail URL to the metadata column
existing_metadata = instance.loaded_metadata
@@ -352,9 +351,14 @@ def media_post_delete(sender, instance, **kwargs):
created = False
create_for_indexing_task = (
not (
#not instance.downloaded and
instance.skip and
instance.manual_skip
(
instance.source and
instance.source.key.endswith('/deleted')
) or
(
instance.skip and
instance.manual_skip
)
)
)
if create_for_indexing_task:
@@ -391,7 +395,7 @@ def media_post_delete(sender, instance, **kwargs):
key=skipped_media.key,
)
try:
if instance_qs.count():
if instance_qs.exists():
with atomic(durable=False):
# clear the link to a media instance
Metadata.objects.filter(media=skipped_media).update(media=None)

122
tubesync/sync/tasks.py
View File

@@ -14,7 +14,7 @@ from collections import deque as queue
from io import BytesIO
from pathlib import Path
from datetime import timedelta
from shutil import copyfile, rmtree
from shutil import rmtree
from django import db
from django.conf import settings
from django.core.files.uploadedfile import SimpleUploadedFile
@@ -45,7 +45,6 @@ from .utils import get_remote_image, resize_image_to_height, filter_response
from .youtube import YouTubeError
atomic = db.transaction.atomic
db_vendor = db.connection.vendor
register_huey_signals()
@@ -128,11 +127,9 @@ def update_task_status(task, status):
else:
task.verbose_name = f'[{status}] {task._verbose_name}'
try:
task.save(update_fields={'verbose_name'})
retry_django_db(3)(task.save)(update_fields={'verbose_name'})
except db.DatabaseError as e:
if 'Save with update_fields did not affect any rows.' == str(e):
pass
else:
if 'Save with update_fields did not affect any rows.' != str(e):
raise
return True
@@ -207,7 +204,7 @@ def cleanup_completed_tasks():
def save_model(instance):
with atomic(durable=False):
instance.save()
if 'sqlite' != db_vendor:
if 'sqlite' != db.connection.vendor:
return
# work around for SQLite and its many
@@ -791,6 +788,13 @@ def delete_media(media_id):
f'media:{media.uuid}',
queue=Val(TaskQueue.DB),
):
# remove thumbnail from storage
if media.thumb_file_exists:
# copy it before its gone
if media.media_file_exists:
media.copy_thumbnail()
media.thumb.delete(save=False)
# intentionally not removing downloaded file here
media.delete()
@@ -917,7 +921,7 @@ def download_media_metadata(media_id):
log.debug(number(parts))
media.published = published_datetime
media.manual_skip = True
media.save()
save_model(media)
raise_exception = False
except (ValueError, IndexError, OverflowError):
log.exception('could not assign published_datetime')
@@ -964,7 +968,7 @@ def download_media_metadata(media_id):
# Don't filter media here, the post_save signal will handle that
try:
media.save()
save_model(media)
# ruff: ignore[TRY203]
except Exception:
raise
@@ -1010,27 +1014,22 @@ def download_media_image(media_id, url):
image_file = BytesIO()
i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True)
image_file.seek(0)
media.thumb.save(
thumbnail_bytes = image_file.read()
i = image_file = None
retry_django_db(3)(media.thumb.save)(
'thumb',
SimpleUploadedFile(
'thumb',
image_file.read(),
thumbnail_bytes,
'image/jpeg',
),
save=True
)
i = image_file = None
thumbnail_bytes = None
log.info(f'Saved thumbnail for: {media} from: {url}')
# After media is downloaded, copy the updated thumbnail.
copy_thumbnail = (
media.downloaded and
media.source.copy_thumbnails and
media.thumb_file_exists
)
if copy_thumbnail:
log.info(f'Copying media thumbnail from: {media.thumb.path} '
f'to: {media.thumbpath}')
copyfile(media.thumb.path, media.thumbpath)
if media.downloaded and media.thumb_file_exists:
media.copy_thumbnail()
return True
@huey_signal(huey_signals.SIGNAL_COMPLETE, queue=Val(TaskQueue.NET))
@@ -1124,7 +1123,7 @@ def download_media_file(media_id, override=False, *, task=None):
# Media has been downloaded successfully
media.download_finished(format_str, container, filepath)
media.save()
save_model(media)
media.rename_files()
media.copy_thumbnail()
media.write_nfo_file()
@@ -1138,6 +1137,17 @@ def download_media_file(media_id, override=False, *, task=None):
schedule_media_servers_update()
finally:
watchdog_result.revoke()
try:
task = TaskHistory.objects.get(task_id=watchdog_result.id)
except TaskHistory.DoesNotExist:
pass
else:
if watchdog_result.is_revoked():
default_verbose_name = f'{task.name}: {task.task_id}'
update_model(
task,
verbose_name=f'[revoked] {task.verbose_name or default_verbose_name}',
)
@db_task(delay=30, expires=210, priority=100, queue=Val(TaskQueue.NET))
@@ -1234,7 +1244,7 @@ def refresh_formats(media_id):
raise exc
# the metadata has already been saved, trigger the post_save signal
log.info(f'Saving refreshed formats for "{media.key}": {msg}')
media.save()
save_model(media)
@db_task(delay=300, priority=80, retries=5, retry_delay=600, queue=Val(TaskQueue.FS))
@@ -1376,29 +1386,61 @@ def delete_all_media_for_source(source_id, source_name, source_directory):
log.warning(f'Task delete_all_media_for_source(pk={source_id}) called but no '
f'source exists with ID: {source_id}')
# this task can run after a source was deleted
mqs = Media.objects.all().defer(
'metadata',
).filter(
source=source or source_id,
)
# no delay for these tasks
delete_media.map({
str(media.pk)
for media in qs_gen(mqs)
})
with atomic(durable=True):
mqs.update(manual_skip=True, skip=True)
log.info(f'Deleting media for source: {source_name}')
mqs.delete()
# Remove the directory, if the user requested that
else:
source_lock = huey_lock_task(
f'source:{source.uuid}',
queue=Val(TaskQueue.FS),
)
if source_lock.acquired:
raise CancelExecution(_('already locked'), retry=True)
source_lock.acquired = True
update_model(
source,
key = source.key + '/deleted',
name = f'[Deleting] {source.name}',
)
directory_path = Path(source_directory)
remove = (
(source and source.delete_removed_media) or
(directory_path / '.to_be_removed').is_file()
)
if source:
mqs = Media.objects.all().defer(
'metadata',
).filter(
source=source or source_id,
)
for media in qs_gen(mqs):
# remove thumbnail from storage
if media.thumb_file_exists:
# copy it before its gone
if media.media_file_exists:
media.copy_thumbnail()
media.thumb.delete(save=False)
# if requested, remove download from storage
if remove and media.media_file_exists:
media.media_file.delete(save=False)
# no delay for these tasks
delete_media(str(media.pk))
try:
with atomic(durable=True):
source.delete()
mqs.update(manual_skip=True, skip=True)
log.info(f'Deleting media for source: {source_name}')
mqs.delete()
if source:
try:
source = Source.objects.get(uuid=source.uuid)
except Source.DoesNotExist:
source = False
else:
with atomic(durable=True):
source.delete()
source = False
finally:
if source is not None:
source_lock.acquired = False
# Remove the directory, if the user requested that
if remove:
log.info(f'Deleting directory for: {source_name}: {directory_path}')
rmtree(directory_path, True)