diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run index da22666b..b2c3a841 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-worker/run @@ -1,4 +1,4 @@ #!/command/with-contenv bash -exec s6-setuidgid app \ +exec nice -n "${TUBESYNC_NICE:-1}" s6-setuidgid app \ /usr/bin/python3 /app/manage.py process_tasks diff --git a/tubesync/sync/fields.py b/tubesync/sync/fields.py index b96f4cd2..31889024 100644 --- a/tubesync/sync/fields.py +++ b/tubesync/sync/fields.py @@ -3,6 +3,25 @@ from django.db import models from typing import Any, Optional, Dict from django.utils.translation import gettext_lazy as _ + +# as stolen from: +# - https://wiki.sponsor.ajay.app/w/Types +# - https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py +# +# The spacing is a little odd, it is for easy copy/paste selection. +# Please don't change it. +# Every possible category fits in a string < 128 characters +class SponsorBlock_Category(models.TextChoices): + SPONSOR = 'sponsor', _( 'Sponsor' ) + INTRO = 'intro', _( 'Intermission/Intro Animation' ) + OUTRO = 'outro', _( 'Endcards/Credits' ) + SELFPROMO = 'selfpromo', _( 'Unpaid/Self Promotion' ) + PREVIEW = 'preview', _( 'Preview/Recap' ) + FILLER = 'filler', _( 'Filler Tangent' ) + INTERACTION = 'interaction', _( 'Interaction Reminder' ) + MUSIC_OFFTOPIC = 'music_offtopic', _( 'Non-Music Section' ) + + # this is a form field! class CustomCheckboxSelectMultiple(CheckboxSelectMultiple): template_name = 'widgets/checkbox_select.html' @@ -32,24 +51,28 @@ class CustomCheckboxSelectMultiple(CheckboxSelectMultiple): class CommaSepChoiceField(models.Field): "Implements comma-separated storage of lists" - def __init__(self, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, *args, **kwargs): - self.separator = separator + # If 'text' isn't correct add the vendor override here. + _DB_TYPES = {} + + def __init__(self, *args, separator=",", possible_choices=(("","")), all_choice="", all_label="All", allow_all=False, **kwargs): + super().__init__(*args, **kwargs) + self.separator = str(separator) self.possible_choices = possible_choices self.selected_choices = [] self.allow_all = allow_all self.all_label = all_label self.all_choice = all_choice - super().__init__(*args, **kwargs) def deconstruct(self): name, path, args, kwargs = super().deconstruct() - if self.separator != ",": + if ',' != self.separator: kwargs['separator'] = self.separator kwargs['possible_choices'] = self.possible_choices return name, path, args, kwargs def db_type(self, connection): - return 'text' + value = self._DB_TYPES.get(connection.vendor, None) + return value if value is not None else 'text' def get_my_choices(self): choiceArray = [] @@ -60,7 +83,7 @@ class CommaSepChoiceField(models.Field): for t in self.possible_choices: choiceArray.append(t) - + return choiceArray def formfield(self, **kwargs): @@ -72,21 +95,13 @@ class CommaSepChoiceField(models.Field): 'label': '', 'required': False} defaults.update(kwargs) - #del defaults.required return super().formfield(**defaults) - def deconstruct(self): - name, path, args, kwargs = super().deconstruct() - # Only include kwarg if it's not the default - if self.separator != ",": - kwargs['separator'] = self.separator - return name, path, args, kwargs - def from_db_value(self, value, expr, conn): - if value is None: + if 0 == len(value) or value is None: self.selected_choices = [] else: - self.selected_choices = value.split(",") + self.selected_choices = value.split(self.separator) return self @@ -97,7 +112,7 @@ class CommaSepChoiceField(models.Field): return "" if self.all_choice not in value: - return ",".join(value) + return self.separator.join(value) else: return self.all_choice diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index b5d7be06..877b62e6 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -14,16 +14,18 @@ from django.core.validators import RegexValidator from django.utils.text import slugify from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from common.logger import log from common.errors import NoFormatException from common.utils import clean_filename, clean_emoji from .youtube import (get_media_info as get_youtube_media_info, download_media as download_youtube_media, get_channel_image_info as get_youtube_channel_image_info) -from .utils import seconds_to_timestr, parse_media_format, filter_response +from .utils import (seconds_to_timestr, parse_media_format, filter_response, + write_text_file, mkdir_p, directory_and_stem, glob_quote) from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) from .mediaservers import PlexMediaServer -from .fields import CommaSepChoiceField +from .fields import CommaSepChoiceField, SponsorBlock_Category media_file_storage = FileSystemStorage(location=str(settings.DOWNLOAD_ROOT), base_url='/media-data/') @@ -114,21 +116,9 @@ class Source(models.Model): EXTENSION_MKV = 'mkv' EXTENSIONS = (EXTENSION_M4A, EXTENSION_OGG, EXTENSION_MKV) - # as stolen from: https://wiki.sponsor.ajay.app/w/Types / https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/postprocessor/sponsorblock.py - SPONSORBLOCK_CATEGORIES_CHOICES = ( - ('sponsor', 'Sponsor'), - ('intro', 'Intermission/Intro Animation'), - ('outro', 'Endcards/Credits'), - ('selfpromo', 'Unpaid/Self Promotion'), - ('preview', 'Preview/Recap'), - ('filler', 'Filler Tangent'), - ('interaction', 'Interaction Reminder'), - ('music_offtopic', 'Non-Music Section'), - ) - sponsorblock_categories = CommaSepChoiceField( _(''), - possible_choices=SPONSORBLOCK_CATEGORIES_CHOICES, + possible_choices=SponsorBlock_Category.choices, all_choice='all', allow_all=True, all_label='(all options)', @@ -537,7 +527,7 @@ class Source(models.Model): def get_image_url(self): if self.source_type == self.SOURCE_TYPE_YOUTUBE_PLAYLIST: raise SuspiciousOperation('This source is a playlist so it doesn\'t have thumbnail.') - + return get_youtube_channel_image_info(self.url) @@ -967,7 +957,7 @@ class Media(models.Model): def get_best_video_format(self): return get_best_video_format(self) - + def get_format_str(self): ''' Returns a youtube-dl compatible format string for the best matches @@ -992,7 +982,7 @@ class Media(models.Model): else: return False return False - + def get_display_format(self, format_str): ''' Returns a tuple used in the format component of the output filename. This @@ -1153,9 +1143,9 @@ class Media(models.Model): from common.utils import json_serial old_mdl = len(self.metadata or "") - data = json.loads(self.metadata or "") + data = json.loads(self.metadata or "{}") compact_json = json.dumps(data, separators=(',', ':'), default=json_serial) - + filtered_data = filter_response(data, True) filtered_json = json.dumps(filtered_data, separators=(',', ':'), default=json_serial) except Exception as e: @@ -1302,13 +1292,20 @@ class Media(models.Model): def filepath(self): return self.source.directory_path / self.filename - @property - def thumbname(self): + def filename_prefix(self): if self.downloaded and self.media_file: filename = self.media_file.path else: filename = self.filename + # The returned prefix should not contain any directories. + # So, we do not care about the different directories + # used for filename in the cases above. prefix, ext = os.path.splitext(os.path.basename(filename)) + return prefix + + @property + def thumbname(self): + prefix = self.filename_prefix() return f'{prefix}.jpg' @property @@ -1317,26 +1314,18 @@ class Media(models.Model): @property def nfoname(self): - if self.downloaded and self.media_file: - filename = self.media_file.path - else: - filename = self.filename - prefix, ext = os.path.splitext(os.path.basename(filename)) + prefix = self.filename_prefix() return f'{prefix}.nfo' - + @property def nfopath(self): return self.directory_path / self.nfoname @property def jsonname(self): - if self.downloaded and self.media_file: - filename = self.media_file.path - else: - filename = self.filename - prefix, ext = os.path.splitext(os.path.basename(filename)) + prefix = self.filename_prefix() return f'{prefix}.info.json' - + @property def jsonpath(self): return self.directory_path / self.jsonname @@ -1564,9 +1553,82 @@ class Media(models.Model): if use_padding: return f'{episode_number:02}' - + return str(episode_number) + def rename_files(self): + if self.downloaded and self.media_file: + old_video_path = Path(self.media_file.path) + new_video_path = Path(get_media_file_path(self, None)) + if old_video_path.exists() and not new_video_path.exists(): + old_video_path = old_video_path.resolve(strict=True) + + # move video to destination + mkdir_p(new_video_path.parent) + log.debug(f'{self!s}: {old_video_path!s} => {new_video_path!s}') + old_video_path.rename(new_video_path) + log.info(f'Renamed video file for: {self!s}') + + # collect the list of files to move + # this should not include the video we just moved + (old_prefix_path, old_stem) = directory_and_stem(old_video_path) + other_paths = list(old_prefix_path.glob(glob_quote(old_stem) + '*')) + log.info(f'Collected {len(other_paths)} other paths for: {self!s}') + + # adopt orphaned files, if possible + media_format = str(self.source.media_format) + top_dir_path = Path(self.source.directory_path) + if '{key}' in media_format: + fuzzy_paths = list(top_dir_path.rglob('*' + glob_quote(str(self.key)) + '*')) + log.info(f'Collected {len(fuzzy_paths)} fuzzy paths for: {self!s}') + + if new_video_path.exists(): + new_video_path = new_video_path.resolve(strict=True) + + # update the media_file in the db + self.media_file.name = str(new_video_path.relative_to(self.media_file.storage.location)) + self.save() + log.info(f'Updated "media_file" in the database for: {self!s}') + + (new_prefix_path, new_stem) = directory_and_stem(new_video_path) + + # move and change names to match stem + for other_path in other_paths: + old_file_str = other_path.name + new_file_str = new_stem + old_file_str[len(old_stem):] + new_file_path = Path(new_prefix_path / new_file_str) + log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}') + # it should exist, but check anyway + if other_path.exists(): + log.debug(f'{self!s}: {other_path!s} => {new_file_path!s}') + other_path.replace(new_file_path) + + for fuzzy_path in fuzzy_paths: + (fuzzy_prefix_path, fuzzy_stem) = directory_and_stem(fuzzy_path) + old_file_str = fuzzy_path.name + new_file_str = new_stem + old_file_str[len(fuzzy_stem):] + new_file_path = Path(new_prefix_path / new_file_str) + log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}') + # it quite possibly was renamed already + if fuzzy_path.exists() and not new_file_path.exists(): + log.debug(f'{self!s}: {fuzzy_path!s} => {new_file_path!s}') + fuzzy_path.rename(new_file_path) + + # The thumbpath inside the .nfo file may have changed + if self.source.write_nfo and self.source.copy_thumbnails: + write_text_file(new_prefix_path / self.nfopath.name, self.nfoxml) + log.info(f'Wrote new ".nfo" file for: {self!s}') + + # try to remove empty dirs + parent_dir = old_video_path.parent + try: + while parent_dir.is_dir(): + parent_dir.rmdir() + log.info(f'Removed empty directory: {parent_dir!s}') + parent_dir = parent_dir.parent + except OSError as e: + pass + class MediaServer(models.Model): ''' diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index cd8cf621..92b1e7ec 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -12,7 +12,8 @@ from .tasks import (delete_task_by_source, delete_task_by_media, index_source_ta download_media_thumbnail, download_media_metadata, map_task_to_instance, check_source_directory_exists, download_media, rescan_media_server, download_source_images, - save_all_media_for_source, get_media_metadata_task) + save_all_media_for_source, rename_all_media_for_source, + get_media_metadata_task) from .utils import delete_file from .filtering import filter_media @@ -53,7 +54,7 @@ def source_post_save(sender, instance, created, **kwargs): if instance.source_type != Source.SOURCE_TYPE_YOUTUBE_PLAYLIST and instance.copy_channel_images: download_source_images( str(instance.pk), - priority=0, + priority=2, verbose_name=verbose_name.format(instance.name) ) if instance.index_schedule > 0: @@ -68,10 +69,28 @@ def source_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) + # Check settings before any rename tasks are scheduled + rename_sources_setting = settings.RENAME_SOURCES or list() + create_rename_task = ( + ( + instance.directory and + instance.directory in rename_sources_setting + ) or + settings.RENAME_ALL_SOURCES + ) + if create_rename_task: + verbose_name = _('Renaming all media for source "{}"') + rename_all_media_for_source( + str(instance.pk), + queue=str(instance.pk), + priority=1, + verbose_name=verbose_name.format(instance.name), + remove_existing_tasks=False + ) verbose_name = _('Checking all media for source "{}"') save_all_media_for_source( str(instance.pk), - priority=0, + priority=2, verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) @@ -174,7 +193,7 @@ def media_post_save(sender, instance, created, **kwargs): download_media( str(instance.pk), queue=str(instance.source.pk), - priority=15, + priority=10, verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index ab92e2c8..d677df40 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -51,6 +51,7 @@ def map_task_to_instance(task): 'sync.tasks.download_media': Media, 'sync.tasks.download_media_metadata': Media, 'sync.tasks.save_all_media_for_source': Source, + 'sync.tasks.rename_all_media_for_source': Source, } MODEL_URL_MAP = { Source: 'sync:source', @@ -442,10 +443,18 @@ def download_media(media_id): media.downloaded_format = 'audio' media.save() # If selected, copy the thumbnail over as well - if media.source.copy_thumbnails and media.thumb: - log.info(f'Copying media thumbnail from: {media.thumb.path} ' - f'to: {media.thumbpath}') - copyfile(media.thumb.path, media.thumbpath) + if media.source.copy_thumbnails: + if not media.thumb_file_exists: + thumbnail_url = media.thumbnail + if thumbnail_url: + args = ( str(media.pk), thumbnail_url, ) + delete_task_by_media('sync.tasks.download_media_thumbnail', args) + if download_media_thumbnail.now(*args): + media.refresh_from_db() + if media.thumb_file_exists: + log.info(f'Copying media thumbnail from: {media.thumb.path} ' + f'to: {media.thumbpath}') + copyfile(media.thumb.path, media.thumbpath) # If selected, write an NFO file if media.source.write_nfo: log.info(f'Writing media NFO file to: {media.nfopath}') @@ -508,3 +517,18 @@ def save_all_media_for_source(source_id): # flags may need to be recalculated for media in Media.objects.filter(source=source): media.save() + + +@background(schedule=0) +def rename_all_media_for_source(source_id): + try: + source = Source.objects.get(pk=source_id) + except Source.DoesNotExist: + # Task triggered but the source no longer exists, do nothing + log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no ' + f'source exists with ID: {source_id}') + return + for media in Media.objects.filter(source=source): + media.rename_files() + + diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html index 026e5a54..d54f4efd 100644 --- a/tubesync/sync/templates/sync/media-item.html +++ b/tubesync/sync/templates/sync/media-item.html @@ -1,4 +1,4 @@ -{% extends 'base.html' %}{% load static %} +{% extends 'base.html' %}{% load static %}{% load humanize %} {% block headtitle %}Media - {{ media.key }}{% endblock %} @@ -142,12 +142,12 @@