Merge branch 'meeb:main' into jellyfin-mediaserver

Dockerfile

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
 # check=error=true
 
-ARG FFMPEG_DATE="2025-02-04-17-39"
+ARG FFMPEG_DATE="2025-02-18-14-16"
-ARG FFMPEG_VERSION="N-118421-ga8595dae0f"
+ARG FFMPEG_VERSION="N-118500-g08e37fa082"
 
 ARG S6_VERSION="3.2.0.2"
 
@@ -382,7 +382,8 @@ HEALTHCHECK --interval=1m --timeout=10s --start-period=3m CMD ["/app/healthcheck
 
 # ENVS and ports
 ENV PYTHONPATH="/app" \
-    PYTHONPYCACHEPREFIX="/config/cache/pycache"
+    PYTHONPYCACHEPREFIX="/config/cache/pycache" \
+    XDG_CACHE_HOME="/config/cache"
 EXPOSE 4848
 
 # Volumes

tubesync/sync/filtering.py

@@ -130,14 +130,7 @@ def filter_max_cap(instance: Media):
         return False
 
     max_cap_age = instance.source.download_cap_date
-    if not max_cap_age:
+    if max_cap_age and instance.published <= max_cap_age:
-        log.debug(
-            f"Media: {instance.source} / {instance} has not max_cap_age "
-            f"so not skipping based on max_cap_age"
-        )
-        return False
-
-    if instance.published <= max_cap_age:
         # log new media instances, not every media instance every time
         if not instance.skip:
             log.info(

tubesync/sync/models.py

@@ -822,6 +822,25 @@ class Media(models.Model):
                 if audio_format and video_format:
                     return f'{video_format}+{audio_format}'
                 else:
+                    # last resort: any combined format
+                    fallback_hd_cutoff = getattr(settings, 'VIDEO_HEIGHT_IS_HD', 500)
+
+                    for fmt in reversed(list(self.iter_formats())):
+                        select_fmt = (
+                            fmt.get('id') and
+                            fmt.get('acodec') and
+                            fmt.get('vcodec') and
+                            self.source.can_fallback and
+                            (
+                                (self.source.fallback == Val(Fallback.NEXT_BEST)) or
+                                (
+                                    self.source.fallback == Val(Fallback.NEXT_BEST_HD) and
+                                    (fmt.get('height') or 0) >= fallback_hd_cutoff
+                                )
+                            )
+                        )
+                        if select_fmt:
+                            return str(fmt.get('id'))
                     return False
         return False
  
@@ -1018,6 +1037,40 @@ class Media(models.Model):
         except Exception as e:
             return {}
 
+    @property
+    def refresh_formats(self):
+        data = self.loaded_metadata
+        metadata_seconds = data.get('epoch', None)
+        if not metadata_seconds:
+            self.metadata = None
+            return False
+
+        now = timezone.now()
+        formats_seconds = data.get('formats_epoch', metadata_seconds)
+        metadata_dt = self.metadata_published(formats_seconds)
+        if (now - metadata_dt) < timedelta(seconds=self.source.index_schedule):
+            return False
+
+        self.skip = False
+        metadata = self.index_metadata()
+        if self.skip:
+            return False
+ 
+        response = metadata
+        if getattr(settings, 'SHRINK_NEW_MEDIA_METADATA', False):
+            response = filter_response(metadata, True)
+
+        field = self.get_metadata_field('formats')
+        data[field] = response.get(field, [])
+        if data.get('availability', 'public') != response.get('availability', 'public'):
+            data['availability'] = response.get('availability', 'public')
+        data['formats_epoch'] = response.get('epoch', formats_seconds)
+
+        from common.utils import json_serial
+        compact_json = json.dumps(data, separators=(',', ':'), default=json_serial)
+        self.metadata = compact_json
+        return True
+
     @property
     def url(self):
         url = self.URLS.get(self.source.source_type, '')

tubesync/sync/signals.py

@@ -94,12 +94,20 @@ def source_post_save(sender, instance, created, **kwargs):
         settings.RENAME_ALL_SOURCES
     )
     if create_rename_tasks:
-        for media in Media.objects.filter(source=instance.pk, downloaded=True):
+        mqs = Media.objects.filter(
+            source=instance.pk,
+            downloaded=True,
+        ).defer(
+            'media_file',
+            'metadata',
+            'thumb',
+        )
+        for media in mqs:
             verbose_name = _('Renaming media for: {}: "{}"')
             rename_media(
                 str(media.pk),
                 queue=str(media.pk),
-                priority=5,
+                priority=16,
                 verbose_name=verbose_name.format(media.key, media.name),
                 remove_existing_tasks=True
             )

tubesync/sync/tasks.py

@@ -553,6 +553,8 @@ def download_media(media_id):
         err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, '
                f'expected outfile does not exist: {filepath}')
         log.error(err)
+        # Try refreshing formats
+        media.refresh_formats
         # Raising an error here triggers the task to be re-attempted (or fail)
         raise DownloadFailedException(err)
 
@@ -587,16 +589,36 @@ def save_all_media_for_source(source_id):
         log.error(f'Task save_all_media_for_source(pk={source_id}) called but no '
                   f'source exists with ID: {source_id}')
         return
+
+    already_saved = set()
+    mqs = Media.objects.filter(source=source)
+    refresh_qs = mqs.filter(
+        can_download=False,
+        skip=False,
+        manual_skip=False,
+        downloaded=False,
+    )
+    for media in refresh_qs:
+        try:
+            media.refresh_formats
+        except YouTubeError as e:
+            log.debug(f'Failed to refresh formats for: {source} / {media.key}: {e!s}')
+            pass
+        else:
+            media.save()
+            already_saved.add(media.uuid)
+
     # Trigger the post_save signal for each media item linked to this source as various
     # flags may need to be recalculated
-    for media in Media.objects.filter(source=source):
+    for media in mqs:
-        media.save()
+        if media.uuid not in already_saved:
+            media.save()
 
 
 @background(schedule=0, remove_existing_tasks=True)
 def rename_media(media_id):
     try:
-        media = Media.objects.get(pk=media_id)
+        media = Media.objects.defer('metadata', 'thumb').get(pk=media_id)
     except Media.DoesNotExist:
         return
     media.rename_files()

tubesync/sync/views.py

@@ -819,13 +819,9 @@ class CompletedTasksView(ListView):
         return super().dispatch(request, *args, **kwargs)
 
     def get_queryset(self):
-        return CompletedTask.objects.all().order_by('-run_at')
+        q = CompletedTask.objects.all()
-
-    def get_queryset(self):
         if self.filter_source:
-            q = CompletedTask.objects.filter(queue=str(self.filter_source.pk))
+            q = q.filter(queue=str(self.filter_source.pk))
-        else:
-            q = CompletedTask.objects.all()
         return q.order_by('-run_at')
 
     def get_context_data(self, *args, **kwargs):

tubesync/sync/youtube.py

@@ -85,6 +85,7 @@ def get_channel_image_info(url):
         'simulate': True,
         'logger': log,
         'extract_flat': True,  # Change to False to get detailed info
+        'check_formats': False,
     })
 
     with yt_dlp.YoutubeDL(opts) as y:
@@ -143,7 +144,11 @@ def get_media_info(url):
         'simulate': True,
         'logger': log,
         'extract_flat': True,
-        'extractor_args': {'youtubetab': {'approximate_date': ['true']}},
+        'check_formats': True,
+        'extractor_args': {
+            'youtube': {'formats': ['missing_pot']},
+            'youtubetab': {'approximate_date': ['true']},
+        },
     })
     response = {}
     with yt_dlp.YoutubeDL(opts) as y:
@@ -248,7 +253,7 @@ def download_media(
         'writeautomaticsub': auto_subtitles,
         'subtitleslangs': sub_langs.split(','),
         'writethumbnail': embed_thumbnail,
-        'check_formats': False,
+        'check_formats': None,
         'overwrites': None,
         'sleep_interval': 10 + int(settings.DOWNLOAD_MEDIA_DELAY / 20),
         'max_sleep_interval': settings.DOWNLOAD_MEDIA_DELAY,

tubesync/tubesync/local_settings.py.container

@@ -57,7 +57,7 @@ BACKGROUND_TASK_ASYNC_THREADS = int(os.getenv('TUBESYNC_WORKERS', DEFAULT_THREAD
 
 MEDIA_ROOT = CONFIG_BASE_DIR / 'media'
 DOWNLOAD_ROOT = DOWNLOADS_BASE_DIR
-YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache/youtube'
+YOUTUBE_DL_CACHEDIR = CONFIG_BASE_DIR / 'cache/yt-dlp'
 YOUTUBE_DL_TEMPDIR = DOWNLOAD_ROOT / 'cache'
 COOKIES_FILE = CONFIG_BASE_DIR / 'cookies.txt'
 
@@ -106,6 +106,7 @@ if not YOUTUBE_DL_CACHEDIR.is_dir():
     YOUTUBE_DL_CACHEDIR.mkdir(parents=True)
 # rename any old yt_dlp cache directories to the current directory
 old_youtube_cache_dirs = list(YOUTUBE_DL_CACHEDIR.parent.glob('youtube-*'))
+old_youtube_cache_dirs.extend(list(YOUTUBE_DL_CACHEDIR.parent.glob('youtube/youtube-*')))
 for cache_dir in old_youtube_cache_dirs:
     cache_dir.rename(YOUTUBE_DL_CACHEDIR / cache_dir.name)
 

tubesync/tubesync/settings.py

@@ -209,6 +209,9 @@ if MAX_RUN_TIME < 600:
 
 DOWNLOAD_MEDIA_DELAY = 60 + (MAX_RUN_TIME / 20)
 
+if RENAME_SOURCES or RENAME_ALL_SOURCES:
+    BACKGROUND_TASK_ASYNC_THREADS += 1
+
 if BACKGROUND_TASK_ASYNC_THREADS > MAX_BACKGROUND_TASK_ASYNC_THREADS:
     BACKGROUND_TASK_ASYNC_THREADS = MAX_BACKGROUND_TASK_ASYNC_THREADS