diff --git a/.gitignore b/.gitignore index cb14cfd6..cf63cfba 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ dmypy.json Pipfile.lock .vscode/launch.json + +# Ignore Jetbrains IDE files +.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ce4b358b..4941821b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,15 +3,15 @@ FROM debian:bookworm-slim ARG TARGETARCH ARG TARGETPLATFORM -ARG S6_VERSION="3.2.0.0" -ARG SHA256_S6_AMD64="ad982a801bd72757c7b1b53539a146cf715e640b4d8f0a6a671a3d1b560fe1e2" -ARG SHA256_S6_ARM64="868973e98210257bba725ff5b17aa092008c9a8e5174499e38ba611a8fc7e473" -ARG SHA256_S6_NOARCH="4b0c0907e6762814c31850e0e6c6762c385571d4656eb8725852b0b1586713b6" +ARG S6_VERSION="3.2.0.2" +ARG SHA256_S6_AMD64="59289456ab1761e277bd456a95e737c06b03ede99158beb24f12b165a904f478" +ARG SHA256_S6_ARM64="8b22a2eaca4bf0b27a43d36e65c89d2701738f628d1abd0cea5569619f66f785" +ARG SHA256_S6_NOARCH="6dbcde158a3e78b9bb141d7bcb5ccb421e563523babbe2c64470e76f4fd02dae" -ARG FFMPEG_DATE="autobuild-2024-11-22-14-18" -ARG FFMPEG_VERSION="117857-g2d077f9acd" -ARG SHA256_FFMPEG_AMD64="427ff38cf1e28521aac4fa7931444f6f2ff6f097f2d4a315c6f92ef1d7f90db8" -ARG SHA256_FFMPEG_ARM64="f7ed3a50b651447477aa2637bf8da6010d8f27f8804a5d0e77b00a1d3725b27a" +ARG FFMPEG_DATE="autobuild-2024-12-09-14-16" +ARG FFMPEG_VERSION="118034-gd21134313f" +ARG SHA256_FFMPEG_AMD64="cd50122fb0939e913585282347a8f95074c2d5477ceb059cd90aca551f14e9ea" +ARG SHA256_FFMPEG_ARM64="33b4edebf9c23701473ba8db696b26072bb9b9c05fc4a156e115f94e44d361e0" ENV S6_VERSION="${S6_VERSION}" \ FFMPEG_DATE="${FFMPEG_DATE}" \ diff --git a/README.md b/README.md index b85b2b4a..a01f9830 100644 --- a/README.md +++ b/README.md @@ -361,21 +361,22 @@ There are a number of other environment variables you can set. These are, mostly **NOT** required to be set in the default container installation, they are really only useful if you are manually installing TubeSync in some other environment. These are: -| Name | What | Example | -| --------------------------- | ------------------------------------------------------------ | ------------------------------------ | -| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | -| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | -| TUBESYNC_DEBUG | Enable debugging | True | -| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | -| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | -| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | -| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | -| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | -| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | -| HTTP_USER | Sets the username for HTTP basic authentication | some-username | -| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | -| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | -| VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| Name | What | Example | +| ---------------------------- | ------------------------------------------------------------- |--------------------------------------| +| DJANGO_SECRET_KEY | Django's SECRET_KEY | YJySXnQLB7UVZw2dXKDWxI5lEZaImK6l | +| DJANGO_URL_PREFIX | Run TubeSync in a sub-URL on the web server | /somepath/ | +| TUBESYNC_DEBUG | Enable debugging | True | +| TUBESYNC_WORKERS | Number of background workers, default is 2, max allowed is 8 | 2 | +| TUBESYNC_HOSTS | Django's ALLOWED_HOSTS, defaults to `*` | tubesync.example.com,otherhost.com | +| TUBESYNC_RESET_DOWNLOAD_DIR | Toggle resetting `/downloads` permissions, defaults to True | True | +| TUBESYNC_VIDEO_HEIGHT_CUTOFF | Smallest video height in pixels permitted to download | 240 | +| TUBESYNC_DIRECTORY_PREFIX | Enable `video` and `audio` directory prefixes in `/downloads` | True | +| GUNICORN_WORKERS | Number of gunicorn workers to spawn | 3 | +| LISTEN_HOST | IP address for gunicorn to listen on | 127.0.0.1 | +| LISTEN_PORT | Port number for gunicorn to listen on | 8080 | +| HTTP_USER | Sets the username for HTTP basic authentication | some-username | +| HTTP_PASS | Sets the password for HTTP basic authentication | some-secure-password | +| DATABASE_CONNECTION | Optional external database connection details | mysql://user:pass@host:port/database | # Manual, non-containerised, installation diff --git a/tubesync/sync/filtering.py b/tubesync/sync/filtering.py index 351b3f83..45710fe9 100644 --- a/tubesync/sync/filtering.py +++ b/tubesync/sync/filtering.py @@ -4,7 +4,7 @@ from common.logger import log from .models import Media -from datetime import datetime, timedelta +from datetime import datetime from django.utils import timezone from .overrides.custom_filter import filter_custom @@ -127,19 +127,15 @@ def filter_max_cap(instance: Media): return False -# If the source has a cut-off, check the upload date is within the allowed delta +# If the source has a cut-off, check the download date is within the allowed delta def filter_source_cutoff(instance: Media): - if instance.source.delete_old_media and instance.source.days_to_keep > 0: - if not isinstance(instance.published, datetime): - # Media has no known published date or incomplete metadata - log.info( - f"Media: {instance.source} / {instance} has no published date, skipping" - ) - return True + if instance.source.delete_old_media and instance.source.days_to_keep_date: + if not instance.downloaded or not isinstance(instance.download_date, datetime): + return False - delta = timezone.now() - timedelta(days=instance.source.days_to_keep) - if instance.published < delta: - # Media was published after the cutoff date, skip it + days_to_keep_age = instance.source.days_to_keep_date + if instance.download_date < days_to_keep_age: + # Media has expired, skip it log.info( f"Media: {instance.source} / {instance} is older than " f"{instance.source.days_to_keep} days, skipping" diff --git a/tubesync/sync/matching.py b/tubesync/sync/matching.py index c453ff96..5993a4d1 100644 --- a/tubesync/sync/matching.py +++ b/tubesync/sync/matching.py @@ -5,6 +5,7 @@ ''' +from .utils import multi_key_sort from django.conf import settings @@ -21,7 +22,7 @@ def get_best_combined_format(media): ''' for fmt in media.iter_formats(): # Check height matches - if media.source.source_resolution.strip().upper() != fmt['format']: + if media.source.source_resolution_height != fmt['height']: continue # Check the video codec matches if media.source.source_vcodec != fmt['vcodec']: @@ -47,7 +48,7 @@ def get_best_audio_format(media): Finds the best match for the source required audio format. If the source has a 'fallback' of fail this can return no match. ''' - # Order all audio-only formats by bitrate + # Reverse order all audio-only formats audio_formats = [] for fmt in media.iter_formats(): # If the format has a video stream, skip it @@ -56,18 +57,18 @@ def get_best_audio_format(media): if not fmt['acodec']: continue audio_formats.append(fmt) - audio_formats = list(reversed(sorted(audio_formats, key=lambda k: k['abr']))) if not audio_formats: # Media has no audio formats at all return False, False - # Find the highest bitrate audio format with a matching codec + audio_formats = list(reversed(audio_formats)) + # Find the first audio format with a matching codec for fmt in audio_formats: if media.source.source_acodec == fmt['acodec']: # Matched! return True, fmt['id'] # No codecs matched if media.source.can_fallback: - # Can fallback, find the next highest bitrate non-matching codec + # Can fallback, find the next non-matching codec return False, audio_formats[0]['id'] else: # Can't fallback @@ -86,6 +87,7 @@ def get_best_video_format(media): return False, False # Filter video-only formats by resolution that matches the source video_formats = [] + sort_keys = [('height', False), ('vcodec', True), ('vbr', False)] # key, reverse for fmt in media.iter_formats(): # If the format has an audio stream, skip it if fmt['acodec'] is not None: @@ -94,6 +96,8 @@ def get_best_video_format(media): continue if media.source.source_resolution.strip().upper() == fmt['format']: video_formats.append(fmt) + elif media.source.source_resolution_height == fmt['height']: + video_formats.append(fmt) # Check we matched some streams if not video_formats: # No streams match the requested resolution, see if we can fallback @@ -109,13 +113,17 @@ def get_best_video_format(media): else: # Can't fallback return False, False - video_formats = list(reversed(sorted(video_formats, key=lambda k: k['height']))) - source_resolution = media.source.source_resolution.strip().upper() - source_vcodec = media.source.source_vcodec if not video_formats: # Still no matches return False, False + video_formats = multi_key_sort(video_formats, sort_keys, True) + source_resolution = media.source.source_resolution.strip().upper() + source_vcodec = media.source.source_vcodec exact_match, best_match = None, None + for fmt in video_formats: + # format_note was blank, match height instead + if '' == fmt['format'] and fmt['height'] == media.source.source_resolution_height: + fmt['format'] = source_resolution # Of our filtered video formats, check for resolution + codec + hdr + fps match if media.source.prefer_60fps and media.source.prefer_hdr: for fmt in video_formats: @@ -331,7 +339,7 @@ def get_best_video_format(media): for fmt in video_formats: # Check for a codec, hdr and fps match but drop the resolution if (source_vcodec == fmt['vcodec'] and - not fmt['is_hdr'] and fmt['is_60fps']): + not fmt['is_hdr'] and not fmt['is_60fps']): # Close match exact_match, best_match = False, fmt break diff --git a/tubesync/sync/migrations/0026_alter_source_sub_langs.py b/tubesync/sync/migrations/0026_alter_source_sub_langs.py new file mode 100644 index 00000000..937c3a14 --- /dev/null +++ b/tubesync/sync/migrations/0026_alter_source_sub_langs.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-12-11 12:43 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0025_add_video_type_support'), + ] + + operations = [ + migrations.AlterField( + model_name='source', + name='sub_langs', + field=models.CharField(default='en', help_text='List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat', max_length=30, validators=[django.core.validators.RegexValidator(message='Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat', regex='^(\\-?[\\_\\.a-zA-Z-]+(,|$))+')], verbose_name='subs langs'), + ), + ] diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index 2f116356..455a38a5 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -422,7 +422,7 @@ class Source(models.Model): help_text=_('List of subtitles langs to download, comma-separated. Example: en,fr or all,-fr,-live_chat'), validators=[ RegexValidator( - regex=r"^(\-?[\_\.a-zA-Z]+,)*(\-?[\_\.a-zA-Z]+){1}$", + regex=r"^(\-?[\_\.a-zA-Z-]+(,|$))+", message=_('Subtitle langs must be a comma-separated list of langs. example: en,fr or all,-fr,-live_chat') ) ] @@ -460,6 +460,14 @@ class Source(models.Model): else: return False + @property + def days_to_keep_date(self): + delta = self.days_to_keep + if delta > 0: + return timezone.now() - timedelta(days=delta) + else: + return False + @property def extension(self): ''' @@ -514,10 +522,13 @@ class Source(models.Model): @property def type_directory_path(self): - if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: - return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + if settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX: + if self.source_resolution == self.SOURCE_RESOLUTION_AUDIO: + return Path(settings.DOWNLOAD_AUDIO_DIR) / self.directory + else: + return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory else: - return Path(settings.DOWNLOAD_VIDEO_DIR) / self.directory + return Path(self.directory) def make_directory(self): return os.makedirs(self.directory_path, exist_ok=True) @@ -1291,10 +1302,7 @@ class Media(models.Model): @property def directory_path(self): - # Otherwise, create a suitable filename from the source media_format - media_format = str(self.source.media_format) - media_details = self.format_dict - dirname = self.source.directory_path / media_format.format(**media_details) + dirname = self.source.directory_path / self.filename return os.path.dirname(str(dirname)) @property diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index 72411894..6aa0ccb6 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -6,7 +6,9 @@ import logging +import os from datetime import datetime, timedelta +from pathlib import Path from urllib.parse import urlsplit from xml.etree import ElementTree from django.conf import settings @@ -626,6 +628,25 @@ class FilepathTestCase(TestCase): ('no-fancy-stuff-title_test_720p-720x1280-opus' '-vp9-30fps-hdr.mkv')) + def test_directory_prefix(self): + # Confirm the setting exists and is valid + self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX')) + self.assertTrue(isinstance(settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX, bool)) + # Test the default behavior for "True", forced "audio" or "video" parent directories for sources + settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True + self.source.source_resolution = Source.SOURCE_RESOLUTION_AUDIO + test_audio_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_audio_prefix_path.parts[-2], 'audio') + self.assertEqual(test_audio_prefix_path.parts[-1], 'testdirectory') + self.source.source_resolution = Source.SOURCE_RESOLUTION_1080P + test_video_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_video_prefix_path.parts[-2], 'video') + self.assertEqual(test_video_prefix_path.parts[-1], 'testdirectory') + # Test the default behavior for "False", no parent directories for sources + settings.SOURCE_DOWNLOAD_DIRECTORY_PREFIX = False + test_no_prefix_path = Path(self.source.directory_path) + self.assertEqual(test_no_prefix_path.parts[-1], 'testdirectory') + class MediaTestCase(TestCase): @@ -1659,7 +1680,7 @@ class FormatMatchingTestCase(TestCase): self.media.get_best_audio_format() def test_is_regex_match(self): - + self.media.metadata = all_test_metadata['boring'] self.media.save() expected_matches = { @@ -1687,7 +1708,9 @@ class FormatMatchingTestCase(TestCase): msg=f'Media title "{self.media.title}" checked against regex "{self.source.filter_text}" failed ' f'expected {expected_match_result}') + class TasksTestCase(TestCase): + def setUp(self): # Disable general logging for test case logging.disable(logging.CRITICAL) diff --git a/tubesync/sync/utils.py b/tubesync/sync/utils.py index cf72462e..1c00b4a2 100644 --- a/tubesync/sync/utils.py +++ b/tubesync/sync/utils.py @@ -1,6 +1,7 @@ import os import re import math +from operator import itemgetter from pathlib import Path import requests from PIL import Image @@ -134,6 +135,30 @@ def seconds_to_timestr(seconds): return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) +def multi_key_sort(sort_dict, specs, use_reversed=False): + result = list(sort_dict) + for key, reverse in reversed(specs): + result = sorted(result, key=itemgetter(key), reverse=reverse) + if use_reversed: + return list(reversed(result)) + return result + + +def normalize_codec(codec_str): + result = str(codec_str).upper() + parts = result.split('.') + if len(parts) > 0: + result = parts[0].strip() + else: + return None + if 'NONE' == result: + return None + if str(0) in result: + prefix = result.rstrip('0123456789') + result = prefix + str(int(result[len(prefix):])) + return result + + def parse_media_format(format_dict): ''' This parser primarily adapts the format dict returned by youtube-dl into a @@ -141,21 +166,9 @@ def parse_media_format(format_dict): any internals, update it here. ''' vcodec_full = format_dict.get('vcodec', '') - vcodec_parts = vcodec_full.split('.') - if len(vcodec_parts) > 0: - vcodec = vcodec_parts[0].strip().upper() - else: - vcodec = None - if vcodec == 'NONE': - vcodec = None + vcodec = normalize_codec(vcodec_full) acodec_full = format_dict.get('acodec', '') - acodec_parts = acodec_full.split('.') - if len(acodec_parts) > 0: - acodec = acodec_parts[0].strip().upper() - else: - acodec = None - if acodec == 'NONE': - acodec = None + acodec = normalize_codec(acodec_full) try: fps = int(format_dict.get('fps', 0)) except (ValueError, TypeError): diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 18b028b3..a0426a4c 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -81,3 +81,10 @@ if BASICAUTH_USERNAME and BASICAUTH_PASSWORD: else: BASICAUTH_DISABLE = True BASICAUTH_USERS = {} + + +SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR = os.getenv('TUBESYNC_DIRECTORY_PREFIX', 'True').strip().lower() +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True if SOURCE_DOWNLOAD_DIRECTORY_PREFIX_STR == 'true' else False + + +VIDEO_HEIGHT_CUTOFF = int(os.getenv("TUBESYNC_VIDEO_HEIGHT_CUTOFF", "240")) diff --git a/tubesync/tubesync/settings.py b/tubesync/tubesync/settings.py index d67d577a..f4865f6a 100644 --- a/tubesync/tubesync/settings.py +++ b/tubesync/tubesync/settings.py @@ -150,8 +150,16 @@ MEDIA_THUMBNAIL_WIDTH = 430 # Width in pixels to resize thumbnai MEDIA_THUMBNAIL_HEIGHT = 240 # Height in pixels to resize thumbnails to -VIDEO_HEIGHT_CUTOFF = int(os.getenv("VIDEO_HEIGHT_CUTOFF", "240")) # Smallest resolution in pixels permitted to download -VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' +VIDEO_HEIGHT_CUTOFF = 240 # Smallest resolution in pixels permitted to download +VIDEO_HEIGHT_IS_HD = 500 # Height in pixels to count as 'HD' + + + +# If True source directories are prefixed with their type (either 'video' or 'audio') +# e.g. /downloads/video/SomeSourceName +# If False, sources are placed directly in /downloads +# e.g. /downloads/SomeSourceName +SOURCE_DOWNLOAD_DIRECTORY_PREFIX = True YOUTUBE_DL_CACHEDIR = None