diff --git a/app/common/errors.py b/app/common/errors.py index 10fe8c1b..9f3eda4e 100644 --- a/app/common/errors.py +++ b/app/common/errors.py @@ -4,3 +4,19 @@ class NoMediaException(Exception): playlist name or similar, or the upstream source returned an error. ''' pass + + +class NoFormatException(Exception): + ''' + Raised when a media item is attempted to be downloaded but it has no valid + format combination. + ''' + pass + + +class DownloadFailedException(Exception): + ''' + Raised when a downloaded media file is expected to be present, but doesn't + exist. + ''' + pass diff --git a/app/common/static/styles/_template.scss b/app/common/static/styles/_template.scss index 4c86f31f..06ec9002 100644 --- a/app/common/static/styles/_template.scss +++ b/app/common/static/styles/_template.scss @@ -77,7 +77,7 @@ main { h2 { margin: 0; - padding: 2rem 0 0.5rem 0; + padding: 1.5rem 0 1rem 0; font-size: 1.5rem; } diff --git a/app/common/templates/infobox.html b/app/common/templates/infobox.html index 269dbd1d..4fb3b43a 100644 --- a/app/common/templates/infobox.html +++ b/app/common/templates/infobox.html @@ -3,7 +3,7 @@
- {{ message|safe }} + {{ message|safe }}
diff --git a/app/sync/migrations/0019_auto_20201209_0857.py b/app/sync/migrations/0019_auto_20201209_0857.py new file mode 100644 index 00000000..bbfae9ae --- /dev/null +++ b/app/sync/migrations/0019_auto_20201209_0857.py @@ -0,0 +1,24 @@ +# Generated by Django 3.1.4 on 2020-12-09 08:57 + +from django.db import migrations, models +import sync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0018_media_can_download'), + ] + + operations = [ + migrations.AddField( + model_name='media', + name='media_file', + field=models.FileField(blank=True, help_text='Media file', max_length=200, null=True, upload_to=sync.models.get_media_file_path, verbose_name='media file'), + ), + migrations.AlterField( + model_name='media', + name='thumb', + field=models.ImageField(blank=True, height_field='thumb_height', help_text='Thumbnail', max_length=200, null=True, upload_to=sync.models.get_media_thumb_path, verbose_name='thumb', width_field='thumb_width'), + ), + ] diff --git a/app/sync/models.py b/app/sync/models.py index ecfd103c..ab08184e 100644 --- a/app/sync/models.py +++ b/app/sync/models.py @@ -1,3 +1,4 @@ +import os import uuid import json from datetime import datetime @@ -6,7 +7,9 @@ from django.conf import settings from django.db import models from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ -from .youtube import get_media_info as get_youtube_media_info +from common.errors import NoFormatException +from .youtube import (get_media_info as get_youtube_media_info, + download_media as download_youtube_media) from .utils import seconds_to_timestr, parse_media_format from .matching import (get_best_combined_format, get_best_audio_format, get_best_video_format) @@ -297,6 +300,13 @@ class Source(models.Model): else: return settings.SYNC_VIDEO_ROOT / self.directory + def make_directory(self): + return os.makedirs(self.directory_path, exist_ok=True) + + def directory_exists(self): + return (os.path.isdir(self.directory_path) and + os.access(self.directory_path, os.W_OK)) + @property def key_field(self): return self.KEY_FIELD.get(self.source_type, '') @@ -344,6 +354,10 @@ def get_media_thumb_path(instance, filename): return Path('thumbs') / prefix / filename +def get_media_file_path(instance, filename): + return instance.filepath + + class Media(models.Model): ''' Media is a single piece of media, such as a single YouTube video linked to a @@ -414,7 +428,7 @@ class Media(models.Model): thumb = models.ImageField( _('thumb'), upload_to=get_media_thumb_path, - max_length=100, + max_length=200, blank=True, null=True, width_field='thumb_width', @@ -445,6 +459,14 @@ class Media(models.Model): default=False, help_text=_('Media has a matching format and can be downloaded') ) + media_file = models.FileField( + _('media file'), + upload_to=get_media_file_path, + max_length=200, + blank=True, + null=True, + help_text=_('Media file') + ) downloaded = models.BooleanField( _('downloaded'), db_index=True, @@ -501,6 +523,9 @@ class Media(models.Model): class Meta: verbose_name = _('Media') verbose_name_plural = _('Media') + unique_together = ( + ('source', 'key'), + ) def get_metadata_field(self, field): fields = self.METADATA_FIELDS.get(field, {}) @@ -539,17 +564,25 @@ class Media(models.Model): audio_match, audio_format = self.get_best_audio_format() video_match, video_format = self.get_best_video_format() if audio_format and video_format: - return f'{audio_format}+{video_format}' + return f'{video_format}+{audio_format}' else: return False return False + + def get_format_by_code(self, format_code): + ''' + Matches a format code, such as '22', to a processed format dict. + ''' + for fmt in self.iter_formats(): + if format_code == fmt['id']: + return fmt + return False @property def loaded_metadata(self): try: return json.loads(self.metadata) except Exception as e: - print('!!!!', e) return {} @property @@ -557,6 +590,11 @@ class Media(models.Model): url = self.URLS.get(self.source.source_type, '') return url.format(key=self.key) + @property + def description(self): + field = self.get_metadata_field('description') + return self.loaded_metadata.get(field, '').strip() + @property def title(self): field = self.get_metadata_field('title') @@ -599,11 +637,31 @@ class Media(models.Model): dateobj = upload_date if upload_date else self.created datestr = dateobj.strftime('%Y-%m-%d') source_name = slugify(self.source.name) - name = slugify(self.name.replace('&', 'and').replace('+', 'and')) + name = slugify(self.name.replace('&', 'and').replace('+', 'and'))[:50] + key = self.key.strip() + fmt = self.source.source_resolution.lower() + codecs = [] + vcodec = self.source.source_vcodec.lower() + acodec = self.source.source_acodec.lower() + if vcodec: + codecs.append(vcodec) + if acodec: + codecs.append(acodec) + codecs = '-'.join(codecs) ext = self.source.extension - fn = f'{datestr}_{source_name}_{name}'[:100] - return f'{fn}.{ext}' + return f'{datestr}_{source_name}_{name}_{key}-{fmt}-{codecs}.{ext}' @property def filepath(self): return self.source.directory_path / self.filename + + def download_media(self): + format_str = self.get_format_str() + if not format_str: + raise NoFormatException(f'Cannot download, media "{self.pk}" ({media}) has ' + f'no valid format available') + # Download the media with youtube-dl + download_youtube_media(self.url, format_str, self.source.extension, + str(self.filepath)) + # Return the download paramaters + return format_str, self.source.extension diff --git a/app/sync/signals.py b/app/sync/signals.py index 37957f34..3ee59342 100644 --- a/app/sync/signals.py +++ b/app/sync/signals.py @@ -6,8 +6,9 @@ from background_task.signals import task_failed from background_task.models import Task from common.logger import log from .models import Source, Media -from .tasks import (delete_task, index_source_task, download_media_thumbnail, - map_task_to_instance) +from .tasks import (delete_task_by_source, delete_task_by_media, index_source_task, + download_media_thumbnail, map_task_to_instance, + check_source_directory_exists, download_media) from .utils import delete_file @@ -22,7 +23,7 @@ def source_pre_save(sender, instance, **kwargs): return if existing_source.index_schedule != instance.index_schedule: # Indexing schedule has changed, recreate the indexing task - delete_task('sync.tasks.index_source_task', instance.pk) + delete_task_by_source('sync.tasks.index_source_task', instance.pk) verbose_name = _('Index media from source "{}"') index_source_task( str(instance.pk), @@ -34,10 +35,11 @@ def source_pre_save(sender, instance, **kwargs): @receiver(post_save, sender=Source) def source_post_save(sender, instance, created, **kwargs): - # Triggered after a source is saved + # Triggered after a source is saved, Create a new task to check the directory exists + check_source_directory_exists(str(instance.pk)) if created: # Create a new indexing task for newly created sources - delete_task('sync.tasks.index_source_task', instance.pk) + delete_task_by_source('sync.tasks.index_source_task', instance.pk) log.info(f'Scheduling media indexing for source: {instance.name}') verbose_name = _('Index media from source "{}"') index_source_task( @@ -65,7 +67,7 @@ def source_pre_delete(sender, instance, **kwargs): def source_post_delete(sender, instance, **kwargs): # Triggered after a source is deleted log.info(f'Deleting tasks for source: {instance.name}') - delete_task('sync.tasks.index_source_task', instance.pk) + delete_task_by_source('sync.tasks.index_source_task', instance.pk) @receiver(task_failed, sender=Task) @@ -88,7 +90,7 @@ def media_post_save(sender, instance, created, **kwargs): if thumbnail_url: log.info(f'Scheduling task to download thumbnail for: {instance.name} ' f'from: {thumbnail_url}') - verbose_name = _('Downloading media thumbnail for "{}"') + verbose_name = _('Downloading thumbnail for "{}"') download_media_thumbnail( str(instance.pk), thumbnail_url, @@ -105,13 +107,22 @@ def media_post_save(sender, instance, created, **kwargs): if instance.can_download: instance.can_download = True instance.save() + # If the media has not yet been downloaded schedule it to be downloaded + if not instance.downloaded: + delete_task_by_media('sync.tasks.download_media', instance.pk) + verbose_name = _('Downloading media for "{}"') + download_media( + str(instance.pk), + queue=str(instance.source.pk), + verbose_name=verbose_name.format(instance.name) + ) @receiver(pre_delete, sender=Media) def media_pre_delete(sender, instance, **kwargs): # Triggered before media is deleted, delete any scheduled tasks log.info(f'Deleting tasks for media: {instance.name}') - delete_task('sync.tasks.download_media_thumbnail', instance.source.pk) + delete_task_by_media('sync.tasks.download_media_thumbnail', instance.pk) # Delete media thumbnail if it exists if instance.thumb: log.info(f'Deleting thumbnail for: {instance} path: {instance.thumb.path}') diff --git a/app/sync/tasks.py b/app/sync/tasks.py index c0c83e5a..52cdcbc7 100644 --- a/app/sync/tasks.py +++ b/app/sync/tasks.py @@ -4,6 +4,7 @@ ''' +import os import json import math import uuid @@ -18,7 +19,7 @@ from django.db.utils import IntegrityError from background_task import background from background_task.models import Task, CompletedTask from common.logger import log -from common.errors import NoMediaException +from common.errors import NoMediaException, DownloadFailedException from .models import Source, Media from .utils import get_remote_image, resize_image_to_height @@ -39,23 +40,14 @@ def map_task_to_instance(task): ''' TASK_MAP = { 'sync.tasks.index_source_task': Source, + 'sync.tasks.check_source_directory_exists': Source, 'sync.tasks.download_media_thumbnail': Media, + 'sync.tasks.download_media': Media, } MODEL_URL_MAP = { Source: 'sync:source', Media: 'sync:media-item', } - # If the task has a UUID set in its .queue it's probably a link to a Source - if task.queue: - try: - queue_uuid = uuid.UUID(task.queue) - try: - url = MODEL_URL_MAP.get(Source, None) - return Source.objects.get(pk=task.queue), url - except Source.DoesNotExist: - pass - except (TypeError, ValueError, AttributeError): - pass # Unpack task_func, task_args_str = task.task_name, task.task_params model = TASK_MAP.get(task_func, None) @@ -82,6 +74,7 @@ def map_task_to_instance(task): instance = model.objects.get(pk=instance_uuid) return instance, url except model.DoesNotExist: + print('!!!', model, instance_uuid) return None, None @@ -111,10 +104,14 @@ def get_source_completed_tasks(source_id, only_errors=False): return CompletedTask.objects.filter(**q).order_by('-failed_at') -def delete_task(task_name, source_id): +def delete_task_by_source(task_name, source_id): return Task.objects.filter(task_name=task_name, queue=str(source_id)).delete() +def delete_task_by_media(task_name, media_id): + return Task.objects.drop_task(task_name, args=(str(media_id),)) + + def cleanup_completed_tasks(): days_to_keep = getattr(settings, 'COMPLETED_TASKS_DAYS_TO_KEEP', 30) delta = timezone.now() - timedelta(days=days_to_keep) @@ -172,16 +169,37 @@ def index_source_task(source_id): cleanup_completed_tasks() + +@background(schedule=0) +def check_source_directory_exists(source_id): + ''' + Checks the output directory for a source exists and is writable, if it does + not attempt to create it. This is a task so if there are permission errors + they are logged as failed tasks. + ''' + try: + source = Source.objects.get(pk=source_id) + except Source.DoesNotExist: + # Task triggered but the Source has been deleted, delete the task + delete_index_source_task(source_id) + return + # Check the source output directory exists + if not source.directory_exists(): + # Try and create it + log.info(f'Creating directory: {source.directory_path}') + source.make_directory() + + @background(schedule=0) def download_media_thumbnail(media_id, url): ''' Downloads an image from a URL and save it as a local thumbnail attached to a - Media object. + Media instance. ''' try: media = Media.objects.get(pk=media_id) except Media.DoesNotExist: - # Task triggered but the media no longer exists, ignore task + # Task triggered but the media no longer exists, do nothing return width = getattr(settings, 'MEDIA_THUMBNAIL_WIDTH', 430) height = getattr(settings, 'MEDIA_THUMBNAIL_HEIGHT', 240) @@ -203,3 +221,51 @@ def download_media_thumbnail(media_id, url): ) log.info(f'Saved thumbnail for: {media} from: {url}') return True + + +@background(schedule=0) +def download_media(media_id): + ''' + Downloads the media to disk and attaches it to the Media instance. + ''' + try: + media = Media.objects.get(pk=media_id) + except Media.DoesNotExist: + # Task triggered but the media no longer exists, do nothing + return + log.info(f'Downloading media: {media} (UUID: {media.pk}) to: {media.filepath}') + format_str, container = media.download_media() + if os.path.exists(media.filepath): + # Media has been downloaded successfully + log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' + f'{media.filepath}') + # Link the media file to the object and update info about the download + media.media_file.name = str(media.filepath) + media.downloaded = True + if '+' in format_str: + vformat_code, aformat_code = format_str.split('+') + aformat = media.get_format_by_code(aformat_code) + vformat = media.get_format_by_code(vformat_code) + media.downloaded_audio_codec = aformat['acodec'] + media.downloaded_video_codec = vformat['vcodec'] + media.downloaded_container = container + media.downloaded_fps = vformat['fps'] + media.downloaded_hdr = vformat['is_hdr'] + media.downloaded_filesize = os.path.getsize(media.filepath) + else: + cformat_code = format_str + cformat = media.get_format_by_code(cformat_code) + media.downloaded_audio_codec = cformat['acodec'] + media.downloaded_video_codec = cformat['vcodec'] + media.downloaded_container = container + media.downloaded_fps = cformat['fps'] + media.downloaded_hdr = cformat['is_hdr'] + media.downloaded_filesize = os.path.getsize(media.filepath) + media.save() + else: + # Expected file doesn't exist on disk + err = (f'Failed to download media: {media} (UUID: {media.pk}) to disk, ' + f'expected outfile does not exist: {media.filepath}') + log.error(err) + # Raising an error here triggers the task to be re-attempted (or fail) + raise DownloadFailedException(err) diff --git a/app/sync/templates/sync/media-item.html b/app/sync/templates/sync/media-item.html index 39588cde..e78f6570 100644 --- a/app/sync/templates/sync/media-item.html +++ b/app/sync/templates/sync/media-item.html @@ -1,4 +1,4 @@ -{% extends 'base.html' %} +{% extends 'base.html' %}{% load static %} {% block headtitle %}Media - {{ media.key }}{% endblock %} @@ -6,11 +6,26 @@

Media {{ media.key }}

+ {% if media.title %}

{{ media.title }}

{% endif %}

{{ media.url }}

-

Saving to: {{ media.source.directory_path }}

+

Downloading to: {{ media.source.directory_path }}

{% if not media.can_download %}{% include 'errorbox.html' with message='Media cannot be downloaded because it has no formats which match the source requirements.' %}{% endif %} +
+
+
+

{% if media.description %}{{ media.description|truncatewords:200 }}{% else %}(Media has no description).{% endif %}

+
+
+
+
+
+ +
+
+
+
@@ -18,18 +33,10 @@ - - - - - - - - @@ -38,19 +45,51 @@ - - - + + + - - - + {% if media.downloaded %} + + + + + + + + + + + + + + + + + + + + + + + + {% else %} + + + + + {% endif %} - - - -
Source Source
{{ media.source }}
TitleTitle
{{ media.title }}
Duration Duration
{{ media.duration_formatted }}
FilenameFilename
{{ media.filename }}
Desired format Desired format
{{ media.source.format_summary }}
Fallback Fallback
{{ media.source.get_fallback_display }}
DownloadedDownloaded
{% if media.downloaded %}{% else %}{% endif %}
Downloaded?Downloaded?
{% if media.downloaded %}{% else %}{% endif %}
Can downloadCan download
{% if youtube_dl_format %}{% else %}{% endif %}
FilenameFilename
{{ media.filename }}
File sizeFile size
{{ media.downloaded_filesize|filesizeformat }}
Downloaded codecsDownloaded codecs
audio:{{ media.downloaded_audio_codec }}{% if media.downloaded_video_codec %}, video:{{ media.downloaded_video_codec }}{% endif %}
ContainerContainer
{{ media.downloaded_container|upper }}
Downloaded FPSDownloaded FPS
{{ media.downloaded_fps }} FPS
Downloaded HDR?Downloaded HDR?
{% if media.downloaded_hdr %}{% else %}{% endif %}
Can download?Can download?
{% if youtube_dl_format %}{% else %}{% endif %}
Available formats Available formats
{% for format in media.formats %} - ID: {{ format.format_id }}{% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %}{% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% endif %} + + ID: {{ format.format_id }} + {% if format.vcodec|lower != 'none' %}, {{ format.format_note }} ({{ format.width }}x{{ format.height }}), fps:{{ format.fps|lower }}, video:{{ format.vcodec }} @{{ format.tbr }}k{% endif %} + {% if format.acodec|lower != 'none' %}, audio:{{ format.acodec }} @{{ format.abr }}k / {{ format.asr }}Hz{% endif %} + {% if format.format_id == combined_format or format.format_id == audio_format or format.format_id == video_format %}(matched){% endif %} + {% empty %} Media has no indexed available formats {% endfor %} @@ -59,15 +98,11 @@
Matched formats Matched formats
- Combined: {% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}
- Audio: {% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %}
- Video: {% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}No match{% endif %} + Combined: {% if combined_format %}{{ combined_format }} {% if combined_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
+ Audio: {% if audio_format %}{{ audio_format }} {% if audio_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
+ Video: {% if video_format %}{{ video_format }} {% if video_exact %}(exact match){% else %}(fallback){% endif %}{% else %}no match{% endif %}
youtube-dl formatyoutube-dl format
{% if youtube_dl_format %}{{ youtube_dl_format }}{% else %}No matching formats{% endif %}
diff --git a/app/sync/templates/sync/media.html b/app/sync/templates/sync/media.html index de8fef42..7f116e04 100644 --- a/app/sync/templates/sync/media.html +++ b/app/sync/templates/sync/media.html @@ -18,7 +18,7 @@ {{ m.source }}
{{ m.name }}
- {% if m.can_download %}{{ m.published|date:'Y-m-d' }}{% else %} No matching formats{% endif %} + {% if m.can_download %}{% if m.downloaded %}{% else %}{% endif %} {{ m.published|date:'Y-m-d' }}{% else %} No matching formats{% endif %}
diff --git a/app/sync/utils.py b/app/sync/utils.py index 25936f7b..c3f82a29 100644 --- a/app/sync/utils.py +++ b/app/sync/utils.py @@ -164,6 +164,7 @@ def parse_media_format(format_dict): 'format_verbose': format_dict.get('format', ''), 'height': format_dict.get('height', 0), 'vcodec': vcodec, + 'fps': format_dict.get('fps', 0), 'vbr': format_dict.get('tbr', 0), 'acodec': acodec, 'abr': format_dict.get('abr', 0), diff --git a/app/sync/youtube.py b/app/sync/youtube.py index 7857a1b1..cf416509 100644 --- a/app/sync/youtube.py +++ b/app/sync/youtube.py @@ -40,3 +40,23 @@ def get_media_info(url): except youtube_dl.utils.DownloadError as e: raise YouTubeError(f'Failed to extract_info for "{url}": {e}') from e return response + + +def download_media(url, media_format, extension, output_file): + ''' + Downloads a YouTube URL to a file on disk. + ''' + opts = copy(_defaults) + opts.update({ + 'format': media_format, + 'merge_output_format': extension, + 'outtmpl': output_file, + 'quiet': True, + }) + print(opts) + with youtube_dl.YoutubeDL(opts) as y: + try: + return y.download([url]) + except youtube_dl.utils.DownloadError as e: + raise YouTubeError(f'Failed to download for "{url}": {e}') from e + return False