diff --git a/tubesync/common/templates/mediaformatvars.html b/tubesync/common/templates/mediaformatvars.html
new file mode 100644
index 00000000..bbad0b4c
--- /dev/null
+++ b/tubesync/common/templates/mediaformatvars.html
@@ -0,0 +1,97 @@
+Available media name variables
+
+
+
+ | Name |
+ Description |
+ Output example |
+
+
+
+
+ | {yyyymmdd} |
+ Media publish date in YYYYMMDD |
+ 20210101 |
+
+
+ | {yyyy_mm_dd} |
+ Media publish date in YYYY-MM-DD |
+ 2021-01-01 |
+
+
+ | {yyyy} |
+ Media publish year in YYYY |
+ 2021 |
+
+
+ | {source} |
+ Lower case source name, max 80 chars |
+ my-source |
+
+
+ | {source_full} |
+ Full source name |
+ My Source |
+
+
+ | {title} |
+ Lower case media title, max 80 chars |
+ my-video |
+
+
+ | {title_full} |
+ Full media title |
+ My Video |
+
+
+ | {key} |
+ Media unique key or video ID |
+ SoMeUnIqUeId |
+
+
+ | {format} |
+ Media format string |
+ 720p-avc1-mp4a |
+
+
+ | {ext} |
+ File extension |
+ mkv |
+
+
+ | {resolution} |
+ Resolution |
+ 720p |
+
+
+ | {height} |
+ Media height in pixels |
+ 720 |
+
+
+ | {width} |
+ Media width in pixels |
+ 1280 |
+
+
+ | {vcodec} |
+ Media video codec |
+ avc1 |
+
+
+ | {acodec} |
+ Media audio codec |
+ opus |
+
+
+ | {fps} |
+ Media fps |
+ 60fps |
+
+
+ | {flag_hdr} |
+ Media has HDR flag |
+ hdr |
+
+
+
\ No newline at end of file
diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py
index 8761c23c..aa716a8f 100644
--- a/tubesync/sync/models.py
+++ b/tubesync/sync/models.py
@@ -7,6 +7,7 @@ from django.conf import settings
from django.db import models
from django.core.files.storage import FileSystemStorage
from django.utils.text import slugify
+from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.errors import NoFormatException
from .youtube import (get_media_info as get_youtube_media_info,
@@ -262,6 +263,11 @@ class Source(models.Model):
def icon(self):
return self.ICONS.get(self.source_type)
+ @property
+ def slugname(self):
+ replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and')
+ return slugify(replaced)[:80]
+
@property
def is_audio(self):
return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO
@@ -336,6 +342,49 @@ class Source(models.Model):
def can_fallback(self):
return self.fallback != self.FALLBACK_FAIL
+ @property
+ def example_media_format_dict(self):
+ '''
+ Populates a dict with real-ish and some placeholder data for media name
+ format strings. Used for example filenames and media_format validation.
+ '''
+ fmt = []
+ if self.source_resolution:
+ fmt.append(self.source_resolution)
+ if self.source_vcodec:
+ fmt.append(self.source_vcodec.lower())
+ if self.source_acodec:
+ fmt.append(self.source_acodec.lower())
+ if self.prefer_60fps:
+ fmt.append('60fps')
+ if self.prefer_hdr:
+ fmt.append('hdr')
+ return {
+ 'yyyymmdd': timezone.now().strftime('%Y%m%d'),
+ 'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
+ 'yyyy': timezone.now().strftime('%Y'),
+ 'source': self.slugname,
+ 'source_full': self.name,
+ 'title': 'some-media-title-name',
+ 'title_full': 'Some Media Title Name',
+ 'key': 'SoMeUnIqUiD',
+ 'format': '-'.join(fmt),
+ 'ext': self.extension,
+ 'resolution': self.source_resolution if self.source_resolution else '',
+ 'height': '720' if self.source_resolution else '',
+ 'width': '1280' if self.source_resolution else '',
+ 'vcodec': self.source_vcodec.lower() if self.source_vcodec else '',
+ 'acodec': self.source_acodec.lower(),
+ 'fps': '24' if self.source_resolution else '',
+ 'hdr': 'hdr' if self.source_resolution else ''
+ }
+
+ def get_example_media_format(self):
+ try:
+ return self.media_format.format(**self.example_media_format_dict)
+ except Exception:
+ return ''
+
def index_media(self):
'''
Index the media source returning a list of media metadata as dicts.
@@ -645,16 +694,39 @@ class Media(models.Model):
('720p', 'avc1', 'mp4a', '60fps', 'hdr')
'''
fmt = []
+ resolution = ''
+ vcodec = ''
+ acodec = ''
+ height = '0'
+ width = '0'
+ fps = ''
+ hdr = ''
# If the download has completed use existing values
if self.downloaded:
+ resolution = f'{self.downloaded_height}p'
if self.downloaded_format != 'audio':
- fmt.append(self.downloaded_video_codec.lower())
- fmt.append(self.downloaded_audio_codec.lower())
+ vcodec = self.downloaded_video_codec.lower()
+ fmt.append(vcodec)
+ acodec = self.downloaded_audio_codec.lower()
+ fmt.append(acodec)
if self.downloaded_format != 'audio':
- fmt.append(str(self.downloaded_fps))
+ fps = str(self.downloaded_fps)
+ fmt.append(f'{fps}fps')
if self.downloaded_hdr:
- fmt.append('hdr')
- return fmt
+ hdr = 'hdr'
+ fmt.append(hdr)
+ height = str(self.downloaded_height)
+ width = str(self.downloaded_width)
+ return {
+ 'resolution': resolution,
+ 'height': height,
+ 'width': width,
+ 'vcodec': vcodec,
+ 'acodec': acodec,
+ 'fps': fps,
+ 'hdr': hdr,
+ 'format': tuple(fmt),
+ }
# Otherwise, calculate from matched format codes
vformat = None
aformat = None
@@ -671,15 +743,31 @@ class Media(models.Model):
# Combined
vformat = cformat
if vformat:
- fmt.append(vformat['format'].lower())
- fmt.append(vformat['vcodec'].lower())
- fmt.append(aformat['acodec'].lower())
+ resolution = vformat['format'].lower()
+ fmt.append(resolution)
+ vcodec = vformat['vcodec'].lower()
+ fmt.append(vcodec)
+ acodec = aformat['acodec'].lower()
+ fmt.append(acodec)
if vformat:
if vformat['is_60fps']:
- fmt.append('60fps')
+ fps = '60fps'
+ fmt.append(fps)
if vformat['is_hdr']:
- fmt.append('hdr')
- return tuple(fmt)
+ hdr = 'hdr'
+ fmt.append(hdr)
+ height = str(vformat['height'])
+ width = str(vformat['width'])
+ return {
+ 'resolution': resolution,
+ 'height': height,
+ 'width': width,
+ 'vcodec': vcodec,
+ 'acodec': acodec,
+ 'fps': fps,
+ 'hdr': hdr,
+ 'format': tuple(fmt),
+ }
def get_format_by_code(self, format_code):
'''
@@ -690,6 +778,35 @@ class Media(models.Model):
return fmt
return False
+ @property
+ def format_dict(self):
+ '''
+ Returns a dict matching the media_format key requirements for this item
+ of media.
+ '''
+ format_str = self.get_format_str()
+ display_format = self.get_display_format(format_str)
+ dateobj = self.upload_date if self.upload_date else self.created
+ return {
+ 'yyyymmdd': dateobj.strftime('%Y%m%d'),
+ 'yyyy_mm_dd': dateobj.strftime('%Y-%m-%d'),
+ 'yyyy': dateobj.strftime('%Y'),
+ 'source': self.source.slugname,
+ 'source_full': self.source.name,
+ 'title': self.slugtitle,
+ 'title_full': self.title,
+ 'key': self.key,
+ 'format': '-'.join(display_format['format']),
+ 'ext': self.source.extension,
+ 'resolution': display_format['resolution'],
+ 'height': display_format['height'],
+ 'width': display_format['width'],
+ 'vcodec': display_format['vcodec'],
+ 'acodec': display_format['acodec'],
+ 'fps': display_format['fps'],
+ 'hdr': display_format['hdr'],
+ }
+
@property
def loaded_metadata(self):
try:
@@ -712,6 +829,11 @@ class Media(models.Model):
field = self.get_metadata_field('title')
return self.loaded_metadata.get(field, '').strip()
+ @property
+ def slugtitle(self):
+ replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and')
+ return slugify(replaced)[:80]
+
@property
def thumbnail(self):
field = self.get_metadata_field('thumbnail')
@@ -750,20 +872,24 @@ class Media(models.Model):
@property
def filename(self):
+ # If a media_file has been downloaded use its existing name
if self.media_file:
return os.path.basename(self.media_file.name)
- upload_date = self.upload_date
- dateobj = upload_date if upload_date else self.created
- datestr = dateobj.strftime('%Y-%m-%d')
- source_name = slugify(self.source.name).replace('_', '-')
- name = slugify(self.name.replace('&', 'and').replace('+', 'and'))
- name = name.replace('_', '-')[:80]
- key = self.key.strip().replace('_', '-')[:20]
- format_str = self.get_format_str()
- format_tuple = self.get_display_format(format_str)
- fmt = '-'.join(format_tuple)
- ext = self.source.extension
- return f'{datestr}_{source_name}_{name}_{key}_{fmt}.{ext}'
+ # Otherwise, create a suitable filename from the source media_format
+ media_format = str(self.source.media_format)
+ media_details = self.format_dict
+ return media_format.format(**media_details)
+
+ @property
+ def directory_path(self):
+ # If a media_file has been downloaded use its existing directory
+ if self.media_file:
+ return os.path.dirname(self.media_file.name)
+ # Otherwise, create a suitable filename from the source media_format
+ media_format = str(self.source.media_format)
+ media_details = self.format_dict
+ dirname = self.source.directory_path / media_format.format(**media_details)
+ return os.path.dirname(str(dirname))
@property
def filepath(self):
diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py
index 129d5e5e..bd45a8a6 100644
--- a/tubesync/sync/signals.py
+++ b/tubesync/sync/signals.py
@@ -1,3 +1,4 @@
+import os
from django.conf import settings
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
from django.dispatch import receiver
@@ -152,11 +153,11 @@ def media_pre_delete(sender, instance, **kwargs):
if instance.media_file:
filepath = instance.media_file.path
log.info(f'Deleting media for: {instance} path: {filepath}')
- delete_file(instance.media_file.path)
+ delete_file(filepath)
# Delete thumbnail copy if it exists
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
- log.info(f'Copying thumbnail: {instance} path: {thumbpath}')
+ log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}')
delete_file(thumbpath)
diff --git a/tubesync/sync/templates/sync/media-item.html b/tubesync/sync/templates/sync/media-item.html
index 580755f8..e1e28b27 100644
--- a/tubesync/sync/templates/sync/media-item.html
+++ b/tubesync/sync/templates/sync/media-item.html
@@ -80,6 +80,10 @@
Filename |
Filename
{{ media.filename }} |
+
+ | Directory |
+ Directory
{{ media.directory_path }} |
+
| File size |
File size
{{ media.downloaded_filesize|filesizeformat }} |
diff --git a/tubesync/sync/templates/sync/source-add.html b/tubesync/sync/templates/sync/source-add.html
index f0b37b67..d0e7a7ab 100644
--- a/tubesync/sync/templates/sync/source-add.html
+++ b/tubesync/sync/templates/sync/source-add.html
@@ -23,4 +23,9 @@
+
+
+{% include 'mediaformatvars.html' %}
+
+
+
+{% include 'mediaformatvars.html' %}
+
+
Media format |
Media format
{{ source.media_format }} |
+
+ | Example filename |
+ Example filename
{{ source.get_example_media_format }} |
+
| Index schedule |
Index schedule
{{ source.get_index_schedule_display }} |
diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py
index 68da59b4..d4af2c80 100644
--- a/tubesync/sync/tests.py
+++ b/tubesync/sync/tests.py
@@ -134,6 +134,7 @@ class FrontEndTestCase(TestCase):
'key': 'testkey',
'name': 'testname',
'directory': 'testdirectory',
+ 'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': 3600,
'delete_old_media': False,
'days_to_keep': 14,
@@ -173,6 +174,7 @@ class FrontEndTestCase(TestCase):
'key': 'updatedkey', # changed
'name': 'testname',
'directory': 'testdirectory',
+ 'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': Source.IndexSchedule.EVERY_HOUR,
'delete_old_media': False,
'days_to_keep': 14,
@@ -200,6 +202,7 @@ class FrontEndTestCase(TestCase):
'key': 'updatedkey',
'name': 'testname',
'directory': 'testdirectory',
+ 'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': Source.IndexSchedule.EVERY_2_HOURS, # changed
'delete_old_media': False,
'days_to_keep': 14,
@@ -425,6 +428,137 @@ all_test_metadata = {
}
+class FilepathTestCase(TestCase):
+
+ def setUp(self):
+ # Disable general logging for test case
+ logging.disable(logging.CRITICAL)
+ # Add a test source
+ self.source = Source.objects.create(
+ source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
+ key='testkey',
+ name='testname',
+ directory='testdirectory',
+ media_format=settings.MEDIA_FORMATSTR_DEFAULT,
+ index_schedule=3600,
+ delete_old_media=False,
+ days_to_keep=14,
+ source_resolution=Source.SOURCE_RESOLUTION_1080P,
+ source_vcodec=Source.SOURCE_VCODEC_VP9,
+ source_acodec=Source.SOURCE_ACODEC_OPUS,
+ prefer_60fps=False,
+ prefer_hdr=False,
+ fallback=Source.FALLBACK_FAIL
+ )
+ # Add some test media
+ self.media = Media.objects.create(
+ key='mediakey',
+ source=self.source,
+ metadata=metadata,
+ )
+
+ def test_source_dirname(self):
+ # Check media format validation is working
+ # Empty
+ self.source.media_format = ''
+ self.assertEqual(self.source.get_example_media_format(), '')
+ # Invalid, bad key
+ self.source.media_format = '{test}'
+ self.assertEqual(self.source.get_example_media_format(), '')
+ # Invalid, extra brackets
+ self.source.media_format = '{key}}'
+ self.assertEqual(self.source.get_example_media_format(), '')
+ # Invalid, not a string
+ self.source.media_format = 1
+ self.assertEqual(self.source.get_example_media_format(), '')
+ # Check all expected keys validate
+ self.source.media_format = 'test-{yyyymmdd}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + timezone.now().strftime('%Y%m%d'))
+ self.source.media_format = 'test-{yyyy_mm_dd}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + timezone.now().strftime('%Y-%m-%d'))
+ self.source.media_format = 'test-{yyyy}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + timezone.now().strftime('%Y'))
+ self.source.media_format = 'test-{source}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.slugname)
+ self.source.media_format = 'test-{source_full}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.name)
+ self.source.media_format = 'test-{title}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-some-media-title-name')
+ self.source.media_format = 'test-{title_full}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-Some Media Title Name')
+ self.source.media_format = 'test-{key}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-SoMeUnIqUiD')
+ self.source.media_format = 'test-{format}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-1080p-vp9-opus')
+ self.source.media_format = 'test-{ext}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.extension)
+ self.source.media_format = 'test-{resolution}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.source_resolution)
+ self.source.media_format = 'test-{height}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-720')
+ self.source.media_format = 'test-{width}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-1280')
+ self.source.media_format = 'test-{vcodec}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.source_vcodec.lower())
+ self.source.media_format = 'test-{acodec}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-' + self.source.source_acodec.lower())
+ self.source.media_format = 'test-{fps}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-24')
+ self.source.media_format = 'test-{hdr}'
+ self.assertEqual(self.source.get_example_media_format(),
+ 'test-hdr')
+
+ def test_media_filename(self):
+ # Check child directories work
+ self.source.media_format = '{yyyy}/{key}.{ext}'
+ self.assertEqual(self.media.directory_path,
+ str(self.source.directory_path / '2017'))
+ self.assertEqual(self.media.filename, '2017/mediakey.mkv')
+ self.source.media_format = '{yyyy}/{yyyy_mm_dd}/{key}.{ext}'
+ self.assertEqual(self.media.directory_path,
+ str(self.source.directory_path / '2017/20179-11'))
+ self.assertEqual(self.media.filename, '2017/20179-11/mediakey.mkv')
+ # Check media specific media format keys work
+ test_media = Media.objects.create(
+ key='test',
+ source=self.source,
+ metadata=metadata,
+ downloaded=True,
+ download_date=timezone.now(),
+ downloaded_format='720p',
+ downloaded_height=720,
+ downloaded_width=1280,
+ downloaded_audio_codec='opus',
+ downloaded_video_codec='vp9',
+ downloaded_container='mkv',
+ downloaded_fps=30,
+ downloaded_hdr=True,
+ downloaded_filesize=12345
+ )
+ # Bypass media-file-exists on-save signal
+ test_media.downloaded = True
+ self.source.media_format = ('{title}_{key}_{resolution}-{height}x{width}-'
+ '{acodec}-{vcodec}-{fps}fps-{hdr}.{ext}')
+ self.assertEqual(test_media.filename,
+ 'no-fancy-stuff_test_720p-720x1280-opus-vp9-30fps-hdr.mkv')
+
+
class FormatMatchingTestCase(TestCase):
def setUp(self):
diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py
index 786ca53f..109b93ce 100644
--- a/tubesync/sync/views.py
+++ b/tubesync/sync/views.py
@@ -256,6 +256,11 @@ class AddSourceView(CreateView):
'index_schedule', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
+ errors = {
+ 'invalid_media_format': _('Invalid media format, the media format contains '
+ 'errors or is empty. Check the table at the end of '
+ 'this page for valid media name variables'),
+ }
def __init__(self, *args, **kwargs):
self.prepopulated_data = {}
@@ -282,6 +287,20 @@ class AddSourceView(CreateView):
initial[k] = v
return initial
+ def form_valid(self, form):
+ # Perform extra validation to make sure the media_format is valid
+ obj = form.save(commit=False)
+ source_type = form.cleaned_data['media_format']
+ example_media_file = obj.get_example_media_format()
+ if example_media_file == '':
+ form.add_error(
+ 'media_format',
+ ValidationError(self.errors['invalid_media_format'])
+ )
+ if form.errors:
+ return super().form_invalid(form)
+ return super().form_valid(form)
+
def get_success_url(self):
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
return append_uri_params(url, {'message': 'source-created'})
@@ -328,6 +347,25 @@ class UpdateSourceView(UpdateView):
'index_schedule', 'delete_old_media', 'days_to_keep',
'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
+ errors = {
+ 'invalid_media_format': _('Invalid media format, the media format contains '
+ 'errors or is empty. Check the table at the end of '
+ 'this page for valid media name variables'),
+ }
+
+ def form_valid(self, form):
+ # Perform extra validation to make sure the media_format is valid
+ obj = form.save(commit=False)
+ source_type = form.cleaned_data['media_format']
+ example_media_file = obj.get_example_media_format()
+ if example_media_file == '':
+ form.add_error(
+ 'media_format',
+ ValidationError(self.errors['invalid_media_format'])
+ )
+ if form.errors:
+ return super().form_invalid(form)
+ return super().form_valid(form)
def get_success_url(self):
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
@@ -497,8 +535,13 @@ class MediaRedownloadView(FormView, SingleObjectMixin):
self.object.thumb = None
# If the media file exists on disk, delete it
if self.object.media_file_exists:
- delete_file(self.object.media_file.path)
+ filepath = self.object.media_file.path
+ delete_file(filepath)
self.object.media_file = None
+ # If the media has an associated thumbnail copied, also delete it
+ barefilepath, fileext = os.path.splitext(filepath)
+ thumbpath = f'{barefilepath}.jpg'
+ delete_file(thumbpath)
# Reset all download data
self.object.downloaded = False
self.object.downloaded_audio_codec = None
@@ -538,8 +581,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the media file exists on disk, delete it
if self.object.media_file_exists:
+ filepath = self.object.media_file.path
delete_file(self.object.media_file.path)
self.object.media_file = None
+ # If the media has an associated thumbnail copied, also delete it
+ barefilepath, fileext = os.path.splitext(filepath)
+ thumbpath = f'{barefilepath}.jpg'
+ delete_file(thumbpath)
# Reset all download data
self.object.downloaded = False
self.object.downloaded_audio_codec = None