12 Commits
v0.3 ... v0.5

Author SHA1 Message Date
meeb
67af70569b bump to 0.5 2020-12-18 18:07:33 +11:00
meeb
68a62d8a7c add full support for YouTube channels with no vanity name, resolves #9 2020-12-18 17:43:58 +11:00
meeb
55578f4de7 add pretty-json-info-spam wrapper command to aid debugging urls 2020-12-18 17:31:47 +11:00
meeb
47313cb6cc bump to v0.4 2020-12-18 16:20:29 +11:00
meeb
a854b804f0 typo in test 2020-12-18 16:01:53 +11:00
meeb
08c1a82c30 custom filenames with media templates, resolves #5 2020-12-18 15:59:01 +11:00
meeb
25a1a82de4 add copy thumbnails status to source overview page 2020-12-18 13:02:13 +11:00
meeb
ff58f2811b add field for media file name format, part of #5 2020-12-18 13:00:33 +11:00
meeb
83b9c167a9 add option at source level to copy over thumbnails with media, resolves #8 2020-12-18 12:51:04 +11:00
meeb
ffe0049bab support youtube channels without vanity URLs, resolves #6 2020-12-18 12:30:31 +11:00
meeb
c1c39d9e17 typos 2020-12-18 12:26:05 +11:00
meeb
8d7f7e2476 change ffmpeg download location, resolves #4 2020-12-18 11:29:35 +11:00
19 changed files with 658 additions and 66 deletions

View File

@@ -13,7 +13,8 @@ ENV DEBIAN_FRONTEND="noninteractive" \
S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \ S6_EXPECTED_SHA256="52460473413601ff7a84ae690b161a074217ddc734990c2cdee9847166cf669e" \
S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \ S6_DOWNLOAD="https://github.com/just-containers/s6-overlay/releases/download/v${S6_VERSION}/s6-overlay-${ARCH}.tar.gz" \
FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \ FFMPEG_EXPECTED_SHA256="47d95c0129fba27d051748a442a44a73ce1bd38d1e3f9fe1e9dd7258c7581fa5" \
FFMPEG_DOWNLOAD="https://johnvansickle.com/ffmpeg/releases/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz" FFMPEG_DOWNLOAD="https://tubesync.sfo2.digitaloceanspaces.com/ffmpeg-${FFMPEG_VERSION}-${ARCH}-static.tar.xz"
# Install third party software # Install third party software
RUN set -x && \ RUN set -x && \

View File

@@ -1,6 +1,6 @@
# TubeSync # TubeSync
**This is a preview release of TubeSync, it may contain the bugs but should be usable** **This is a preview release of TubeSync, it may contain bugs but should be usable**
TubeSync is a PVR (personal video recorder) for YouTube. Or, like Sonarr but for TubeSync is a PVR (personal video recorder) for YouTube. Or, like Sonarr but for
YouTube (with a built-in download client). It is designed to synchronize channels and YouTube (with a built-in download client). It is designed to synchronize channels and
@@ -18,6 +18,12 @@ hands-free as possible, TubeSync has gradual retrying of failures with back-off
so media which fails to download will be retried for an extended period making it, so media which fails to download will be retried for an extended period making it,
hopefully, quite reliable. hopefully, quite reliable.
# Latest container image
```yaml
ghcr.io/meeb/tubesync:v0.5
```
# Screenshots # Screenshots
### Dashboard ### Dashboard
@@ -92,7 +98,7 @@ Finally, download and run the container:
```bash ```bash
# Pull a versioned image # Pull a versioned image
$ docker pull ghcr.io/meeb/tubesync:v0.3 $ docker pull ghcr.io/meeb/tubesync:v0.5
# Start the container using your user ID and group ID # Start the container using your user ID and group ID
$ docker run \ $ docker run \
-d \ -d \
@@ -103,7 +109,7 @@ $ docker run \
-v /some/directory/tubesync-config:/config \ -v /some/directory/tubesync-config:/config \
-v /some/directory/tubesync-downloads:/downloads \ -v /some/directory/tubesync-downloads:/downloads \
-p 4848:4848 \ -p 4848:4848 \
ghcr.io/meeb/tubesync:v0.3 ghcr.io/meeb/tubesync:v0.5
``` ```
Once running, open `http://localhost:4848` in your browser and you should see the Once running, open `http://localhost:4848` in your browser and you should see the
@@ -115,7 +121,7 @@ Alternatively, for Docker Compose, you can use something like:
```yaml ```yaml
tubesync: tubesync:
image: ghcr.io/meeb/tubesync:v0.3 image: ghcr.io/meeb/tubesync:v0.5
container_name: tubesync container_name: tubesync
restart: unless-stopped restart: unless-stopped
ports: ports:
@@ -124,7 +130,7 @@ Alternatively, for Docker Compose, you can use something like:
- /some/directory/tubesync-config:/config - /some/directory/tubesync-config:/config
- /some/directory/tubesync-downloads:/downloads - /some/directory/tubesync-downloads:/downloads
environment: environment:
- TZ=$TIMEZONE - TZ=Europe/London
- PUID=1000 - PUID=1000
- PGID=1000 - PGID=1000
``` ```
@@ -134,7 +140,7 @@ Alternatively, for Docker Compose, you can use something like:
To update, you can just pull a new version of the container image as they are released. To update, you can just pull a new version of the container image as they are released.
```bash ```bash
$ docker pull pull ghcr.io/meeb/tubesync:v[number] $ docker pull ghcr.io/meeb/tubesync:v[number]
``` ```
Back-end updates such as database migrations should be automatic. Back-end updates such as database migrations should be automatic.

View File

@@ -0,0 +1,97 @@
<h2>Available media name variables</h2>
<table class="striped">
<thead>
<tr>
<th>Name</th>
<th>Description</th>
<th>Output example</th>
</tr>
</thead>
<tbody>
<tr>
<td>{yyyymmdd}</td>
<td>Media publish date in YYYYMMDD</td>
<td>20210101</td>
</tr>
<tr>
<td>{yyyy_mm_dd}</td>
<td>Media publish date in YYYY-MM-DD</td>
<td>2021-01-01</td>
</tr>
<tr>
<td>{yyyy}</td>
<td>Media publish year in YYYY</td>
<td>2021</td>
</tr>
<tr>
<td>{source}</td>
<td>Lower case source name, max 80 chars</td>
<td>my-source</td>
</tr>
<tr>
<td>{source_full}</td>
<td>Full source name</td>
<td>My Source</td>
</tr>
<tr>
<td>{title}</td>
<td>Lower case media title, max 80 chars</td>
<td>my-video</td>
</tr>
<tr>
<td>{title_full}</td>
<td>Full media title</td>
<td>My Video</td>
</tr>
<tr>
<td>{key}</td>
<td>Media unique key or video ID</td>
<td>SoMeUnIqUeId</td>
</tr>
<tr>
<td>{format}</td>
<td>Media format string</td>
<td>720p-avc1-mp4a</td>
</tr>
<tr>
<td>{ext}</td>
<td>File extension</td>
<td>mkv</td>
</tr>
<tr>
<td>{resolution}</td>
<td>Resolution</td>
<td>720p</td>
</tr>
<tr>
<td>{height}</td>
<td>Media height in pixels</td>
<td>720</td>
</tr>
<tr>
<td>{width}</td>
<td>Media width in pixels</td>
<td>1280</td>
</tr>
<tr>
<td>{vcodec}</td>
<td>Media video codec</td>
<td>avc1</td>
</tr>
<tr>
<td>{acodec}</td>
<td>Media audio codec</td>
<td>opus</td>
</tr>
<tr>
<td>{fps}</td>
<td>Media fps</td>
<td>60fps</td>
</tr>
<tr>
<td>{flag_hdr}</td>
<td>Media has HDR flag</td>
<td>hdr</td>
</tr>
</tbody>
</table>

View File

View File

@@ -0,0 +1,18 @@
import json
from django.core.management.base import BaseCommand, CommandError
from sync.youtube import get_media_info
class Command(BaseCommand):
help = 'Displays information obtained by youtube-dl in JSON to the console'
def add_arguments(self, parser):
parser.add_argument('url', type=str)
def handle(self, *args, **options):
url = options['url']
self.stdout.write(f'Showing information for URL: {url}')
info = get_media_info(url)
self.stdout.write(json.dumps(info, indent=4, sort_keys=True))
self.stdout.write('Done')

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.1.4 on 2020-12-18 01:34
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0002_auto_20201213_0817'),
]
operations = [
migrations.AddField(
model_name='source',
name='copy_thumbnails',
field=models.BooleanField(default=False, help_text='Copy thumbnails with the media, these may be detected and used by some media servers', verbose_name='copy thumbnails'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.1.4 on 2020-12-18 01:55
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('sync', '0003_source_copy_thumbnails'),
]
operations = [
migrations.AddField(
model_name='source',
name='media_format',
field=models.CharField(default='{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}', help_text='File format to use for saving files', max_length=200, verbose_name='media format'),
),
]

View File

@@ -7,6 +7,7 @@ from django.conf import settings
from django.db import models from django.db import models
from django.core.files.storage import FileSystemStorage from django.core.files.storage import FileSystemStorage
from django.utils.text import slugify from django.utils.text import slugify
from django.utils import timezone
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from common.errors import NoFormatException from common.errors import NoFormatException
from .youtube import (get_media_info as get_youtube_media_info, from .youtube import (get_media_info as get_youtube_media_info,
@@ -27,10 +28,13 @@ class Source(models.Model):
''' '''
SOURCE_TYPE_YOUTUBE_CHANNEL = 'c' SOURCE_TYPE_YOUTUBE_CHANNEL = 'c'
SOURCE_TYPE_YOUTUBE_CHANNEL_ID = 'i'
SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p' SOURCE_TYPE_YOUTUBE_PLAYLIST = 'p'
SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_PLAYLIST) SOURCE_TYPES = (SOURCE_TYPE_YOUTUBE_CHANNEL, SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
SOURCE_TYPE_YOUTUBE_PLAYLIST)
SOURCE_TYPE_CHOICES = ( SOURCE_TYPE_CHOICES = (
(SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')), (SOURCE_TYPE_YOUTUBE_CHANNEL, _('YouTube channel')),
(SOURCE_TYPE_YOUTUBE_CHANNEL_ID, _('YouTube channel by ID')),
(SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')), (SOURCE_TYPE_YOUTUBE_PLAYLIST, _('YouTube playlist')),
) )
@@ -97,21 +101,25 @@ class Source(models.Model):
# Fontawesome icons used for the source on the front end # Fontawesome icons used for the source on the front end
ICONS = { ICONS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_CHANNEL: '<i class="fab fa-youtube"></i>',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: '<i class="fab fa-youtube"></i>',
SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>', SOURCE_TYPE_YOUTUBE_PLAYLIST: '<i class="fab fa-youtube"></i>',
} }
# Format to use to display a URL for the source # Format to use to display a URL for the source
URLS = { URLS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}', SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/c/{key}',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/channel/{key}',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/playlist?list={key}',
} }
# Callback functions to get a list of media from the source # Callback functions to get a list of media from the source
INDEXERS = { INDEXERS = {
SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_CHANNEL: get_youtube_media_info,
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: get_youtube_media_info,
SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info, SOURCE_TYPE_YOUTUBE_PLAYLIST: get_youtube_media_info,
} }
# Field names to find the media ID used as the key when storing media # Field names to find the media ID used as the key when storing media
KEY_FIELD = { KEY_FIELD = {
SOURCE_TYPE_YOUTUBE_CHANNEL: 'id', SOURCE_TYPE_YOUTUBE_CHANNEL: 'id',
SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'id',
SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id', SOURCE_TYPE_YOUTUBE_PLAYLIST: 'id',
} }
@@ -174,6 +182,12 @@ class Source(models.Model):
unique=True, unique=True,
help_text=_('Directory name to save the media into') help_text=_('Directory name to save the media into')
) )
media_format = models.CharField(
_('media format'),
max_length=200,
default=settings.MEDIA_FORMATSTR_DEFAULT,
help_text=_('File format to use for saving files')
)
index_schedule = models.IntegerField( index_schedule = models.IntegerField(
_('index schedule'), _('index schedule'),
choices=IndexSchedule.choices, choices=IndexSchedule.choices,
@@ -234,6 +248,11 @@ class Source(models.Model):
default=FALLBACK_NEXT_BEST_HD, default=FALLBACK_NEXT_BEST_HD,
help_text=_('What do do when media in your source resolution and codecs is not available') help_text=_('What do do when media in your source resolution and codecs is not available')
) )
copy_thumbnails = models.BooleanField(
_('copy thumbnails'),
default=False,
help_text=_('Copy thumbnails with the media, these may be detected and used by some media servers')
)
has_failed = models.BooleanField( has_failed = models.BooleanField(
_('has failed'), _('has failed'),
default=False, default=False,
@@ -251,6 +270,11 @@ class Source(models.Model):
def icon(self): def icon(self):
return self.ICONS.get(self.source_type) return self.ICONS.get(self.source_type)
@property
def slugname(self):
replaced = self.name.replace('_', '-').replace('&', 'and').replace('+', 'and')
return slugify(replaced)[:80]
@property @property
def is_audio(self): def is_audio(self):
return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO return self.source_resolution == self.SOURCE_RESOLUTION_AUDIO
@@ -325,6 +349,49 @@ class Source(models.Model):
def can_fallback(self): def can_fallback(self):
return self.fallback != self.FALLBACK_FAIL return self.fallback != self.FALLBACK_FAIL
@property
def example_media_format_dict(self):
'''
Populates a dict with real-ish and some placeholder data for media name
format strings. Used for example filenames and media_format validation.
'''
fmt = []
if self.source_resolution:
fmt.append(self.source_resolution)
if self.source_vcodec:
fmt.append(self.source_vcodec.lower())
if self.source_acodec:
fmt.append(self.source_acodec.lower())
if self.prefer_60fps:
fmt.append('60fps')
if self.prefer_hdr:
fmt.append('hdr')
return {
'yyyymmdd': timezone.now().strftime('%Y%m%d'),
'yyyy_mm_dd': timezone.now().strftime('%Y-%m-%d'),
'yyyy': timezone.now().strftime('%Y'),
'source': self.slugname,
'source_full': self.name,
'title': 'some-media-title-name',
'title_full': 'Some Media Title Name',
'key': 'SoMeUnIqUiD',
'format': '-'.join(fmt),
'ext': self.extension,
'resolution': self.source_resolution if self.source_resolution else '',
'height': '720' if self.source_resolution else '',
'width': '1280' if self.source_resolution else '',
'vcodec': self.source_vcodec.lower() if self.source_vcodec else '',
'acodec': self.source_acodec.lower(),
'fps': '24' if self.source_resolution else '',
'hdr': 'hdr' if self.source_resolution else ''
}
def get_example_media_format(self):
try:
return self.media_format.format(**self.example_media_format_dict)
except Exception:
return ''
def index_media(self): def index_media(self):
''' '''
Index the media source returning a list of media metadata as dicts. Index the media source returning a list of media metadata as dicts.
@@ -373,32 +440,39 @@ class Media(models.Model):
# Format to use to display a URL for the media # Format to use to display a URL for the media
URLS = { URLS = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'https://www.youtube.com/watch?v={key}',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'https://www.youtube.com/watch?v={key}',
} }
# Maps standardised names to names used in source metdata # Maps standardised names to names used in source metdata
METADATA_FIELDS = { METADATA_FIELDS = {
'upload_date': { 'upload_date': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'upload_date',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'upload_date',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'upload_date',
}, },
'title': { 'title': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'title',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'title',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'title',
}, },
'thumbnail': { 'thumbnail': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'thumbnail',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'thumbnail',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'thumbnail',
}, },
'description': { 'description': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'description',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'description',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'description', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'description',
}, },
'duration': { 'duration': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'duration',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'duration',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'duration',
}, },
'formats': { 'formats': {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'formats',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: 'formats',
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats', Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: 'formats',
} }
} }
@@ -634,16 +708,39 @@ class Media(models.Model):
('720p', 'avc1', 'mp4a', '60fps', 'hdr') ('720p', 'avc1', 'mp4a', '60fps', 'hdr')
''' '''
fmt = [] fmt = []
resolution = ''
vcodec = ''
acodec = ''
height = '0'
width = '0'
fps = ''
hdr = ''
# If the download has completed use existing values # If the download has completed use existing values
if self.downloaded: if self.downloaded:
resolution = f'{self.downloaded_height}p'
if self.downloaded_format != 'audio': if self.downloaded_format != 'audio':
fmt.append(self.downloaded_video_codec.lower()) vcodec = self.downloaded_video_codec.lower()
fmt.append(self.downloaded_audio_codec.lower()) fmt.append(vcodec)
acodec = self.downloaded_audio_codec.lower()
fmt.append(acodec)
if self.downloaded_format != 'audio': if self.downloaded_format != 'audio':
fmt.append(str(self.downloaded_fps)) fps = str(self.downloaded_fps)
fmt.append(f'{fps}fps')
if self.downloaded_hdr: if self.downloaded_hdr:
fmt.append('hdr') hdr = 'hdr'
return fmt fmt.append(hdr)
height = str(self.downloaded_height)
width = str(self.downloaded_width)
return {
'resolution': resolution,
'height': height,
'width': width,
'vcodec': vcodec,
'acodec': acodec,
'fps': fps,
'hdr': hdr,
'format': tuple(fmt),
}
# Otherwise, calculate from matched format codes # Otherwise, calculate from matched format codes
vformat = None vformat = None
aformat = None aformat = None
@@ -660,15 +757,31 @@ class Media(models.Model):
# Combined # Combined
vformat = cformat vformat = cformat
if vformat: if vformat:
fmt.append(vformat['format'].lower()) resolution = vformat['format'].lower()
fmt.append(vformat['vcodec'].lower()) fmt.append(resolution)
fmt.append(aformat['acodec'].lower()) vcodec = vformat['vcodec'].lower()
fmt.append(vcodec)
acodec = aformat['acodec'].lower()
fmt.append(acodec)
if vformat: if vformat:
if vformat['is_60fps']: if vformat['is_60fps']:
fmt.append('60fps') fps = '60fps'
fmt.append(fps)
if vformat['is_hdr']: if vformat['is_hdr']:
fmt.append('hdr') hdr = 'hdr'
return tuple(fmt) fmt.append(hdr)
height = str(vformat['height'])
width = str(vformat['width'])
return {
'resolution': resolution,
'height': height,
'width': width,
'vcodec': vcodec,
'acodec': acodec,
'fps': fps,
'hdr': hdr,
'format': tuple(fmt),
}
def get_format_by_code(self, format_code): def get_format_by_code(self, format_code):
''' '''
@@ -679,6 +792,35 @@ class Media(models.Model):
return fmt return fmt
return False return False
@property
def format_dict(self):
'''
Returns a dict matching the media_format key requirements for this item
of media.
'''
format_str = self.get_format_str()
display_format = self.get_display_format(format_str)
dateobj = self.upload_date if self.upload_date else self.created
return {
'yyyymmdd': dateobj.strftime('%Y%m%d'),
'yyyy_mm_dd': dateobj.strftime('%Y-%m-%d'),
'yyyy': dateobj.strftime('%Y'),
'source': self.source.slugname,
'source_full': self.source.name,
'title': self.slugtitle,
'title_full': self.title,
'key': self.key,
'format': '-'.join(display_format['format']),
'ext': self.source.extension,
'resolution': display_format['resolution'],
'height': display_format['height'],
'width': display_format['width'],
'vcodec': display_format['vcodec'],
'acodec': display_format['acodec'],
'fps': display_format['fps'],
'hdr': display_format['hdr'],
}
@property @property
def loaded_metadata(self): def loaded_metadata(self):
try: try:
@@ -701,6 +843,11 @@ class Media(models.Model):
field = self.get_metadata_field('title') field = self.get_metadata_field('title')
return self.loaded_metadata.get(field, '').strip() return self.loaded_metadata.get(field, '').strip()
@property
def slugtitle(self):
replaced = self.title.replace('_', '-').replace('&', 'and').replace('+', 'and')
return slugify(replaced)[:80]
@property @property
def thumbnail(self): def thumbnail(self):
field = self.get_metadata_field('thumbnail') field = self.get_metadata_field('thumbnail')
@@ -739,20 +886,24 @@ class Media(models.Model):
@property @property
def filename(self): def filename(self):
# If a media_file has been downloaded use its existing name
if self.media_file: if self.media_file:
return os.path.basename(self.media_file.name) return os.path.basename(self.media_file.name)
upload_date = self.upload_date # Otherwise, create a suitable filename from the source media_format
dateobj = upload_date if upload_date else self.created media_format = str(self.source.media_format)
datestr = dateobj.strftime('%Y-%m-%d') media_details = self.format_dict
source_name = slugify(self.source.name).replace('_', '-') return media_format.format(**media_details)
name = slugify(self.name.replace('&', 'and').replace('+', 'and'))
name = name.replace('_', '-')[:80] @property
key = self.key.strip().replace('_', '-')[:20] def directory_path(self):
format_str = self.get_format_str() # If a media_file has been downloaded use its existing directory
format_tuple = self.get_display_format(format_str) if self.media_file:
fmt = '-'.join(format_tuple) return os.path.dirname(self.media_file.name)
ext = self.source.extension # Otherwise, create a suitable filename from the source media_format
return f'{datestr}_{source_name}_{name}_{key}_{fmt}.{ext}' media_format = str(self.source.media_format)
media_details = self.format_dict
dirname = self.source.directory_path / media_format.format(**media_details)
return os.path.dirname(str(dirname))
@property @property
def filepath(self): def filepath(self):

View File

@@ -1,3 +1,4 @@
import os
from django.conf import settings from django.conf import settings
from django.db.models.signals import pre_save, post_save, pre_delete, post_delete from django.db.models.signals import pre_save, post_save, pre_delete, post_delete
from django.dispatch import receiver from django.dispatch import receiver
@@ -150,8 +151,15 @@ def media_pre_delete(sender, instance, **kwargs):
delete_file(instance.thumb.path) delete_file(instance.thumb.path)
# Delete the media file if it exists # Delete the media file if it exists
if instance.media_file: if instance.media_file:
log.info(f'Deleting media for: {instance} path: {instance.media_file.path}') filepath = instance.media_file.path
delete_file(instance.media_file.path) log.info(f'Deleting media for: {instance} path: {filepath}')
delete_file(filepath)
# Delete thumbnail copy if it exists
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
log.info(f'Deleting thumbnail for: {instance} path: {thumbpath}')
delete_file(thumbpath)
@receiver(post_delete, sender=Media) @receiver(post_delete, sender=Media)
def media_post_delete(sender, instance, **kwargs): def media_post_delete(sender, instance, **kwargs):

View File

@@ -11,6 +11,7 @@ import uuid
from io import BytesIO from io import BytesIO
from hashlib import sha1 from hashlib import sha1
from datetime import timedelta from datetime import timedelta
from shutil import copyfile
from PIL import Image from PIL import Image
from django.conf import settings from django.conf import settings
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
@@ -242,7 +243,7 @@ def download_media_thumbnail(media_id, url):
f'{width}x{height}: {url}') f'{width}x{height}: {url}')
i = resize_image_to_height(i, width, height) i = resize_image_to_height(i, width, height)
image_file = BytesIO() image_file = BytesIO()
i.save(image_file, 'JPEG', quality=80, optimize=True, progressive=True) i.save(image_file, 'JPEG', quality=85, optimize=True, progressive=True)
image_file.seek(0) image_file.seek(0)
media.thumb.save( media.thumb.save(
'thumb', 'thumb',
@@ -272,17 +273,18 @@ def download_media(media_id):
log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it ' log.warn(f'Download task triggeredd media: {media} (UUID: {media.pk}) but it '
f'is now marked to be skipped, not downloading') f'is now marked to be skipped, not downloading')
return return
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{media.filepath}"') filepath = media.filepath
log.info(f'Downloading media: {media} (UUID: {media.pk}) to: "{filepath}"')
format_str, container = media.download_media() format_str, container = media.download_media()
if os.path.exists(media.filepath): if os.path.exists(filepath):
# Media has been downloaded successfully # Media has been downloaded successfully
log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: ' log.info(f'Successfully downloaded media: {media} (UUID: {media.pk}) to: '
f'"{media.filepath}"') f'"{filepath}"')
# Link the media file to the object and update info about the download # Link the media file to the object and update info about the download
media.media_file.name = str(media.filepath) media.media_file.name = str(filepath)
media.downloaded = True media.downloaded = True
media.download_date = timezone.now() media.download_date = timezone.now()
media.downloaded_filesize = os.path.getsize(media.filepath) media.downloaded_filesize = os.path.getsize(filepath)
media.downloaded_container = container media.downloaded_container = container
if '+' in format_str: if '+' in format_str:
# Seperate audio and video streams # Seperate audio and video streams
@@ -313,6 +315,13 @@ def download_media(media_id):
else: else:
media.downloaded_format = 'audio' media.downloaded_format = 'audio'
media.save() media.save()
# If selected, copy the thumbnail over as well
if media.source.copy_thumbnails and media.thumb:
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
log.info(f'Copying media thumbnail from: {media.thumb.path} '
f'to: {thumbpath}')
copyfile(media.thumb.path, thumbpath)
# Schedule a task to update media servers # Schedule a task to update media servers
for mediaserver in MediaServer.objects.all(): for mediaserver in MediaServer.objects.all():
log.info(f'Scheduling media server updates') log.info(f'Scheduling media server updates')

View File

@@ -80,6 +80,10 @@
<td class="hide-on-small-only">Filename</td> <td class="hide-on-small-only">Filename</td>
<td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td> <td><span class="hide-on-med-and-up">Filename<br></span><strong>{{ media.filename }}</strong></td>
</tr> </tr>
<tr title="The filename the media will be downloaded as">
<td class="hide-on-small-only">Directory</td>
<td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ media.directory_path }}</strong></td>
</tr>
<tr title="Size of the file on disk"> <tr title="Size of the file on disk">
<td class="hide-on-small-only">File size</td> <td class="hide-on-small-only">File size</td>
<td><span class="hide-on-med-and-up">File size<br></span><strong>{{ media.downloaded_filesize|filesizeformat }}</strong></td> <td><span class="hide-on-med-and-up">File size<br></span><strong>{{ media.downloaded_filesize|filesizeformat }}</strong></td>

View File

@@ -23,4 +23,9 @@
</div> </div>
</form> </form>
</div> </div>
<div class="row">
<div class="col s12">
{% include 'mediaformatvars.html' %}
</div>
</div>
{% endblock %} {% endblock %}

View File

@@ -25,4 +25,9 @@
</div> </div>
</form> </form>
</div> </div>
<div class="row">
<div class="col s12">
{% include 'mediaformatvars.html' %}
</div>
</div>
{% endblock %} {% endblock %}

View File

@@ -43,6 +43,14 @@
<td class="hide-on-small-only">Directory</td> <td class="hide-on-small-only">Directory</td>
<td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td> <td><span class="hide-on-med-and-up">Directory<br></span><strong>{{ source.directory }}</strong></td>
</tr> </tr>
<tr title="Media file name format to use for saving files">
<td class="hide-on-small-only">Media format</td>
<td><span class="hide-on-med-and-up">Media format<br></span><strong>{{ source.media_format }}</strong></td>
</tr>
<tr title="Example file name for media format">
<td class="hide-on-small-only">Example filename</td>
<td><span class="hide-on-med-and-up">Example filename<br></span><strong>{{ source.get_example_media_format }}</strong></td>
</tr>
<tr title="Schedule of how often to index the source for new media"> <tr title="Schedule of how often to index the source for new media">
<td class="hide-on-small-only">Index schedule</td> <td class="hide-on-small-only">Index schedule</td>
<td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td> <td><span class="hide-on-med-and-up">Index schedule<br></span><strong>{{ source.get_index_schedule_display }}</strong></td>
@@ -85,6 +93,10 @@
<td class="hide-on-small-only">Fallback</td> <td class="hide-on-small-only">Fallback</td>
<td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ source.get_fallback_display }}</strong></td> <td><span class="hide-on-med-and-up">Fallback<br></span><strong>{{ source.get_fallback_display }}</strong></td>
</tr> </tr>
<tr title="Should media thumbnails be copied over with the media?">
<td class="hide-on-small-only">Copy thumbnails?</td>
<td><span class="hide-on-med-and-up">Copy thumbnails?<br></span><strong>{% if source.copy_thumbnails %}<i class="fas fa-check"></i>{% else %}<i class="fas fa-times"></i>{% endif %}</strong></td>
</tr>
{% if source.delete_old_media and source.days_to_keep > 0 %} {% if source.delete_old_media and source.days_to_keep > 0 %}
<tr title="Days after which your media from this source will be locally deleted"> <tr title="Days after which your media from this source will be locally deleted">
<td class="hide-on-small-only">Delete old media</td> <td class="hide-on-small-only">Delete old media</td>

View File

@@ -10,10 +10,13 @@
</div> </div>
{% include 'infobox.html' with message=message %} {% include 'infobox.html' with message=message %}
<div class="row"> <div class="row">
<div class="col s12 l6 margin-bottom"> <div class="col m12 xl4 margin-bottom">
<a href="{% url 'sync:validate-source' source_type='youtube-channel' %}" class="btn">Add a YouTube channel <i class="fab fa-youtube"></i></a> <a href="{% url 'sync:validate-source' source_type='youtube-channel' %}" class="btn">Add a YouTube channel <i class="fab fa-youtube"></i></a>
</div> </div>
<div class="col s12 l6 margin-bottom"> <div class="col m12 xl4 margin-bottom">
<a href="{% url 'sync:validate-source' source_type='youtube-channel-id' %}" class="btn">Add a YouTube channel by ID <i class="fab fa-youtube"></i></a>
</div>
<div class="col m12 xl4 margin-bottom">
<a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a> <a href="{% url 'sync:validate-source' source_type='youtube-playlist' %}" class="btn">Add a YouTube playlist <i class="fab fa-youtube"></i></a>
</div> </div>
</div> </div>

View File

@@ -28,6 +28,7 @@ class FrontEndTestCase(TestCase):
def test_validate_source(self): def test_validate_source(self):
test_source_types = { test_source_types = {
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL, 'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST, 'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
} }
test_sources = { test_sources = {
@@ -35,6 +36,7 @@ class FrontEndTestCase(TestCase):
'valid': ( 'valid': (
'https://www.youtube.com/testchannel', 'https://www.youtube.com/testchannel',
'https://www.youtube.com/c/testchannel', 'https://www.youtube.com/c/testchannel',
'https://www.youtube.com/c/testchannel/videos',
), ),
'invalid_schema': ( 'invalid_schema': (
'http://www.youtube.com/c/playlist', 'http://www.youtube.com/c/playlist',
@@ -50,13 +52,37 @@ class FrontEndTestCase(TestCase):
), ),
'invalid_is_playlist': ( 'invalid_is_playlist': (
'https://www.youtube.com/c/playlist', 'https://www.youtube.com/c/playlist',
'https://www.youtube.com/c/playlist', ),
'invalid_channel_with_id': (
'https://www.youtube.com/channel/channelid',
'https://www.youtube.com/channel/channelid/videos',
),
},
'youtube-channel-id': {
'valid': (
'https://www.youtube.com/channel/channelid',
'https://www.youtube.com/channel/channelid/videos',
),
'invalid_schema': (
'http://www.youtube.com/channel/channelid',
'ftp://www.youtube.com/channel/channelid',
),
'invalid_domain': (
'https://www.test.com/channel/channelid',
'https://www.example.com/channel/channelid',
),
'invalid_path': (
'https://www.youtube.com/test/invalid',
'https://www.youtube.com/channel/test/invalid',
),
'invalid_is_named_channel': (
'https://www.youtube.com/c/testname',
), ),
}, },
'youtube-playlist': { 'youtube-playlist': {
'valid': ( 'valid': (
'https://www.youtube.com/playlist?list=testplaylist' 'https://www.youtube.com/playlist?list=testplaylist',
'https://www.youtube.com/watch?v=testvideo&list=testplaylist' 'https://www.youtube.com/watch?v=testvideo&list=testplaylist',
), ),
'invalid_schema': ( 'invalid_schema': (
'http://www.youtube.com/playlist?list=testplaylist', 'http://www.youtube.com/playlist?list=testplaylist',
@@ -73,6 +99,7 @@ class FrontEndTestCase(TestCase):
'invalid_is_channel': ( 'invalid_is_channel': (
'https://www.youtube.com/testchannel', 'https://www.youtube.com/testchannel',
'https://www.youtube.com/c/testchannel', 'https://www.youtube.com/c/testchannel',
'https://www.youtube.com/channel/testchannel',
), ),
} }
} }
@@ -83,19 +110,21 @@ class FrontEndTestCase(TestCase):
response = c.get('/source-validate/invalid') response = c.get('/source-validate/invalid')
self.assertEqual(response.status_code, 404) self.assertEqual(response.status_code, 404)
for (source_type, tests) in test_sources.items(): for (source_type, tests) in test_sources.items():
for test, field in tests.items(): for test, urls in tests.items():
source_type_char = test_source_types.get(source_type) for url in urls:
data = {'source_url': field, 'source_type': source_type_char} source_type_char = test_source_types.get(source_type)
response = c.post(f'/source-validate/{source_type}', data) data = {'source_url': url, 'source_type': source_type_char}
if test == 'valid': response = c.post(f'/source-validate/{source_type}', data)
# Valid source tests should bounce to /source-add if test == 'valid':
self.assertEqual(response.status_code, 302) # Valid source tests should bounce to /source-add
url_parts = urlsplit(response.url) self.assertEqual(response.status_code, 302)
self.assertEqual(url_parts.path, '/source-add') url_parts = urlsplit(response.url)
else: self.assertEqual(url_parts.path, '/source-add')
# Invalid source tests should reload the page with an error message else:
self.assertEqual(response.status_code, 200) # Invalid source tests should reload the page with an error
self.assertIn('<ul class="errorlist">', response.content.decode()) self.assertEqual(response.status_code, 200)
self.assertIn('<ul class="errorlist">',
response.content.decode())
def test_add_source_prepopulation(self): def test_add_source_prepopulation(self):
c = Client() c = Client()
@@ -131,6 +160,7 @@ class FrontEndTestCase(TestCase):
'key': 'testkey', 'key': 'testkey',
'name': 'testname', 'name': 'testname',
'directory': 'testdirectory', 'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': 3600, 'index_schedule': 3600,
'delete_old_media': False, 'delete_old_media': False,
'days_to_keep': 14, 'days_to_keep': 14,
@@ -170,6 +200,7 @@ class FrontEndTestCase(TestCase):
'key': 'updatedkey', # changed 'key': 'updatedkey', # changed
'name': 'testname', 'name': 'testname',
'directory': 'testdirectory', 'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': Source.IndexSchedule.EVERY_HOUR, 'index_schedule': Source.IndexSchedule.EVERY_HOUR,
'delete_old_media': False, 'delete_old_media': False,
'days_to_keep': 14, 'days_to_keep': 14,
@@ -197,6 +228,7 @@ class FrontEndTestCase(TestCase):
'key': 'updatedkey', 'key': 'updatedkey',
'name': 'testname', 'name': 'testname',
'directory': 'testdirectory', 'directory': 'testdirectory',
'media_format': settings.MEDIA_FORMATSTR_DEFAULT,
'index_schedule': Source.IndexSchedule.EVERY_2_HOURS, # changed 'index_schedule': Source.IndexSchedule.EVERY_2_HOURS, # changed
'delete_old_media': False, 'delete_old_media': False,
'days_to_keep': 14, 'days_to_keep': 14,
@@ -422,6 +454,137 @@ all_test_metadata = {
} }
class FilepathTestCase(TestCase):
def setUp(self):
# Disable general logging for test case
logging.disable(logging.CRITICAL)
# Add a test source
self.source = Source.objects.create(
source_type=Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
key='testkey',
name='testname',
directory='testdirectory',
media_format=settings.MEDIA_FORMATSTR_DEFAULT,
index_schedule=3600,
delete_old_media=False,
days_to_keep=14,
source_resolution=Source.SOURCE_RESOLUTION_1080P,
source_vcodec=Source.SOURCE_VCODEC_VP9,
source_acodec=Source.SOURCE_ACODEC_OPUS,
prefer_60fps=False,
prefer_hdr=False,
fallback=Source.FALLBACK_FAIL
)
# Add some test media
self.media = Media.objects.create(
key='mediakey',
source=self.source,
metadata=metadata,
)
def test_source_dirname(self):
# Check media format validation is working
# Empty
self.source.media_format = ''
self.assertEqual(self.source.get_example_media_format(), '')
# Invalid, bad key
self.source.media_format = '{test}'
self.assertEqual(self.source.get_example_media_format(), '')
# Invalid, extra brackets
self.source.media_format = '{key}}'
self.assertEqual(self.source.get_example_media_format(), '')
# Invalid, not a string
self.source.media_format = 1
self.assertEqual(self.source.get_example_media_format(), '')
# Check all expected keys validate
self.source.media_format = 'test-{yyyymmdd}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + timezone.now().strftime('%Y%m%d'))
self.source.media_format = 'test-{yyyy_mm_dd}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + timezone.now().strftime('%Y-%m-%d'))
self.source.media_format = 'test-{yyyy}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + timezone.now().strftime('%Y'))
self.source.media_format = 'test-{source}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.slugname)
self.source.media_format = 'test-{source_full}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.name)
self.source.media_format = 'test-{title}'
self.assertEqual(self.source.get_example_media_format(),
'test-some-media-title-name')
self.source.media_format = 'test-{title_full}'
self.assertEqual(self.source.get_example_media_format(),
'test-Some Media Title Name')
self.source.media_format = 'test-{key}'
self.assertEqual(self.source.get_example_media_format(),
'test-SoMeUnIqUiD')
self.source.media_format = 'test-{format}'
self.assertEqual(self.source.get_example_media_format(),
'test-1080p-vp9-opus')
self.source.media_format = 'test-{ext}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.extension)
self.source.media_format = 'test-{resolution}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.source_resolution)
self.source.media_format = 'test-{height}'
self.assertEqual(self.source.get_example_media_format(),
'test-720')
self.source.media_format = 'test-{width}'
self.assertEqual(self.source.get_example_media_format(),
'test-1280')
self.source.media_format = 'test-{vcodec}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.source_vcodec.lower())
self.source.media_format = 'test-{acodec}'
self.assertEqual(self.source.get_example_media_format(),
'test-' + self.source.source_acodec.lower())
self.source.media_format = 'test-{fps}'
self.assertEqual(self.source.get_example_media_format(),
'test-24')
self.source.media_format = 'test-{hdr}'
self.assertEqual(self.source.get_example_media_format(),
'test-hdr')
def test_media_filename(self):
# Check child directories work
self.source.media_format = '{yyyy}/{key}.{ext}'
self.assertEqual(self.media.directory_path,
str(self.source.directory_path / '2017'))
self.assertEqual(self.media.filename, '2017/mediakey.mkv')
self.source.media_format = '{yyyy}/{yyyy_mm_dd}/{key}.{ext}'
self.assertEqual(self.media.directory_path,
str(self.source.directory_path / '2017/2017-09-11'))
self.assertEqual(self.media.filename, '2017/2017-09-11/mediakey.mkv')
# Check media specific media format keys work
test_media = Media.objects.create(
key='test',
source=self.source,
metadata=metadata,
downloaded=True,
download_date=timezone.now(),
downloaded_format='720p',
downloaded_height=720,
downloaded_width=1280,
downloaded_audio_codec='opus',
downloaded_video_codec='vp9',
downloaded_container='mkv',
downloaded_fps=30,
downloaded_hdr=True,
downloaded_filesize=12345
)
# Bypass media-file-exists on-save signal
test_media.downloaded = True
self.source.media_format = ('{title}_{key}_{resolution}-{height}x{width}-'
'{acodec}-{vcodec}-{fps}fps-{hdr}.{ext}')
self.assertEqual(test_media.filename,
'no-fancy-stuff_test_720p-720x1280-opus-vp9-30fps-hdr.mkv')
class FormatMatchingTestCase(TestCase): class FormatMatchingTestCase(TestCase):
def setUp(self): def setUp(self):

View File

@@ -128,10 +128,12 @@ class ValidateSourceView(FormView):
} }
source_types = { source_types = {
'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL, 'youtube-channel': Source.SOURCE_TYPE_YOUTUBE_CHANNEL,
'youtube-channel-id': Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID,
'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST, 'youtube-playlist': Source.SOURCE_TYPE_YOUTUBE_PLAYLIST,
} }
help_item = { help_item = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL: _('YouTube channel'),
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _('YouTube channel ID'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'), Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _('YouTube playlist'),
} }
help_texts = { help_texts = {
@@ -141,6 +143,13 @@ class ValidateSourceView(FormView):
'where <strong>CHANNELNAME</strong> is the name of the channel you want ' 'where <strong>CHANNELNAME</strong> is the name of the channel you want '
'to add.' 'to add.'
), ),
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: _(
'Enter a YouTube channel URL by channel ID into the box below. A channel '
'URL by channel ID will be in the format of <strong>'
'https://www.youtube.com/channel/BiGLoNgUnIqUeId</strong> '
'where <strong>BiGLoNgUnIqUeId</strong> is the ID of the channel you want '
'to add.'
),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _( Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: _(
'Enter a YouTube playlist URL into the box below. A playlist URL will be ' 'Enter a YouTube playlist URL into the box below. A playlist URL will be '
'in the format of <strong>https://www.youtube.com/playlist?list=' 'in the format of <strong>https://www.youtube.com/playlist?list='
@@ -150,6 +159,8 @@ class ValidateSourceView(FormView):
} }
help_examples = { help_examples = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google', Source.SOURCE_TYPE_YOUTUBE_CHANNEL: 'https://www.youtube.com/google',
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('https://www.youtube.com/channel/'
'UCK8sQmJBp8GCxrOtXWBpyEA'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list=' Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('https://www.youtube.com/playlist?list='
'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r') 'PL590L5WQmH8dpP0RyH5pCfIaDEdt9nk7r')
} }
@@ -157,12 +168,21 @@ class ValidateSourceView(FormView):
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: { Source.SOURCE_TYPE_YOUTUBE_CHANNEL: {
'scheme': 'https', 'scheme': 'https',
'domain': 'www.youtube.com', 'domain': 'www.youtube.com',
'path_regex': '^\/(c\/)?([^\/]+)$', 'path_regex': '^\/(c\/)?([^\/]+)(\/videos)?$',
'path_must_not_match': ('/playlist', '/c/playlist'), 'path_must_not_match': ('/playlist', '/c/playlist'),
'qs_args': [], 'qs_args': [],
'extract_key': ('path_regex', 1), 'extract_key': ('path_regex', 1),
'example': 'https://www.youtube.com/SOMECHANNEL' 'example': 'https://www.youtube.com/SOMECHANNEL'
}, },
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: {
'scheme': 'https',
'domain': 'www.youtube.com',
'path_regex': '^\/channel\/([^\/]+)(\/videos)?$',
'path_must_not_match': ('/playlist', '/c/playlist'),
'qs_args': [],
'extract_key': ('path_regex', 0),
'example': 'https://www.youtube.com/channel/CHANNELID'
},
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: { Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: {
'scheme': 'https', 'scheme': 'https',
'domain': 'www.youtube.com', 'domain': 'www.youtube.com',
@@ -175,6 +195,7 @@ class ValidateSourceView(FormView):
} }
prepopulate_fields = { prepopulate_fields = {
Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'), Source.SOURCE_TYPE_YOUTUBE_CHANNEL: ('source_type', 'key', 'name', 'directory'),
Source.SOURCE_TYPE_YOUTUBE_CHANNEL_ID: ('source_type', 'key'),
Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'), Source.SOURCE_TYPE_YOUTUBE_PLAYLIST: ('source_type', 'key'),
} }
@@ -252,9 +273,15 @@ class AddSourceView(CreateView):
template_name = 'sync/source-add.html' template_name = 'sync/source-add.html'
model = Source model = Source
fields = ('source_type', 'key', 'name', 'directory', 'index_schedule', fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'index_schedule', 'delete_old_media', 'days_to_keep',
'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback') 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
'this page for valid media name variables'),
}
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.prepopulated_data = {} self.prepopulated_data = {}
@@ -281,6 +308,20 @@ class AddSourceView(CreateView):
initial[k] = v initial[k] = v
return initial return initial
def form_valid(self, form):
# Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False)
source_type = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
if example_media_file == '':
form.add_error(
'media_format',
ValidationError(self.errors['invalid_media_format'])
)
if form.errors:
return super().form_invalid(form)
return super().form_valid(form)
def get_success_url(self): def get_success_url(self):
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk}) url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
return append_uri_params(url, {'message': 'source-created'}) return append_uri_params(url, {'message': 'source-created'})
@@ -323,9 +364,29 @@ class UpdateSourceView(UpdateView):
template_name = 'sync/source-update.html' template_name = 'sync/source-update.html'
model = Source model = Source
fields = ('source_type', 'key', 'name', 'directory', 'index_schedule', fields = ('source_type', 'key', 'name', 'directory', 'media_format',
'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'index_schedule', 'delete_old_media', 'days_to_keep',
'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback') 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps',
'prefer_hdr', 'fallback', 'copy_thumbnails')
errors = {
'invalid_media_format': _('Invalid media format, the media format contains '
'errors or is empty. Check the table at the end of '
'this page for valid media name variables'),
}
def form_valid(self, form):
# Perform extra validation to make sure the media_format is valid
obj = form.save(commit=False)
source_type = form.cleaned_data['media_format']
example_media_file = obj.get_example_media_format()
if example_media_file == '':
form.add_error(
'media_format',
ValidationError(self.errors['invalid_media_format'])
)
if form.errors:
return super().form_invalid(form)
return super().form_valid(form)
def get_success_url(self): def get_success_url(self):
url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk}) url = reverse_lazy('sync:source', kwargs={'pk': self.object.pk})
@@ -495,8 +556,13 @@ class MediaRedownloadView(FormView, SingleObjectMixin):
self.object.thumb = None self.object.thumb = None
# If the media file exists on disk, delete it # If the media file exists on disk, delete it
if self.object.media_file_exists: if self.object.media_file_exists:
delete_file(self.object.media_file.path) filepath = self.object.media_file.path
delete_file(filepath)
self.object.media_file = None self.object.media_file = None
# If the media has an associated thumbnail copied, also delete it
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
delete_file(thumbpath)
# Reset all download data # Reset all download data
self.object.downloaded = False self.object.downloaded = False
self.object.downloaded_audio_codec = None self.object.downloaded_audio_codec = None
@@ -536,8 +602,13 @@ class MediaSkipView(FormView, SingleObjectMixin):
delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),)) delete_task_by_media('sync.tasks.download_media', (str(self.object.pk),))
# If the media file exists on disk, delete it # If the media file exists on disk, delete it
if self.object.media_file_exists: if self.object.media_file_exists:
filepath = self.object.media_file.path
delete_file(self.object.media_file.path) delete_file(self.object.media_file.path)
self.object.media_file = None self.object.media_file = None
# If the media has an associated thumbnail copied, also delete it
barefilepath, fileext = os.path.splitext(filepath)
thumbpath = f'{barefilepath}.jpg'
delete_file(thumbpath)
# Reset all download data # Reset all download data
self.object.downloaded = False self.object.downloaded = False
self.object.downloaded_audio_codec = None self.object.downloaded_audio_codec = None

View File

@@ -6,7 +6,7 @@ CONFIG_BASE_DIR = BASE_DIR
DOWNLOADS_BASE_DIR = BASE_DIR DOWNLOADS_BASE_DIR = BASE_DIR
VERSION = 0.3 VERSION = 0.5
SECRET_KEY = '' SECRET_KEY = ''
DEBUG = False DEBUG = False
ALLOWED_HOSTS = [] ALLOWED_HOSTS = []
@@ -147,6 +147,9 @@ YOUTUBE_DEFAULTS = {
} }
MEDIA_FORMATSTR_DEFAULT = '{yyyymmdd}_{source}_{title}_{key}_{format}.{ext}'
try: try:
from .local_settings import * from .local_settings import *
except ImportError as e: except ImportError as e: