mikhailnov
/
youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[soompi] Improve and simplify

Browse Source
master
Sergey M․ 9 years ago
parent 0385d64223
commit b2cf6543b2
1 changed files with 81 additions and 65 deletions
Show Stats Download Patch File Download Diff File

146
youtube_dl/extractor/soompi.py
Unescape Escape View File

@ -2,17 +2,31 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import re import re
import json
import base64
import xml.etree.ElementTree
# Soompi uses the same subtitle encryption as crunchyroll
from .crunchyroll import CrunchyrollIE from .crunchyroll import CrunchyrollIE
from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
ExtractorError,
int_or_none,
remove_start,
xpath_text,
)
class SoompiIE(CrunchyrollIE):
class SoompiBaseIE(InfoExtractor):
def _get_episodes(self, webpage, episode_filter=None):
episodes = self._parse_json(
self._search_regex(
r'VIDEOS\s*=\s*(\[.+?\]);', webpage, 'episodes JSON'),
None)
return list(filter(episode_filter, episodes))
class SoompiIE(SoompiBaseIE, CrunchyrollIE):
IE_NAME = 'soompi' IE_NAME = 'soompi'
_VALID_URL = r'^https?://tv\.soompi\.com/en/watch/(?P<id>[0-9]+)' _VALID_URL = r'https?://tv\.soompi\.com/(?:en/)?watch/(?P<id>[0-9]+)'
_TESTS = [{ _TESTS = [{
'url': 'http://tv.soompi.com/en/watch/29235', 'url': 'http://tv.soompi.com/en/watch/29235',
'info_dict': { 'info_dict': {
@ -26,84 +40,86 @@ class SoompiIE(CrunchyrollIE):
}, },
}] }]
def _get_episodes(self, webpage, episode_filter=None): def _get_episode(self, webpage, video_id):
episodes = json.loads( return self._get_episodes(webpage, lambda x: x['id'] == video_id)[0]
self._search_regex(r'\s+VIDEOS\s+= (\[.+?\]);', webpage, "episodes meta"))
return [ep for ep in episodes if episode_filter is None or episode_filter(ep)]
def _get_subtitles(self, video_id, show_format_xml):
subtitles = {}
subtitle_info_nodes = show_format_xml.findall('./{default}preload/subtitles/subtitle')
subtitle_nodes = show_format_xml.findall('./{default}preload/subtitle')
def _get_subtitles(self, config, video_id):
sub_langs = {} sub_langs = {}
for i in subtitle_info_nodes: for subtitle in config.findall('./{default}preload/subtitles/subtitle'):
sub_langs[i.attrib["id"]] = i.attrib["title"] sub_langs[subtitle.attrib['id']] = subtitle.attrib['title']
for s in subtitle_nodes: subtitles = {}
lang_code = sub_langs.get(s.attrib["id"], None) for s in config.findall('./{default}preload/subtitle'):
if lang_code is None: lang_code = sub_langs.get(s.attrib['id'])
if not lang_code:
continue
sub_id = s.get('id')
data = xpath_text(s, './data', 'data')
iv = xpath_text(s, './iv', 'iv')
if not id or not iv or not data:
continue continue
sub_id = int(s.attrib["id"])
iv = base64.b64decode(s.find("iv").text)
data = base64.b64decode(s.find("data").text)
subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8') subtitle = self._decrypt_subtitles(data, iv, sub_id).decode('utf-8')
sub_root = xml.etree.ElementTree.fromstring(subtitle) subtitles[lang_code] = self._extract_subtitles(subtitle)
subtitles[lang_code] = [{
'ext': 'srt', 'data': self._convert_subtitles_to_srt(sub_root)
}, {
'ext': 'ass', 'data': self._convert_subtitles_to_ass(sub_root)
}]
return subtitles return subtitles
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage( try:
url, video_id, note="Downloading episode page", webpage = self._download_webpage(
errnote="Video may not be available for your location") url, video_id, 'Downloading episode page')
vid_formats = re.findall(r"\?quality=q([0-9]+)", webpage) except ExtractorError as ee:
if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
show_meta = json.loads( webpage = ee.cause.read()
self._search_regex(r'\s+var show = (\{.+?\});', webpage, "show meta")) block_message = self._html_search_regex(
episodes = self._get_episodes( r'(?s)<div class="block-message">(.+?)</div>', webpage,
webpage, episode_filter=lambda x: x['id'] == video_id) 'block message', default=None)
if block_message:
title = episodes[0]["name"] raise ExtractorError(block_message, expected=True)
description = episodes[0]["description"] raise
duration = int(episodes[0]["duration"])
slug = show_meta["slug"]
formats = [] formats = []
show_format_xml = None config = None
for vf in vid_formats: for format_id in re.findall(r'\?quality=([0-9a-zA-Z]+)', webpage):
show_format_url = "http://tv.soompi.com/en/show/%s/%s-config.xml?mode=hls&quality=q%s" \ config = self._download_xml(
% (slug, video_id, vf) 'http://tv.soompi.com/en/show/_/%s-config.xml?mode=hls&quality=%s' % (video_id, format_id),
show_format_xml = self._download_xml( video_id, 'Downloading %s XML' % format_id)
show_format_url, video_id, note="Downloading q%s show xml" % vf) m3u8_url = xpath_text(
avail_formats = self._extract_m3u8_formats( config, './{default}preload/stream_info/file',
show_format_xml.find('./{default}preload/stream_info/file').text, '%s m3u8 URL' % format_id)
video_id, ext="mp4", m3u8_id=vf, preference=int(vf)) if not m3u8_url:
formats.extend(avail_formats) continue
formats.extend(self._extract_m3u8_formats(
m3u8_url, video_id, 'mp4', m3u8_id=format_id))
self._sort_formats(formats) self._sort_formats(formats)
subtitles = self.extract_subtitles(video_id, show_format_xml) episode = self._get_episode(webpage, video_id)
title = episode['name']
description = episode.get('description')
duration = int_or_none(episode.get('duration'))
thumbnails = [{
'id': thumbnail_id,
'url': thumbnail_url,
} for thumbnail_id, thumbnail_url in episode.get('img_url', {}).items()]
subtitles = self.extract_subtitles(config, video_id)
return { return {
'id': video_id, 'id': video_id,
'title': title, 'title': title,
'description': description, 'description': description,
'thumbnails': thumbnails,
'duration': duration, 'duration': duration,
'formats': formats, 'formats': formats,
'subtitles': subtitles 'subtitles': subtitles
} }
class SoompiShowIE(SoompiIE): class SoompiShowIE(SoompiBaseIE):
IE_NAME = 'soompi:show' IE_NAME = 'soompi:show'
_VALID_URL = r'^https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)' _VALID_URL = r'https?://tv\.soompi\.com/en/shows/(?P<id>[0-9a-zA-Z\-_]+)'
_TESTS = [{ _TESTS = [{
'url': 'http://tv.soompi.com/en/shows/liar-game', 'url': 'http://tv.soompi.com/en/shows/liar-game',
'info_dict': { 'info_dict': {
@ -117,14 +133,14 @@ class SoompiShowIE(SoompiIE):
def _real_extract(self, url): def _real_extract(self, url):
show_id = self._match_id(url) show_id = self._match_id(url)
webpage = self._download_webpage(url, show_id, note="Downloading show page") webpage = self._download_webpage(
title = self._og_search_title(webpage).replace("SoompiTV | ", "") url, show_id, 'Downloading show page')
title = remove_start(self._og_search_title(webpage), 'SoompiTV | ')
description = self._og_search_description(webpage) description = self._og_search_description(webpage)
episodes = self._get_episodes(webpage) entries = [
entries = [] self.url_result('http://tv.soompi.com/en/watch/%s' % episode['id'], 'Soompi')
for ep in episodes: for episode in self._get_episodes(webpage)]
entries.append(self.url_result(
'http://tv.soompi.com/en/watch/%s' % ep['id'], 'Soompi', ep['id']))
return self.playlist_result(entries, show_id, title, description) return self.playlist_result(entries, show_id, title, description)

Write Preview
Loading…
Cancel
Save