Merge remote-tracking branch 'origin/master'
Philipp Hagemeister
10 years ago
commit
35f76e0061
@ -0,0 +1,134 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
import string
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import find_xpath_attr
|
||||
|
||||
|
||||
class HowStuffWorksIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
|
||||
'info_dict': {
|
||||
'id': '450221',
|
||||
'display_id': 'cool-jobs-iditarod-musher',
|
||||
'ext': 'flv',
|
||||
'title': 'Cool Jobs - Iditarod Musher',
|
||||
'description': 'md5:82bb58438a88027b8186a1fccb365f90',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
},
|
||||
'params': {
|
||||
# md5 is not consistent
|
||||
'skip_download': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm',
|
||||
'info_dict': {
|
||||
'id': '553470',
|
||||
'display_id': 'deadliest-catch-jakes-farewell-pots',
|
||||
'ext': 'mp4',
|
||||
'title': 'Deadliest Catch: Jake\'s Farewell Pots',
|
||||
'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
},
|
||||
'params': {
|
||||
# md5 is not consistent
|
||||
'skip_download': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
|
||||
'info_dict': {
|
||||
'id': '440011',
|
||||
'display_id': 'sword-swallowing-1-by-dan-meyer',
|
||||
'ext': 'flv',
|
||||
'title': 'Sword Swallowing #1 by Dan Meyer',
|
||||
'description': 'md5:b2409e88172913e2e7d3d1159b0ef735',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
},
|
||||
'params': {
|
||||
# md5 is not consistent
|
||||
'skip_download': True
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
display_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id')
|
||||
|
||||
mp4 = self._search_regex(
|
||||
r'''(?xs)var\s+clip\s*=\s*{\s*
|
||||
.+?\s*
|
||||
content_id\s*:\s*%s\s*,\s*
|
||||
.+?\s*
|
||||
mp4\s*:\s*\[(.*?),?\]\s*
|
||||
};\s*
|
||||
videoData\.push\(clip\);''' % content_id,
|
||||
webpage, 'mp4', fatal=False, default=None)
|
||||
|
||||
smil = self._download_xml(
|
||||
'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id,
|
||||
content_id, 'Downloading video SMIL')
|
||||
|
||||
http_base = find_xpath_attr(
|
||||
smil,
|
||||
'./{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
|
||||
'name',
|
||||
'httpBase').get('content')
|
||||
|
||||
def random_string(str_len=0):
|
||||
return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)])
|
||||
|
||||
URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12))
|
||||
|
||||
formats = []
|
||||
|
||||
if mp4:
|
||||
for video in json.loads('[%s]' % mp4):
|
||||
bitrate = video['bitrate']
|
||||
fmt = {
|
||||
'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX,
|
||||
'format_id': bitrate,
|
||||
}
|
||||
m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate)
|
||||
if m:
|
||||
fmt['vbr'] = int(m.group('vbr'))
|
||||
formats.append(fmt)
|
||||
else:
|
||||
for video in smil.findall(
|
||||
'.//{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
|
||||
vbr = int(video.attrib['system-bitrate']) / 1000
|
||||
formats.append({
|
||||
'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
|
||||
'format_id': '%dk' % vbr,
|
||||
'vbr': vbr,
|
||||
})
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
title = self._og_search_title(webpage)
|
||||
TITLE_SUFFIX = ' : HowStuffWorks'
|
||||
if title.endswith(TITLE_SUFFIX):
|
||||
title = title[:-len(TITLE_SUFFIX)]
|
||||
|
||||
description = self._og_search_description(webpage)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
return {
|
||||
'id': content_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
}
|
||||
@ -0,0 +1,80 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
unified_strdate
|
||||
)
|
||||
|
||||
|
||||
class JoveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)'
|
||||
_CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current',
|
||||
'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b',
|
||||
'info_dict': {
|
||||
'id': '2744',
|
||||
'ext': 'mp4',
|
||||
'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation',
|
||||
'description': 'md5:015dd4509649c0908bc27f049e0262c6',
|
||||
'thumbnail': 're:^https?://.*\.png$',
|
||||
'upload_date': '20110523',
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation',
|
||||
'md5': '914aeb356f416811d911996434811beb',
|
||||
'info_dict': {
|
||||
'id': '51796',
|
||||
'ext': 'mp4',
|
||||
'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment',
|
||||
'description': 'md5:35ff029261900583970c4023b70f1dc9',
|
||||
'thumbnail': 're:^https?://.*\.png$',
|
||||
'upload_date': '20140802',
|
||||
}
|
||||
},
|
||||
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
chapters_id = self._html_search_regex(
|
||||
r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id')
|
||||
|
||||
chapters_xml = self._download_xml(
|
||||
self._CHAPTERS_URL.format(video_id=chapters_id),
|
||||
video_id, note='Downloading chapters XML',
|
||||
errnote='Failed to download chapters XML')
|
||||
|
||||
video_url = chapters_xml.attrib.get('video')
|
||||
if not video_url:
|
||||
raise ExtractorError('Failed to get the video URL')
|
||||
|
||||
title = self._html_search_meta('citation_title', webpage, 'title')
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
description = self._html_search_regex(
|
||||
r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>',
|
||||
webpage, 'description', fatal=False)
|
||||
publish_date = unified_strdate(self._html_search_meta(
|
||||
'citation_publication_date', webpage, 'publish date', fatal=False))
|
||||
comment_count = self._html_search_regex(
|
||||
r'<meta name="num_comments" content="(\d+) Comments?"',
|
||||
webpage, 'comment count', fatal=False)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'url': video_url,
|
||||
'thumbnail': thumbnail,
|
||||
'description': description,
|
||||
'upload_date': publish_date,
|
||||
'comment_count': comment_count,
|
||||
}
|
||||
@ -0,0 +1,60 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse,
|
||||
get_element_by_attribute,
|
||||
parse_duration,
|
||||
strip_jsonp,
|
||||
)
|
||||
|
||||
|
||||
class MiTeleIE(InfoExtractor):
|
||||
IE_NAME = 'mitele.es'
|
||||
_VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
|
||||
'md5': '6a75fe9d0d3275bead0cb683c616fddb',
|
||||
'info_dict': {
|
||||
'id': '0fce117d',
|
||||
'ext': 'mp4',
|
||||
'title': 'Programa 144 - Tor, la web invisible',
|
||||
'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
|
||||
'display_id': 'programa-144',
|
||||
'duration': 2913,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
episode = mobj.group('episode')
|
||||
webpage = self._download_webpage(url, episode)
|
||||
embed_data_json = self._search_regex(
|
||||
r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
|
||||
flags=re.DOTALL
|
||||
).replace('\'', '"')
|
||||
embed_data = json.loads(embed_data_json)
|
||||
|
||||
info_url = embed_data['flashvars']['host']
|
||||
info_el = self._download_xml(info_url, episode).find('./video/info')
|
||||
|
||||
video_link = info_el.find('videoUrl/link').text
|
||||
token_query = compat_urllib_parse.urlencode({'id': video_link})
|
||||
token_info = self._download_json(
|
||||
'http://token.mitele.es/?' + token_query, episode,
|
||||
transform_source=strip_jsonp
|
||||
)
|
||||
|
||||
return {
|
||||
'id': embed_data['videoId'],
|
||||
'display_id': episode,
|
||||
'title': info_el.find('title').text,
|
||||
'url': token_info['tokenizedUrl'],
|
||||
'description': get_element_by_attribute('class', 'text', webpage),
|
||||
'thumbnail': info_el.find('thumb').text,
|
||||
'duration': parse_duration(info_el.find('duration').text),
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class RtlXlIE(InfoExtractor):
|
||||
IE_NAME = 'rtlxl.nl'
|
||||
_VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
|
||||
'info_dict': {
|
||||
'id': '6e4203a6-0a5e-3596-8424-c599a59e0677',
|
||||
'ext': 'flv',
|
||||
'title': 'RTL Nieuws - Laat',
|
||||
'description': 'Dagelijks het laatste nieuws uit binnen- en '
|
||||
'buitenland. Voor nog meer nieuws kunt u ook gebruikmaken van '
|
||||
'onze mobiele apps.',
|
||||
'timestamp': 1408051800,
|
||||
'upload_date': '20140814',
|
||||
},
|
||||
'params': {
|
||||
# We download the first bytes of the first fragment, it can't be
|
||||
# processed by the f4m downloader beacuse it isn't complete
|
||||
'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
uuid = mobj.group('uuid')
|
||||
|
||||
info = self._download_json(
|
||||
'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
|
||||
uuid)
|
||||
meta = info['meta']
|
||||
material = info['material'][0]
|
||||
episode_info = info['episodes'][0]
|
||||
|
||||
f4m_url = 'http://manifest.us.rtl.nl' + material['videopath']
|
||||
progname = info['abstracts'][0]['name']
|
||||
subtitle = material['title'] or info['episodes'][0]['name']
|
||||
|
||||
return {
|
||||
'id': uuid,
|
||||
'title': '%s - %s' % (progname, subtitle),
|
||||
'formats': self._extract_f4m_formats(f4m_url, uuid),
|
||||
'timestamp': material['original_date'],
|
||||
'description': episode_info['synopsis'],
|
||||
}
|
||||
Loading…
Reference in New Issue