From 8708d764252a83e650f36fe47f89ace4f2a91a5b Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 11 Feb 2015 16:39:15 +0800 Subject: [PATCH 1/4] [camdemy] Add new extractor Single file download done, while folder extaction in plan --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/camdemy.py | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 youtube_dl/extractor/camdemy.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index da3b2b2d0..496e4d0b7 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -49,6 +49,7 @@ from .brightcove import BrightcoveIE from .buzzfeed import BuzzFeedIE from .byutv import BYUtvIE from .c56 import C56IE +from .camdemy import CamdemyIE from .canal13cl import Canal13clIE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py new file mode 100644 index 000000000..54959f2cf --- /dev/null +++ b/youtube_dl/extractor/camdemy.py @@ -0,0 +1,78 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..compat import compat_urlparse +from ..utils import parse_iso8601 + + +class CamdemyIE(InfoExtractor): + _VALID_URL = r'http://www.camdemy.com/media/(?P\d+).*' + _TESTS = [{ + # single file + 'url': 'http://www.camdemy.com/media/5181/', + 'md5': '5a5562b6a98b37873119102e052e311b', + 'info_dict': { + 'id': '5181', + 'ext': 'mp4', + 'title': 'Ch1-1 Introduction, Signals (02-23-2012)', + 'thumbnail': 're:^https?://.*\.jpg$', + 'description': '', + 'creator': 'ss11spring', + 'upload_date': '20130114', + 'timestamp': 1358154556, + } + }, { + # With non-empty description + 'url': 'http://www.camdemy.com/media/13885', + 'md5': '4576a3bb2581f86c61044822adbd1249', + 'info_dict': { + 'id': '13885', + 'ext': 'mp4', + 'title': 'EverCam + Camdemy QuickStart', + 'thumbnail': 're:^https?://.*\.jpg$', + 'description': 'md5:050b62f71ed62928f8a35f1a41e186c9', + 'creator': 'evercam', + 'upload_date': '20140620', + 'timestamp': 1403271569, + } + }] + + def _real_extract(self, url): + video_id = self._match_id(url) + + page = self._download_webpage(url, video_id) + + oembed_obj = self._download_json( + 'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id) + + thumb_url = oembed_obj['thumbnail_url'] + video_folder = compat_urlparse.urljoin(thumb_url, 'video/') + fileListXML = self._download_xml( + compat_urlparse.urljoin(video_folder, 'fileList.xml'), + video_id, 'Filelist XML') + fileName = fileListXML.find('./video/item/fileName').text + + creation_time = self._html_search_regex( + r"
Posted :
.*
([0-9:\- ]+)<", + page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00' + creation_timestamp = parse_iso8601(creation_time, delimiter=' ') + + view_count_str = self._html_search_regex( + r"
Views :
.*
([0-9,]+)<", + page, 'view count', flags=re.MULTILINE | re.DOTALL) + views = int(view_count_str.replace(',', '')) + + return { + 'id': video_id, + 'url': compat_urlparse.urljoin(video_folder, fileName), + 'title': oembed_obj['title'], + 'thumbnail': thumb_url, + 'description': self._html_search_meta('description', page), + 'creator': oembed_obj['author_name'], + 'duration': oembed_obj['duration'], + 'timestamp': creation_timestamp, + 'view_count': views, + } From 8367d3f3cbad7ddae5116c3cd5d39ebfed9711c5 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 12 Feb 2015 00:11:33 +0800 Subject: [PATCH 2/4] [camdemy] Detection of external sources --- youtube_dl/extractor/camdemy.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py index 54959f2cf..9a6341f0f 100644 --- a/youtube_dl/extractor/camdemy.py +++ b/youtube_dl/extractor/camdemy.py @@ -38,6 +38,19 @@ class CamdemyIE(InfoExtractor): 'upload_date': '20140620', 'timestamp': 1403271569, } + }, { + # External source + 'url': 'http://www.camdemy.com/media/14842', + 'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7', + 'info_dict': { + 'id': '2vsYQzNIsJo', + 'ext': 'mp4', + 'upload_date': '20130211', + 'uploader': 'Hun Kim', + 'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection', + 'uploader_id': 'hunkimtutorials', + 'title': 'Excel 2013 Tutorial - How to add Password Protection', + } }] def _real_extract(self, url): @@ -45,6 +58,13 @@ class CamdemyIE(InfoExtractor): page = self._download_webpage(url, video_id) + srcFrom = self._html_search_regex( + r"
Source: Date: Thu, 12 Feb 2015 14:13:19 +0800 Subject: [PATCH 3/4] [camdemy] Add support for folders --- youtube_dl/extractor/__init__.py | 5 ++- youtube_dl/extractor/camdemy.py | 68 +++++++++++++++++++++++++++----- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 496e4d0b7..095feabf9 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -49,7 +49,10 @@ from .brightcove import BrightcoveIE from .buzzfeed import BuzzFeedIE from .byutv import BYUtvIE from .c56 import C56IE -from .camdemy import CamdemyIE +from .camdemy import ( + CamdemyIE, + CamdemyFolderIE +) from .canal13cl import Canal13clIE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py index 9a6341f0f..0d64b5a16 100644 --- a/youtube_dl/extractor/camdemy.py +++ b/youtube_dl/extractor/camdemy.py @@ -4,12 +4,12 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urlparse +from ..compat import compat_urllib_parse from ..utils import parse_iso8601 class CamdemyIE(InfoExtractor): - _VALID_URL = r'http://www.camdemy.com/media/(?P\d+).*' + _VALID_URL = r'http://www.camdemy.com/media/(?P\d+)' _TESTS = [{ # single file 'url': 'http://www.camdemy.com/media/5181/', @@ -69,25 +69,25 @@ class CamdemyIE(InfoExtractor): 'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id) thumb_url = oembed_obj['thumbnail_url'] - video_folder = compat_urlparse.urljoin(thumb_url, 'video/') + video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/') fileListXML = self._download_xml( - compat_urlparse.urljoin(video_folder, 'fileList.xml'), + compat_urllib_parse.urljoin(video_folder, 'fileList.xml'), video_id, 'Filelist XML') fileName = fileListXML.find('./video/item/fileName').text creation_time = self._html_search_regex( - r"
Posted :
.*
([0-9:\- ]+)<", - page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00' + r"
Posted :
[\r\n ]*
([^<>]+)<", + page, 'creation time', flags=re.MULTILINE) + '+08:00' creation_timestamp = parse_iso8601(creation_time, delimiter=' ') view_count_str = self._html_search_regex( - r"
Views :
.*
([0-9,]+)<", - page, 'view count', flags=re.MULTILINE | re.DOTALL) + r"
Views :
[\r\n ]*
([^<>]+)<", + page, 'view count', flags=re.MULTILINE) views = int(view_count_str.replace(',', '')) return { 'id': video_id, - 'url': compat_urlparse.urljoin(video_folder, fileName), + 'url': compat_urllib_parse.urljoin(video_folder, fileName), 'title': oembed_obj['title'], 'thumbnail': thumb_url, 'description': self._html_search_meta('description', page), @@ -96,3 +96,53 @@ class CamdemyIE(InfoExtractor): 'timestamp': creation_timestamp, 'view_count': views, } + + +class CamdemyFolderIE(InfoExtractor): + _VALID_URL = r'http://www.camdemy.com/folder/(?P\d+)' + _TESTS = [{ + # links with trailing slash + 'url': 'http://www.camdemy.com/folder/450', + 'info_dict': { + 'id': '450', + 'title': '信號與系統 2012 & 2011 (Signals and Systems)', + }, + 'playlist_mincount': 145 + }, { + # links without trailing slash + # and multi-page + 'url': 'http://www.camdemy.com/folder/853', + 'info_dict': { + 'id': '853', + 'title': '科學計算 - 使用 Matlab' + }, + 'playlist_mincount': 20 + }, { + # with displayMode parameter. For testing the codes to add parameters + 'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg', + 'info_dict': { + 'id': '853', + 'title': '科學計算 - 使用 Matlab' + }, + 'playlist_mincount': 20 + }] + + def _real_extract(self, url): + folder_id = self._match_id(url) + + # Add displayMode=list so that all links are displayed in a single page + parsed_url = list(compat_urllib_parse.urlparse(url)) + query = dict(compat_urllib_parse.parse_qsl(parsed_url[4])) + query.update({'displayMode': 'list'}) + parsed_url[4] = compat_urllib_parse.urlencode(query) + final_url = compat_urllib_parse.urlunparse(parsed_url) + + page = self._download_webpage(final_url, folder_id) + matches = re.findall(r"href='(/media/\d+/?)'", page) + + entries = [self.url_result('http://www.camdemy.com' + media_path) + for media_path in matches] + + folder_title = self._html_search_meta('keywords', page) + + return self.playlist_result(entries, folder_id, folder_title) From 7e6011101f00e4f7216467e80091cc89c2846f01 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Thu, 12 Feb 2015 14:23:25 +0800 Subject: [PATCH 4/4] [camdemy] Python2 compatibility --- youtube_dl/extractor/camdemy.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py index 0d64b5a16..1bc602c31 100644 --- a/youtube_dl/extractor/camdemy.py +++ b/youtube_dl/extractor/camdemy.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urllib_parse +from ..compat import (compat_urllib_parse, compat_urlparse) from ..utils import parse_iso8601 @@ -69,9 +69,9 @@ class CamdemyIE(InfoExtractor): 'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id) thumb_url = oembed_obj['thumbnail_url'] - video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/') + video_folder = compat_urlparse.urljoin(thumb_url, 'video/') fileListXML = self._download_xml( - compat_urllib_parse.urljoin(video_folder, 'fileList.xml'), + compat_urlparse.urljoin(video_folder, 'fileList.xml'), video_id, 'Filelist XML') fileName = fileListXML.find('./video/item/fileName').text @@ -87,7 +87,7 @@ class CamdemyIE(InfoExtractor): return { 'id': video_id, - 'url': compat_urllib_parse.urljoin(video_folder, fileName), + 'url': compat_urlparse.urljoin(video_folder, fileName), 'title': oembed_obj['title'], 'thumbnail': thumb_url, 'description': self._html_search_meta('description', page), @@ -131,11 +131,11 @@ class CamdemyFolderIE(InfoExtractor): folder_id = self._match_id(url) # Add displayMode=list so that all links are displayed in a single page - parsed_url = list(compat_urllib_parse.urlparse(url)) - query = dict(compat_urllib_parse.parse_qsl(parsed_url[4])) + parsed_url = list(compat_urlparse.urlparse(url)) + query = dict(compat_urlparse.parse_qsl(parsed_url[4])) query.update({'displayMode': 'list'}) parsed_url[4] = compat_urllib_parse.urlencode(query) - final_url = compat_urllib_parse.urlunparse(parsed_url) + final_url = compat_urlparse.urlunparse(parsed_url) page = self._download_webpage(final_url, folder_id) matches = re.findall(r"href='(/media/\d+/?)'", page)