From 717b1f72ed6070212a72ac823547cb3c776a6264 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 17:20:25 +0100 Subject: [PATCH 1/7] default info_dict['format'] to info_dict['ext'] and make the YT one more verbose --- youtube_dl/FileDownloader.py | 3 ++ youtube_dl/InfoExtractors.py | 70 +++++++++++++++--------------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index b6aebe4ac..69d169904 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -355,6 +355,9 @@ class FileDownloader(object): # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] + if not 'format' in info_dict: + info_dict['format'] = info_dict['ext'] + reason = self._match_entry(info_dict) if reason is not None: self.to_screen(u'[download] ' + reason) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 13b04ab5b..44b2472c2 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -29,33 +29,34 @@ class InfoExtractor(object): """Information Extractor class. Information extractors are the classes that, given a URL, extract - information from the video (or videos) the URL refers to. This - information includes the real video URL, the video title and simplified - title, author and others. The information is stored in a dictionary - which is then passed to the FileDownloader. The FileDownloader - processes this information possibly downloading the video to the file - system, among other possible outcomes. The dictionaries must include - the following fields: - - id: Video identifier. - url: Final video URL. - uploader: Nickname of the video uploader. - title: Literal title. - ext: Video filename extension. - format: Video format. - player_url: SWF Player URL (may be None). - - The following fields are optional. Their primary purpose is to allow - youtube-dl to serve as the backend for a video search function, such - as the one in youtube2mp3. They are only used when their respective - forced printing functions are called: - - thumbnail: Full URL to a video thumbnail image. - description: One-line video description. + information about the video (or videos) the URL refers to. This + information includes the real video URL, the video title, author and + others. The information is stored in a dictionary which is then + passed to the FileDownloader. The FileDownloader processes this + information possibly downloading the video to the file system, among + other possible outcomes. + + The dictionaries must include the following fields: + + id: Video identifier. + url: Final video URL. + uploader: Nickname of the video uploader. + title: Video title, unescaped. + ext: Video filename extension. + player_url: SWF Player URL (may be None). + + The following fields are optional: + + format: The video format, defaults to ext. Used by --get-format + thumbnail: Full URL to a video thumbnail image. + description One-line video description. Subclasses of this one should re-define the _real_initialize() and _real_extract() methods and define a _VALID_URL regexp. Probably, they should also be added to the list of extractors. + + _real_extract() must return a *list* of information dictionaries as + described above. """ _ready = False @@ -475,6 +476,9 @@ class YoutubeIE(InfoExtractor): # Extension video_extension = self._video_extensions.get(format_param, 'flv') + video_format = '{} - {}'.format(format_param.decode('utf-8') if format_param else video_extension.decode('utf-8'), + self._video_dimensions.get(format_param, '???')) + results.append({ 'id': video_id.decode('utf-8'), 'url': video_real_url.decode('utf-8'), @@ -482,7 +486,7 @@ class YoutubeIE(InfoExtractor): 'upload_date': upload_date, 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), + 'format': video_format, 'thumbnail': video_thumbnail.decode('utf-8'), 'description': video_description, 'player_url': player_url, @@ -616,7 +620,6 @@ class MetacafeIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -715,7 +718,6 @@ class DailymotionIE(InfoExtractor): 'upload_date': video_upload_date, 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -810,7 +812,6 @@ class GoogleIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -877,7 +878,6 @@ class PhotobucketIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -1282,7 +1282,6 @@ class GenericIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -1888,7 +1887,6 @@ class DepositFilesIE(InfoExtractor): 'upload_date': u'NA', 'title': file_title, 'ext': file_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] @@ -2243,7 +2241,6 @@ class MyVideoIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': u'flv', - 'format': u'NA', 'player_url': None, }] @@ -2501,7 +2498,6 @@ class EscapistIE(InfoExtractor): 'upload_date': None, 'title': showName, 'ext': 'flv', - 'format': 'flv', 'thumbnail': imgUrl, 'description': description, 'player_url': playerUrl, @@ -2566,7 +2562,6 @@ class CollegeHumorIE(InfoExtractor): info['url'] = videoNode.findall('./file')[0].text info['thumbnail'] = videoNode.findall('./thumbnail')[0].text info['ext'] = info['url'].rpartition('.')[2] - info['format'] = info['ext'] except IndexError: self._downloader.trouble(u'\nERROR: Invalid metadata XML file') return @@ -2637,7 +2632,6 @@ class XVideosIE(InfoExtractor): 'upload_date': None, 'title': video_title, 'ext': 'flv', - 'format': 'flv', 'thumbnail': video_thumbnail, 'description': None, 'player_url': None, @@ -2734,7 +2728,6 @@ class SoundcloudIE(InfoExtractor): 'upload_date': upload_date, 'title': title, 'ext': u'mp3', - 'format': u'NA', 'player_url': None, 'description': description.decode('utf-8') }] @@ -2802,8 +2795,7 @@ class InfoQIE(InfoExtractor): 'uploader': None, 'upload_date': None, 'title': video_title, - 'ext': extension, - 'format': extension, # Extension is always(?) mp4, but seems to be flv + 'ext': extension, # Extension is always(?) mp4, but seems to be flv 'thumbnail': None, 'description': video_description, 'player_url': None, @@ -2967,7 +2959,6 @@ class StanfordOpenClassroomIE(InfoExtractor): self._downloader.trouble(u'\nERROR: Invalid metadata XML file') return info['ext'] = info['url'].rpartition('.')[2] - info['format'] = info['ext'] return [info] elif mobj.group('course'): # A course page course = mobj.group('course') @@ -3241,7 +3232,6 @@ class YoukuIE(InfoExtractor): 'uploader': None, 'title': video_title, 'ext': ext, - 'format': u'NA' } files_info.append(info) @@ -3305,7 +3295,6 @@ class XNXXIE(InfoExtractor): 'upload_date': None, 'title': video_title, 'ext': 'flv', - 'format': 'flv', 'thumbnail': video_thumbnail, 'description': None, 'player_url': None} @@ -3432,6 +3421,5 @@ class GooglePlusIE(InfoExtractor): 'upload_date': upload_date.decode('utf-8'), 'title': video_title.decode('utf-8'), 'ext': video_extension.decode('utf-8'), - 'format': u'NA', 'player_url': None, }] From abe7a3ac2a85b0323820270f0920d44682b5cd11 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 17:32:25 +0100 Subject: [PATCH 2/7] info_dict['player_url'] is used only for rtmpdump, indicate it as optional in the info_dict --- youtube_dl/InfoExtractors.py | 39 +++++++++++------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 44b2472c2..a39e865dc 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -43,13 +43,13 @@ class InfoExtractor(object): uploader: Nickname of the video uploader. title: Video title, unescaped. ext: Video filename extension. - player_url: SWF Player URL (may be None). The following fields are optional: - format: The video format, defaults to ext. Used by --get-format + format: The video format, defaults to ext (used for --get-format) thumbnail: Full URL to a video thumbnail image. description One-line video description. + player_url: SWF Player URL (used for rtmpdump). Subclasses of this one should re-define the _real_initialize() and _real_extract() methods and define a _VALID_URL regexp. @@ -620,7 +620,6 @@ class MetacafeIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] @@ -718,7 +717,6 @@ class DailymotionIE(InfoExtractor): 'upload_date': video_upload_date, 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] @@ -812,7 +810,6 @@ class GoogleIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] @@ -878,7 +875,6 @@ class PhotobucketIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] @@ -1022,7 +1018,6 @@ class YahooIE(InfoExtractor): 'thumbnail': video_thumbnail.decode('utf-8'), 'description': video_description, 'thumbnail': video_thumbnail, - 'player_url': None, }] @@ -1136,7 +1131,6 @@ class VimeoIE(InfoExtractor): 'ext': video_extension, 'thumbnail': video_thumbnail, 'description': video_description, - 'player_url': None, }] @@ -1282,7 +1276,6 @@ class GenericIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] @@ -1887,7 +1880,6 @@ class DepositFilesIE(InfoExtractor): 'upload_date': u'NA', 'title': file_title, 'ext': file_extension.decode('utf-8'), - 'player_url': None, }] @@ -2093,7 +2085,6 @@ class FacebookIE(InfoExtractor): 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), 'thumbnail': video_thumbnail.decode('utf-8'), 'description': video_description.decode('utf-8'), - 'player_url': None, }) return results @@ -2241,7 +2232,6 @@ class MyVideoIE(InfoExtractor): 'upload_date': u'NA', 'title': video_title, 'ext': u'flv', - 'player_url': None, }] class ComedyCentralIE(InfoExtractor): @@ -2634,7 +2624,6 @@ class XVideosIE(InfoExtractor): 'ext': 'flv', 'thumbnail': video_thumbnail, 'description': None, - 'player_url': None, } return [info] @@ -2728,7 +2717,6 @@ class SoundcloudIE(InfoExtractor): 'upload_date': upload_date, 'title': title, 'ext': u'mp3', - 'player_url': None, 'description': description.decode('utf-8') }] @@ -2798,7 +2786,6 @@ class InfoQIE(InfoExtractor): 'ext': extension, # Extension is always(?) mp4, but seems to be flv 'thumbnail': None, 'description': video_description, - 'player_url': None, } return [info] @@ -3289,17 +3276,16 @@ class XNXXIE(InfoExtractor): return video_thumbnail = result.group(1).decode('utf-8') - info = {'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'ext': 'flv', - 'thumbnail': video_thumbnail, - 'description': None, - 'player_url': None} - - return [info] + return [{ + 'id': video_id, + 'url': video_url, + 'uploader': None, + 'upload_date': None, + 'title': video_title, + 'ext': 'flv', + 'thumbnail': video_thumbnail, + 'description': None, + }] class GooglePlusIE(InfoExtractor): @@ -3421,5 +3407,4 @@ class GooglePlusIE(InfoExtractor): 'upload_date': upload_date.decode('utf-8'), 'title': video_title.decode('utf-8'), 'ext': video_extension.decode('utf-8'), - 'player_url': None, }] From b49e75ff9a423eca11d71e07daa69150d3066288 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 17:38:22 +0100 Subject: [PATCH 3/7] info_dict['upload_date'] is documented in --output, IEs MUST specify it --- youtube_dl/InfoExtractors.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index a39e865dc..8ea03ea01 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -38,17 +38,18 @@ class InfoExtractor(object): The dictionaries must include the following fields: - id: Video identifier. - url: Final video URL. - uploader: Nickname of the video uploader. - title: Video title, unescaped. - ext: Video filename extension. + id: Video identifier. + url: Final video URL. + uploader: Nickname of the video uploader. + upload_date: Video upload date (YYYYMMDD). + title: Video title, unescaped. + ext: Video filename extension. The following fields are optional: format: The video format, defaults to ext (used for --get-format) thumbnail: Full URL to a video thumbnail image. - description One-line video description. + description: One-line video description. player_url: SWF Player URL (used for rtmpdump). Subclasses of this one should re-define the _real_initialize() and From 9ce5d9ee7513c433cbbb8949214ada078ffddea6 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 17:57:12 +0100 Subject: [PATCH 4/7] make all IEs return 'upload_date' and 'uploader', even if only u'NA' --- youtube_dl/InfoExtractors.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 8ea03ea01..3adf56a0b 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -40,7 +40,7 @@ class InfoExtractor(object): id: Video identifier. url: Final video URL. - uploader: Nickname of the video uploader. + uploader: Nickname of the video uploader, unescaped. upload_date: Video upload date (YYYYMMDD). title: Video title, unescaped. ext: Video filename extension. @@ -52,6 +52,8 @@ class InfoExtractor(object): description: One-line video description. player_url: SWF Player URL (used for rtmpdump). + The fields should all be Unicode strings. + Subclasses of this one should re-define the _real_initialize() and _real_extract() methods and define a _VALID_URL regexp. Probably, they should also be added to the list of extractors. @@ -1018,7 +1020,6 @@ class YahooIE(InfoExtractor): 'ext': video_extension.decode('utf-8'), 'thumbnail': video_thumbnail.decode('utf-8'), 'description': video_description, - 'thumbnail': video_thumbnail, }] @@ -2129,6 +2130,8 @@ class BlipTVIE(InfoExtractor): info = { 'id': title, 'url': url, + 'uploader': u'NA', + 'upload_date': u'NA', 'title': title, 'ext': ext, 'urlhandle': urlh @@ -2486,7 +2489,7 @@ class EscapistIE(InfoExtractor): 'id': videoId, 'url': videoUrl, 'uploader': showName, - 'upload_date': None, + 'upload_date': u'NA', 'title': showName, 'ext': 'flv', 'thumbnail': imgUrl, @@ -2535,6 +2538,8 @@ class CollegeHumorIE(InfoExtractor): info = { 'id': video_id, 'internal_id': internal_video_id, + 'uploader': u'NA', + 'upload_date': u'NA', } self.report_extraction(video_id) @@ -2619,8 +2624,8 @@ class XVideosIE(InfoExtractor): info = { 'id': video_id, 'url': video_url, - 'uploader': None, - 'upload_date': None, + 'uploader': u'NA', + 'upload_date': u'NA', 'title': video_title, 'ext': 'flv', 'thumbnail': video_thumbnail, @@ -2700,7 +2705,7 @@ class SoundcloudIE(InfoExtractor): description = mobj.group(1) # upload date - upload_date = None + upload_date = u'NA' mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage) if mobj: try: @@ -2781,8 +2786,8 @@ class InfoQIE(InfoExtractor): info = { 'id': video_id, 'url': video_url, - 'uploader': None, - 'upload_date': None, + 'uploader': u'NA', + 'upload_date': u'NA', 'title': video_title, 'ext': extension, # Extension is always(?) mp4, but seems to be flv 'thumbnail': None, @@ -2929,6 +2934,8 @@ class StanfordOpenClassroomIE(InfoExtractor): video = mobj.group('video') info = { 'id': course + '_' + video, + 'uploader': u'NA', + 'upload_date': u'NA', } self.report_extraction(info['id']) @@ -2953,6 +2960,8 @@ class StanfordOpenClassroomIE(InfoExtractor): info = { 'id': course, 'type': 'playlist', + 'uploader': u'NA', + 'upload_date': u'NA', } self.report_download_webpage(info['id']) @@ -2989,6 +2998,8 @@ class StanfordOpenClassroomIE(InfoExtractor): info = { 'id': 'Stanford OpenClassroom', 'type': 'playlist', + 'uploader': u'NA', + 'upload_date': u'NA', } self.report_download_webpage(info['id']) @@ -3097,6 +3108,7 @@ class MTVIE(InfoExtractor): 'id': video_id, 'url': video_url, 'uploader': performer, + 'upload_date': u'NA', 'title': video_title, 'ext': ext, 'format': format, @@ -3217,7 +3229,8 @@ class YoukuIE(InfoExtractor): info = { 'id': '%s_part%02d' % (video_id, index), 'url': download_url, - 'uploader': None, + 'uploader': u'NA', + 'upload_date': u'NA', 'title': video_title, 'ext': ext, } @@ -3280,8 +3293,8 @@ class XNXXIE(InfoExtractor): return [{ 'id': video_id, 'url': video_url, - 'uploader': None, - 'upload_date': None, + 'uploader': u'NA', + 'upload_date': u'NA', 'title': video_title, 'ext': 'flv', 'thumbnail': video_thumbnail, From 7e4674830ed0d85764792b1cd5605813643932b6 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 18:08:07 +0100 Subject: [PATCH 5/7] document info_dict['subtitles'] and info_dict['urlhandle'] --- youtube_dl/InfoExtractors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 3adf56a0b..4c664ce6e 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -51,6 +51,9 @@ class InfoExtractor(object): thumbnail: Full URL to a video thumbnail image. description: One-line video description. player_url: SWF Player URL (used for rtmpdump). + subtitles: The .srt file contents. + urlhandle: [internal] The urlHandle to be used to download the file, + like returned by urllib2.urlopen The fields should all be Unicode strings. From 03c5b0fbd4de40ebf21a7ba2e389fb5fba50666c Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 19:30:09 +0100 Subject: [PATCH 6/7] IE._WORKING attribute in order to warn the users and skip the tests on broken IEs --- youtube_dl/FileDownloader.py | 5 +++++ youtube_dl/InfoExtractors.py | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 69d169904..1fdd2071f 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -481,6 +481,11 @@ class FileDownloader(object): if not ie.suitable(url): continue + # Warn if the _WORKING attribute is False + if not ie.working(): + self.trouble(u'WARNING: the program functionality for this site has been marked as broken, ' + u'and will probably not work. If you want to go on, use the -i option.') + # Suitable InfoExtractor found suitable_found = True diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 4c664ce6e..7c6e1becc 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -63,10 +63,14 @@ class InfoExtractor(object): _real_extract() must return a *list* of information dictionaries as described above. + + Finally, the _WORKING attribute should be set to False for broken IEs + in order to warn the users and skip the tests. """ _ready = False _downloader = None + _WORKING = True def __init__(self, downloader=None): """Constructor. Receives an optional downloader.""" @@ -77,6 +81,10 @@ class InfoExtractor(object): """Receives a URL and returns True if suitable for this IE.""" return re.match(self._VALID_URL, url) is not None + def working(self): + """Getter method for _WORKING.""" + return self._WORKING + def initialize(self): """Initializes an instance (authentication, etc).""" if not self._ready: @@ -1891,6 +1899,7 @@ class DepositFilesIE(InfoExtractor): class FacebookIE(InfoExtractor): """Information Extractor for Facebook""" + _WORKING = False _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' _NETRC_MACHINE = 'facebook' From f462df021a881c406afefc62bbf905ea582700ea Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 27 Nov 2012 23:15:33 +0100 Subject: [PATCH 7/7] Use None on missing required info_dict fields --- youtube_dl/FileDownloader.py | 1 + youtube_dl/InfoExtractors.py | 78 ++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 1fdd2071f..a7997c4f2 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -327,6 +327,7 @@ class FileDownloader(object): template_dict = dict(info_dict) template_dict['epoch'] = unicode(int(time.time())) template_dict['autonumber'] = unicode('%05d' % self._num_downloads) + template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items()) filename = self.params['outtmpl'] % template_dict return filename except (ValueError, KeyError), err: diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 7c6e1becc..b730b5773 100644 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -380,7 +380,7 @@ class YoutubeIE(InfoExtractor): video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0]) # upload date - upload_date = u'NA' + upload_date = None mobj = re.search(r'id="eow-date.*?>(.*?)', video_webpage, re.DOTALL) if mobj is not None: upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) @@ -631,7 +631,7 @@ class MetacafeIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), - 'upload_date': u'NA', + 'upload_date': None, 'title': video_title, 'ext': video_extension.decode('utf-8'), }] @@ -707,7 +707,7 @@ class DailymotionIE(InfoExtractor): return video_title = unescapeHTML(mobj.group('title').decode('utf-8')) - video_uploader = u'NA' + video_uploader = None mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) if mobj is None: # lookin for official user @@ -719,7 +719,7 @@ class DailymotionIE(InfoExtractor): else: video_uploader = mobj.group(1) - video_upload_date = u'NA' + video_upload_date = None mobj = re.search(r'
([0-9]{2})-([0-9]{2})-([0-9]{4})
', webpage) if mobj is not None: video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) @@ -820,8 +820,8 @@ class GoogleIE(InfoExtractor): return [{ 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': video_extension.decode('utf-8'), }] @@ -886,7 +886,7 @@ class PhotobucketIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, - 'upload_date': u'NA', + 'upload_date': None, 'title': video_title, 'ext': video_extension.decode('utf-8'), }] @@ -1026,7 +1026,7 @@ class YahooIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url, 'uploader': video_uploader, - 'upload_date': u'NA', + 'upload_date': None, 'title': video_title, 'ext': video_extension.decode('utf-8'), 'thumbnail': video_thumbnail.decode('utf-8'), @@ -1098,7 +1098,7 @@ class VimeoIE(InfoExtractor): else: video_description = '' # Extract upload date - video_upload_date = u'NA' + video_upload_date = None mobj = re.search(r'', webpage) if mobj is not None: video_upload_date = mobj.group(1) @@ -1286,7 +1286,7 @@ class GenericIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, - 'upload_date': u'NA', + 'upload_date': None, 'title': video_title, 'ext': video_extension.decode('utf-8'), }] @@ -1889,8 +1889,8 @@ class DepositFilesIE(InfoExtractor): return [{ 'id': file_id.decode('utf-8'), 'url': file_url.decode('utf-8'), - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': file_title, 'ext': file_extension.decode('utf-8'), }] @@ -2044,7 +2044,7 @@ class FacebookIE(InfoExtractor): video_thumbnail = video_info['thumbnail'] # upload date - upload_date = u'NA' + upload_date = None if 'upload_date' in video_info: upload_time = video_info['upload_date'] timetuple = email.utils.parsedate_tz(upload_time) @@ -2142,8 +2142,8 @@ class BlipTVIE(InfoExtractor): info = { 'id': title, 'url': url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': title, 'ext': ext, 'urlhandle': urlh @@ -2244,8 +2244,8 @@ class MyVideoIE(InfoExtractor): return [{ 'id': video_id, 'url': video_url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': u'flv', }] @@ -2501,7 +2501,7 @@ class EscapistIE(InfoExtractor): 'id': videoId, 'url': videoUrl, 'uploader': showName, - 'upload_date': u'NA', + 'upload_date': None, 'title': showName, 'ext': 'flv', 'thumbnail': imgUrl, @@ -2550,8 +2550,8 @@ class CollegeHumorIE(InfoExtractor): info = { 'id': video_id, 'internal_id': internal_video_id, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, } self.report_extraction(video_id) @@ -2636,8 +2636,8 @@ class XVideosIE(InfoExtractor): info = { 'id': video_id, 'url': video_url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': 'flv', 'thumbnail': video_thumbnail, @@ -2717,7 +2717,7 @@ class SoundcloudIE(InfoExtractor): description = mobj.group(1) # upload date - upload_date = u'NA' + upload_date = None mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)", webpage) if mobj: try: @@ -2798,8 +2798,8 @@ class InfoQIE(InfoExtractor): info = { 'id': video_id, 'url': video_url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': extension, # Extension is always(?) mp4, but seems to be flv 'thumbnail': None, @@ -2912,7 +2912,7 @@ class MixcloudIE(InfoExtractor): 'id': file_id.decode('utf-8'), 'url': file_url.decode('utf-8'), 'uploader': uploader.decode('utf-8'), - 'upload_date': u'NA', + 'upload_date': None, 'title': json_data['name'], 'ext': file_url.split('.')[-1].decode('utf-8'), 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), @@ -2946,8 +2946,8 @@ class StanfordOpenClassroomIE(InfoExtractor): video = mobj.group('video') info = { 'id': course + '_' + video, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, } self.report_extraction(info['id']) @@ -2972,8 +2972,8 @@ class StanfordOpenClassroomIE(InfoExtractor): info = { 'id': course, 'type': 'playlist', - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, } self.report_download_webpage(info['id']) @@ -3010,8 +3010,8 @@ class StanfordOpenClassroomIE(InfoExtractor): info = { 'id': 'Stanford OpenClassroom', 'type': 'playlist', - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, } self.report_download_webpage(info['id']) @@ -3120,7 +3120,7 @@ class MTVIE(InfoExtractor): 'id': video_id, 'url': video_url, 'uploader': performer, - 'upload_date': u'NA', + 'upload_date': None, 'title': video_title, 'ext': ext, 'format': format, @@ -3241,8 +3241,8 @@ class YoukuIE(InfoExtractor): info = { 'id': '%s_part%02d' % (video_id, index), 'url': download_url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': ext, } @@ -3305,8 +3305,8 @@ class XNXXIE(InfoExtractor): return [{ 'id': video_id, 'url': video_url, - 'uploader': u'NA', - 'upload_date': u'NA', + 'uploader': None, + 'upload_date': None, 'title': video_title, 'ext': 'flv', 'thumbnail': video_thumbnail, @@ -3365,7 +3365,7 @@ class GooglePlusIE(InfoExtractor): return # Extract update date - upload_date = u'NA' + upload_date = None pattern = 'title="Timestamp">(.*?)' mobj = re.search(pattern, webpage) if mobj: @@ -3376,7 +3376,7 @@ class GooglePlusIE(InfoExtractor): self.report_date(upload_date) # Extract uploader - uploader = u'NA' + uploader = None pattern = r'rel\="author".*?>(.*?)' mobj = re.search(pattern, webpage) if mobj: