From 83317f693870424c2c769e4d964453401063fdf1 Mon Sep 17 00:00:00 2001 From: riking Date: Sat, 16 Aug 2014 14:28:41 -0700 Subject: [PATCH 1/2] [youtube] Add two-factor account signin (TOTP only) Additional work is required to prompt the user for the SMS or phone call codes, as there is no framework currently to prompt the user during an extraction operation. Fixes #3533 --- youtube_dl/__init__.py | 3 ++ youtube_dl/extractor/common.py | 18 ++++++++ youtube_dl/extractor/youtube.py | 74 ++++++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 962aedbff..80de211e7 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -314,6 +314,8 @@ def parseOpts(overrideArguments=None): dest='username', metavar='USERNAME', help='account username') authentication.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password') + authentication.add_option('-2', '--twofactor', + dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code') authentication.add_option('-n', '--netrc', action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) authentication.add_option('--video-password', @@ -748,6 +750,7 @@ def _real_main(argv=None): 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, + 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'quiet': (opts.quiet or any_printing), 'no_warnings': opts.no_warnings, diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 342bfb8b3..9f6a8a889 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -434,6 +434,24 @@ class InfoExtractor(object): return (username, password) + def _get_tfa_info(self): + """ + Get the two-factor authentication info + TODO - asking the user will be required for sms/phone verify + currently just uses the command line option + If there's no info available, return None + """ + if self._downloader is None: + self.to_screen("no downloader") + return None + downloader_params = self._downloader.params + + if downloader_params.get('twofactor', None) is not None: + return downloader_params['twofactor'] + + self.to_screen("param is None") + return None + # Helper functions for extracting OpenGraph info @staticmethod def _og_regexes(prop): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 3c9b0b584..5bfe5e7e5 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -37,6 +37,7 @@ from ..utils import ( class YoutubeBaseInfoExtractor(InfoExtractor): """Provide base functions for Youtube extractors""" _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' + _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor' _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NETRC_MACHINE = 'youtube' @@ -50,12 +51,19 @@ class YoutubeBaseInfoExtractor(InfoExtractor): fatal=False)) def _login(self): + """ + Attempt to log in to YouTube. + True is returned if successful or skipped. + False is returned if login failed. + + If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised. + """ (username, password) = self._get_login_info() # No authentication to be performed if username is None: if self._LOGIN_REQUIRED: raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True) - return False + return True login_page = self._download_webpage( self._LOGIN_URL, None, @@ -73,6 +81,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): u'Email': username, u'GALX': galx, u'Passwd': password, + u'PersistentCookie': u'yes', u'_utf8': u'霱', u'bgresponse': u'js_disabled', @@ -88,6 +97,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): u'uilel': u'3', u'hl': u'en_US', } + # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode # chokes on unicode login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) @@ -99,6 +109,68 @@ class YoutubeBaseInfoExtractor(InfoExtractor): note=u'Logging in', errnote=u'unable to log in', fatal=False) if login_results is False: return False + + if re.search(r'id="errormsg_0_Passwd"', login_results) is not None: + raise ExtractorError(u'Please use your account password and a two-factor code instead of an application-specific password.', expected=True) + + # Two-Factor + # TODO add SMS and phone call support - these require making a request and then prompting the user + + if re.search(r'(?i)]* id="gaia_secondfactorform"', login_results) is not None: + tfa_code = self._get_tfa_info() + + if tfa_code is None: + self._downloader.report_warning(u'Two-factor authentication required. Provide it with --twofactor ') + self._downloader.report_warning(u'(Note that only TOTP (Google Authenticator App) codes work at this time.)') + return False + + # Unlike the first login form, secTok and timeStmp are both required for the TFA form + + match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U) + if match is None: + self._downloader.report_warning(u'Failed to get secTok - did the page structure change?') + secTok = match.group(1) + match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U) + if match is None: + self._downloader.report_warning(u'Failed to get timeStmp - did the page structure change?') + timeStmp = match.group(1) + + tfa_form_strs = { + u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1', + u'smsToken': u'', + u'smsUserPin': tfa_code, + u'smsVerifyPin': u'Verify', + + u'PersistentCookie': u'yes', + u'checkConnection': u'', + u'checkedDomains': u'youtube', + u'pstMsg': u'1', + u'secTok': secTok, + u'timeStmp': timeStmp, + u'service': u'youtube', + u'hl': u'en_US', + } + tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in tfa_form_strs.items()) + tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii') + + tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data) + tfa_results = self._download_webpage( + tfa_req, None, + note=u'Submitting TFA code', errnote=u'unable to submit tfa', fatal=False) + + if tfa_results is False: + return False + + if re.search(r'(?i)]* id="gaia_secondfactorform"', tfa_results) is not None: + self._downloader.report_warning(u'Two-factor code expired. Please try again, or use a one-use backup code instead.') + return False + if re.search(r'(?i)]* id="gaia_loginform"', tfa_results) is not None: + self._downloader.report_warning(u'unable to log in - did the page structure change?') + return False + if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None: + self._downloader.report_warning(u'Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.') + return False + if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: self._downloader.report_warning(u'unable to log in: bad username or password') return False From 165250ff5e33d1b246dffaade23b31b4fa3f3b02 Mon Sep 17 00:00:00 2001 From: riking Date: Sat, 16 Aug 2014 14:49:30 -0700 Subject: [PATCH 2/2] Remove debug prints --- youtube_dl/extractor/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 9f6a8a889..45a17f8ad 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -442,14 +442,12 @@ class InfoExtractor(object): If there's no info available, return None """ if self._downloader is None: - self.to_screen("no downloader") return None downloader_params = self._downloader.params if downloader_params.get('twofactor', None) is not None: return downloader_params['twofactor'] - self.to_screen("param is None") return None # Helper functions for extracting OpenGraph info