PageRenderTime 64ms CodeModel.GetById 31ms RepoModel.GetById 0ms app.codeStats 1ms

/youtube_dl/extractor/safari.py

https://gitlab.com/angelbirth/youtube-dl
Python | 192 lines | 155 code | 34 blank | 3 comment | 10 complexity | 0f450f52cba32b9021f745da8830712d MD5 | raw file
  1. # encoding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. from .common import InfoExtractor
  5. from ..utils import (
  6. ExtractorError,
  7. sanitized_Request,
  8. std_headers,
  9. urlencode_postdata,
  10. update_url_query,
  11. )
  12. class SafariBaseIE(InfoExtractor):
  13. _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
  14. _SUCCESSFUL_LOGIN_REGEX = r'<a href="/accounts/logout/"[^>]*>Sign Out</a>'
  15. _NETRC_MACHINE = 'safari'
  16. _API_BASE = 'https://www.safaribooksonline.com/api/v1'
  17. _API_FORMAT = 'json'
  18. LOGGED_IN = False
  19. def _real_initialize(self):
  20. self._login()
  21. def _login(self):
  22. # We only need to log in once for courses or individual videos
  23. if self.LOGGED_IN:
  24. return
  25. (username, password) = self._get_login_info()
  26. if username is None:
  27. return
  28. headers = std_headers.copy()
  29. if 'Referer' not in headers:
  30. headers['Referer'] = self._LOGIN_URL
  31. login_page_request = sanitized_Request(self._LOGIN_URL, headers=headers)
  32. login_page = self._download_webpage(
  33. login_page_request, None,
  34. 'Downloading login form')
  35. csrf = self._html_search_regex(
  36. r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
  37. login_page, 'csrf token')
  38. login_form = {
  39. 'csrfmiddlewaretoken': csrf,
  40. 'email': username,
  41. 'password1': password,
  42. 'login': 'Sign In',
  43. 'next': '',
  44. }
  45. request = sanitized_Request(
  46. self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
  47. login_page = self._download_webpage(
  48. request, None, 'Logging in as %s' % username)
  49. if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
  50. raise ExtractorError(
  51. 'Login failed; make sure your credentials are correct and try again.',
  52. expected=True)
  53. SafariBaseIE.LOGGED_IN = True
  54. self.to_screen('Login successful')
  55. class SafariIE(SafariBaseIE):
  56. IE_NAME = 'safari'
  57. IE_DESC = 'safaribooksonline.com online video'
  58. _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?#&]+)\.html'
  59. _TESTS = [{
  60. 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
  61. 'md5': 'dcc5a425e79f2564148652616af1f2a3',
  62. 'info_dict': {
  63. 'id': '0_qbqx90ic',
  64. 'ext': 'mp4',
  65. 'title': 'Introduction to Hadoop Fundamentals LiveLessons',
  66. 'timestamp': 1437758058,
  67. 'upload_date': '20150724',
  68. 'uploader_id': 'stork',
  69. },
  70. }, {
  71. # non-digits in course id
  72. 'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
  73. 'only_matching': True,
  74. }, {
  75. 'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
  76. 'only_matching': True,
  77. }]
  78. def _real_extract(self, url):
  79. mobj = re.match(self._VALID_URL, url)
  80. video_id = '%s/%s' % (mobj.group('course_id'), mobj.group('part'))
  81. webpage = self._download_webpage(url, video_id)
  82. reference_id = self._search_regex(
  83. r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  84. webpage, 'kaltura reference id', group='id')
  85. partner_id = self._search_regex(
  86. r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  87. webpage, 'kaltura widget id', group='id')
  88. ui_id = self._search_regex(
  89. r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
  90. webpage, 'kaltura uiconf id', group='id')
  91. query = {
  92. 'wid': '_%s' % partner_id,
  93. 'uiconf_id': ui_id,
  94. 'flashvars[referenceId]': reference_id,
  95. }
  96. if self.LOGGED_IN:
  97. kaltura_session = self._download_json(
  98. '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
  99. video_id, 'Downloading kaltura session JSON',
  100. 'Unable to download kaltura session JSON', fatal=False)
  101. if kaltura_session:
  102. session = kaltura_session.get('session')
  103. if session:
  104. query['flashvars[ks]'] = session
  105. return self.url_result(update_url_query(
  106. 'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
  107. 'Kaltura')
  108. class SafariApiIE(SafariBaseIE):
  109. IE_NAME = 'safari:api'
  110. _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
  111. _TESTS = [{
  112. 'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
  113. 'only_matching': True,
  114. }, {
  115. 'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
  116. 'only_matching': True,
  117. }]
  118. def _real_extract(self, url):
  119. mobj = re.match(self._VALID_URL, url)
  120. part = self._download_json(
  121. url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
  122. 'Downloading part JSON')
  123. return self.url_result(part['web_url'], SafariIE.ie_key())
  124. class SafariCourseIE(SafariBaseIE):
  125. IE_NAME = 'safari:course'
  126. IE_DESC = 'safaribooksonline.com online courses'
  127. _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)/(?P<id>[^/]+)/?(?:[#?]|$)'
  128. _TESTS = [{
  129. 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
  130. 'info_dict': {
  131. 'id': '9780133392838',
  132. 'title': 'Hadoop Fundamentals LiveLessons',
  133. },
  134. 'playlist_count': 22,
  135. 'skip': 'Requires safaribooksonline account credentials',
  136. }, {
  137. 'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
  138. 'only_matching': True,
  139. }]
  140. def _real_extract(self, url):
  141. course_id = self._match_id(url)
  142. course_json = self._download_json(
  143. '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
  144. course_id, 'Downloading course JSON')
  145. if 'chapters' not in course_json:
  146. raise ExtractorError(
  147. 'No chapters found for course %s' % course_id, expected=True)
  148. entries = [
  149. self.url_result(chapter, SafariApiIE.ie_key())
  150. for chapter in course_json['chapters']]
  151. course_title = course_json['title']
  152. return self.playlist_result(entries, course_id, course_title)