PageRenderTime 35ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 0ms

/yt_dlp/extractor/lecturio.py

https://gitlab.com/vitalii.dr/yt-dlp
Python | 236 lines | 203 code | 28 blank | 5 comment | 36 complexity | 2302eddbc99774374f8b5174fae112da MD5 | raw file
  1. # coding: utf-8
  2. from __future__ import unicode_literals
  3. import re
  4. from .common import InfoExtractor
  5. from ..utils import (
  6. clean_html,
  7. determine_ext,
  8. ExtractorError,
  9. float_or_none,
  10. int_or_none,
  11. str_or_none,
  12. url_or_none,
  13. urlencode_postdata,
  14. urljoin,
  15. )
  16. class LecturioBaseIE(InfoExtractor):
  17. _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
  18. _LOGIN_URL = 'https://app.lecturio.com/en/login'
  19. _NETRC_MACHINE = 'lecturio'
  20. def _perform_login(self, username, password):
  21. # Sets some cookies
  22. _, urlh = self._download_webpage_handle(
  23. self._LOGIN_URL, None, 'Downloading login popup')
  24. def is_logged(url_handle):
  25. return self._LOGIN_URL not in url_handle.geturl()
  26. # Already logged in
  27. if is_logged(urlh):
  28. return
  29. login_form = {
  30. 'signin[email]': username,
  31. 'signin[password]': password,
  32. 'signin[remember]': 'on',
  33. }
  34. response, urlh = self._download_webpage_handle(
  35. self._LOGIN_URL, None, 'Logging in',
  36. data=urlencode_postdata(login_form))
  37. # Logged in successfully
  38. if is_logged(urlh):
  39. return
  40. errors = self._html_search_regex(
  41. r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
  42. 'errors', default=None)
  43. if errors:
  44. raise ExtractorError('Unable to login: %s' % errors, expected=True)
  45. raise ExtractorError('Unable to log in')
  46. class LecturioIE(LecturioBaseIE):
  47. _VALID_URL = r'''(?x)
  48. https://
  49. (?:
  50. app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
  51. (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
  52. )
  53. '''
  54. _TESTS = [{
  55. 'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
  56. 'md5': '9a42cf1d8282a6311bf7211bbde26fde',
  57. 'info_dict': {
  58. 'id': '39634',
  59. 'ext': 'mp4',
  60. 'title': 'Important Concepts and Terms — Introduction to Microbiology',
  61. },
  62. 'skip': 'Requires lecturio account credentials',
  63. }, {
  64. 'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
  65. 'only_matching': True,
  66. }, {
  67. 'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
  68. 'only_matching': True,
  69. }]
  70. _CC_LANGS = {
  71. 'Arabic': 'ar',
  72. 'Bulgarian': 'bg',
  73. 'German': 'de',
  74. 'English': 'en',
  75. 'Spanish': 'es',
  76. 'Persian': 'fa',
  77. 'French': 'fr',
  78. 'Japanese': 'ja',
  79. 'Polish': 'pl',
  80. 'Pashto': 'ps',
  81. 'Russian': 'ru',
  82. }
  83. def _real_extract(self, url):
  84. mobj = self._match_valid_url(url)
  85. nt = mobj.group('nt') or mobj.group('nt_de')
  86. lecture_id = mobj.group('id')
  87. display_id = nt or lecture_id
  88. api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
  89. video = self._download_json(
  90. self._API_BASE_URL + api_path, display_id)
  91. title = video['title'].strip()
  92. if not lecture_id:
  93. pid = video.get('productId') or video.get('uid')
  94. if pid:
  95. spid = pid.split('_')
  96. if spid and len(spid) == 2:
  97. lecture_id = spid[1]
  98. formats = []
  99. for format_ in video['content']['media']:
  100. if not isinstance(format_, dict):
  101. continue
  102. file_ = format_.get('file')
  103. if not file_:
  104. continue
  105. ext = determine_ext(file_)
  106. if ext == 'smil':
  107. # smil contains only broken RTMP formats anyway
  108. continue
  109. file_url = url_or_none(file_)
  110. if not file_url:
  111. continue
  112. label = str_or_none(format_.get('label'))
  113. filesize = int_or_none(format_.get('fileSize'))
  114. f = {
  115. 'url': file_url,
  116. 'format_id': label,
  117. 'filesize': float_or_none(filesize, invscale=1000)
  118. }
  119. if label:
  120. mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
  121. if mobj:
  122. f.update({
  123. 'format_id': mobj.group(2),
  124. 'height': int(mobj.group(1)),
  125. })
  126. formats.append(f)
  127. self._sort_formats(formats)
  128. subtitles = {}
  129. automatic_captions = {}
  130. captions = video.get('captions') or []
  131. for cc in captions:
  132. cc_url = cc.get('url')
  133. if not cc_url:
  134. continue
  135. cc_label = cc.get('translatedCode')
  136. lang = cc.get('languageCode') or self._search_regex(
  137. r'/([a-z]{2})_', cc_url, 'lang',
  138. default=cc_label.split()[0] if cc_label else 'en')
  139. original_lang = self._search_regex(
  140. r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
  141. default=None)
  142. sub_dict = (automatic_captions
  143. if 'auto-translated' in cc_label or original_lang
  144. else subtitles)
  145. sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
  146. 'url': cc_url,
  147. })
  148. return {
  149. 'id': lecture_id or nt,
  150. 'title': title,
  151. 'formats': formats,
  152. 'subtitles': subtitles,
  153. 'automatic_captions': automatic_captions,
  154. }
  155. class LecturioCourseIE(LecturioBaseIE):
  156. _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
  157. _TESTS = [{
  158. 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
  159. 'info_dict': {
  160. 'id': 'microbiology-introduction',
  161. 'title': 'Microbiology: Introduction',
  162. 'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
  163. },
  164. 'playlist_count': 45,
  165. 'skip': 'Requires lecturio account credentials',
  166. }, {
  167. 'url': 'https://app.lecturio.com/#/course/c/6434',
  168. 'only_matching': True,
  169. }]
  170. def _real_extract(self, url):
  171. nt, course_id = self._match_valid_url(url).groups()
  172. display_id = nt or course_id
  173. api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
  174. course = self._download_json(
  175. self._API_BASE_URL + api_path, display_id)
  176. entries = []
  177. for lecture in course.get('lectures', []):
  178. lecture_id = str_or_none(lecture.get('id'))
  179. lecture_url = lecture.get('url')
  180. if lecture_url:
  181. lecture_url = urljoin(url, lecture_url)
  182. else:
  183. lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
  184. entries.append(self.url_result(
  185. lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
  186. return self.playlist_result(
  187. entries, display_id, course.get('title'),
  188. clean_html(course.get('description')))
  189. class LecturioDeCourseIE(LecturioBaseIE):
  190. _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
  191. _TEST = {
  192. 'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
  193. 'only_matching': True,
  194. }
  195. def _real_extract(self, url):
  196. display_id = self._match_id(url)
  197. webpage = self._download_webpage(url, display_id)
  198. entries = []
  199. for mobj in re.finditer(
  200. r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
  201. webpage):
  202. lecture_url = urljoin(url, mobj.group('url'))
  203. lecture_id = mobj.group('id')
  204. entries.append(self.url_result(
  205. lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
  206. title = self._search_regex(
  207. r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
  208. return self.playlist_result(entries, display_id, title)