/yt_dlp/extractor/lecturio.py
Python | 236 lines | 203 code | 28 blank | 5 comment | 36 complexity | 2302eddbc99774374f8b5174fae112da MD5 | raw file
1# coding: utf-8
2from __future__ import unicode_literals
3
4import re
5
6from .common import InfoExtractor
7from ..utils import (
8 clean_html,
9 determine_ext,
10 ExtractorError,
11 float_or_none,
12 int_or_none,
13 str_or_none,
14 url_or_none,
15 urlencode_postdata,
16 urljoin,
17)
18
19
20class LecturioBaseIE(InfoExtractor):
21 _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
22 _LOGIN_URL = 'https://app.lecturio.com/en/login'
23 _NETRC_MACHINE = 'lecturio'
24
25 def _perform_login(self, username, password):
26 # Sets some cookies
27 _, urlh = self._download_webpage_handle(
28 self._LOGIN_URL, None, 'Downloading login popup')
29
30 def is_logged(url_handle):
31 return self._LOGIN_URL not in url_handle.geturl()
32
33 # Already logged in
34 if is_logged(urlh):
35 return
36
37 login_form = {
38 'signin[email]': username,
39 'signin[password]': password,
40 'signin[remember]': 'on',
41 }
42
43 response, urlh = self._download_webpage_handle(
44 self._LOGIN_URL, None, 'Logging in',
45 data=urlencode_postdata(login_form))
46
47 # Logged in successfully
48 if is_logged(urlh):
49 return
50
51 errors = self._html_search_regex(
52 r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
53 'errors', default=None)
54 if errors:
55 raise ExtractorError('Unable to login: %s' % errors, expected=True)
56 raise ExtractorError('Unable to log in')
57
58
59class LecturioIE(LecturioBaseIE):
60 _VALID_URL = r'''(?x)
61 https://
62 (?:
63 app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
64 (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
65 )
66 '''
67 _TESTS = [{
68 'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
69 'md5': '9a42cf1d8282a6311bf7211bbde26fde',
70 'info_dict': {
71 'id': '39634',
72 'ext': 'mp4',
73 'title': 'Important Concepts and Terms — Introduction to Microbiology',
74 },
75 'skip': 'Requires lecturio account credentials',
76 }, {
77 'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
78 'only_matching': True,
79 }, {
80 'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
81 'only_matching': True,
82 }]
83
84 _CC_LANGS = {
85 'Arabic': 'ar',
86 'Bulgarian': 'bg',
87 'German': 'de',
88 'English': 'en',
89 'Spanish': 'es',
90 'Persian': 'fa',
91 'French': 'fr',
92 'Japanese': 'ja',
93 'Polish': 'pl',
94 'Pashto': 'ps',
95 'Russian': 'ru',
96 }
97
98 def _real_extract(self, url):
99 mobj = self._match_valid_url(url)
100 nt = mobj.group('nt') or mobj.group('nt_de')
101 lecture_id = mobj.group('id')
102 display_id = nt or lecture_id
103 api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
104 video = self._download_json(
105 self._API_BASE_URL + api_path, display_id)
106 title = video['title'].strip()
107 if not lecture_id:
108 pid = video.get('productId') or video.get('uid')
109 if pid:
110 spid = pid.split('_')
111 if spid and len(spid) == 2:
112 lecture_id = spid[1]
113
114 formats = []
115 for format_ in video['content']['media']:
116 if not isinstance(format_, dict):
117 continue
118 file_ = format_.get('file')
119 if not file_:
120 continue
121 ext = determine_ext(file_)
122 if ext == 'smil':
123 # smil contains only broken RTMP formats anyway
124 continue
125 file_url = url_or_none(file_)
126 if not file_url:
127 continue
128 label = str_or_none(format_.get('label'))
129 filesize = int_or_none(format_.get('fileSize'))
130 f = {
131 'url': file_url,
132 'format_id': label,
133 'filesize': float_or_none(filesize, invscale=1000)
134 }
135 if label:
136 mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
137 if mobj:
138 f.update({
139 'format_id': mobj.group(2),
140 'height': int(mobj.group(1)),
141 })
142 formats.append(f)
143 self._sort_formats(formats)
144
145 subtitles = {}
146 automatic_captions = {}
147 captions = video.get('captions') or []
148 for cc in captions:
149 cc_url = cc.get('url')
150 if not cc_url:
151 continue
152 cc_label = cc.get('translatedCode')
153 lang = cc.get('languageCode') or self._search_regex(
154 r'/([a-z]{2})_', cc_url, 'lang',
155 default=cc_label.split()[0] if cc_label else 'en')
156 original_lang = self._search_regex(
157 r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
158 default=None)
159 sub_dict = (automatic_captions
160 if 'auto-translated' in cc_label or original_lang
161 else subtitles)
162 sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
163 'url': cc_url,
164 })
165
166 return {
167 'id': lecture_id or nt,
168 'title': title,
169 'formats': formats,
170 'subtitles': subtitles,
171 'automatic_captions': automatic_captions,
172 }
173
174
175class LecturioCourseIE(LecturioBaseIE):
176 _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
177 _TESTS = [{
178 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
179 'info_dict': {
180 'id': 'microbiology-introduction',
181 'title': 'Microbiology: Introduction',
182 'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
183 },
184 'playlist_count': 45,
185 'skip': 'Requires lecturio account credentials',
186 }, {
187 'url': 'https://app.lecturio.com/#/course/c/6434',
188 'only_matching': True,
189 }]
190
191 def _real_extract(self, url):
192 nt, course_id = self._match_valid_url(url).groups()
193 display_id = nt or course_id
194 api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
195 course = self._download_json(
196 self._API_BASE_URL + api_path, display_id)
197 entries = []
198 for lecture in course.get('lectures', []):
199 lecture_id = str_or_none(lecture.get('id'))
200 lecture_url = lecture.get('url')
201 if lecture_url:
202 lecture_url = urljoin(url, lecture_url)
203 else:
204 lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
205 entries.append(self.url_result(
206 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
207 return self.playlist_result(
208 entries, display_id, course.get('title'),
209 clean_html(course.get('description')))
210
211
212class LecturioDeCourseIE(LecturioBaseIE):
213 _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
214 _TEST = {
215 'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
216 'only_matching': True,
217 }
218
219 def _real_extract(self, url):
220 display_id = self._match_id(url)
221
222 webpage = self._download_webpage(url, display_id)
223
224 entries = []
225 for mobj in re.finditer(
226 r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
227 webpage):
228 lecture_url = urljoin(url, mobj.group('url'))
229 lecture_id = mobj.group('id')
230 entries.append(self.url_result(
231 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
232
233 title = self._search_regex(
234 r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
235
236 return self.playlist_result(entries, display_id, title)