PageRenderTime 88ms CodeModel.GetById 31ms app.highlight 46ms RepoModel.GetById 1ms app.codeStats 0ms

/yt_dlp/extractor/lecturio.py

https://gitlab.com/vitalii.dr/yt-dlp
Python | 236 lines | 203 code | 28 blank | 5 comment | 36 complexity | 2302eddbc99774374f8b5174fae112da MD5 | raw file
  1# coding: utf-8
  2from __future__ import unicode_literals
  3
  4import re
  5
  6from .common import InfoExtractor
  7from ..utils import (
  8    clean_html,
  9    determine_ext,
 10    ExtractorError,
 11    float_or_none,
 12    int_or_none,
 13    str_or_none,
 14    url_or_none,
 15    urlencode_postdata,
 16    urljoin,
 17)
 18
 19
 20class LecturioBaseIE(InfoExtractor):
 21    _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
 22    _LOGIN_URL = 'https://app.lecturio.com/en/login'
 23    _NETRC_MACHINE = 'lecturio'
 24
 25    def _perform_login(self, username, password):
 26        # Sets some cookies
 27        _, urlh = self._download_webpage_handle(
 28            self._LOGIN_URL, None, 'Downloading login popup')
 29
 30        def is_logged(url_handle):
 31            return self._LOGIN_URL not in url_handle.geturl()
 32
 33        # Already logged in
 34        if is_logged(urlh):
 35            return
 36
 37        login_form = {
 38            'signin[email]': username,
 39            'signin[password]': password,
 40            'signin[remember]': 'on',
 41        }
 42
 43        response, urlh = self._download_webpage_handle(
 44            self._LOGIN_URL, None, 'Logging in',
 45            data=urlencode_postdata(login_form))
 46
 47        # Logged in successfully
 48        if is_logged(urlh):
 49            return
 50
 51        errors = self._html_search_regex(
 52            r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
 53            'errors', default=None)
 54        if errors:
 55            raise ExtractorError('Unable to login: %s' % errors, expected=True)
 56        raise ExtractorError('Unable to log in')
 57
 58
 59class LecturioIE(LecturioBaseIE):
 60    _VALID_URL = r'''(?x)
 61                    https://
 62                        (?:
 63                            app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
 64                            (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
 65                        )
 66                    '''
 67    _TESTS = [{
 68        'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
 69        'md5': '9a42cf1d8282a6311bf7211bbde26fde',
 70        'info_dict': {
 71            'id': '39634',
 72            'ext': 'mp4',
 73            'title': 'Important Concepts and Terms — Introduction to Microbiology',
 74        },
 75        'skip': 'Requires lecturio account credentials',
 76    }, {
 77        'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
 78        'only_matching': True,
 79    }, {
 80        'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
 81        'only_matching': True,
 82    }]
 83
 84    _CC_LANGS = {
 85        'Arabic': 'ar',
 86        'Bulgarian': 'bg',
 87        'German': 'de',
 88        'English': 'en',
 89        'Spanish': 'es',
 90        'Persian': 'fa',
 91        'French': 'fr',
 92        'Japanese': 'ja',
 93        'Polish': 'pl',
 94        'Pashto': 'ps',
 95        'Russian': 'ru',
 96    }
 97
 98    def _real_extract(self, url):
 99        mobj = self._match_valid_url(url)
100        nt = mobj.group('nt') or mobj.group('nt_de')
101        lecture_id = mobj.group('id')
102        display_id = nt or lecture_id
103        api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
104        video = self._download_json(
105            self._API_BASE_URL + api_path, display_id)
106        title = video['title'].strip()
107        if not lecture_id:
108            pid = video.get('productId') or video.get('uid')
109            if pid:
110                spid = pid.split('_')
111                if spid and len(spid) == 2:
112                    lecture_id = spid[1]
113
114        formats = []
115        for format_ in video['content']['media']:
116            if not isinstance(format_, dict):
117                continue
118            file_ = format_.get('file')
119            if not file_:
120                continue
121            ext = determine_ext(file_)
122            if ext == 'smil':
123                # smil contains only broken RTMP formats anyway
124                continue
125            file_url = url_or_none(file_)
126            if not file_url:
127                continue
128            label = str_or_none(format_.get('label'))
129            filesize = int_or_none(format_.get('fileSize'))
130            f = {
131                'url': file_url,
132                'format_id': label,
133                'filesize': float_or_none(filesize, invscale=1000)
134            }
135            if label:
136                mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
137                if mobj:
138                    f.update({
139                        'format_id': mobj.group(2),
140                        'height': int(mobj.group(1)),
141                    })
142            formats.append(f)
143        self._sort_formats(formats)
144
145        subtitles = {}
146        automatic_captions = {}
147        captions = video.get('captions') or []
148        for cc in captions:
149            cc_url = cc.get('url')
150            if not cc_url:
151                continue
152            cc_label = cc.get('translatedCode')
153            lang = cc.get('languageCode') or self._search_regex(
154                r'/([a-z]{2})_', cc_url, 'lang',
155                default=cc_label.split()[0] if cc_label else 'en')
156            original_lang = self._search_regex(
157                r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
158                default=None)
159            sub_dict = (automatic_captions
160                        if 'auto-translated' in cc_label or original_lang
161                        else subtitles)
162            sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
163                'url': cc_url,
164            })
165
166        return {
167            'id': lecture_id or nt,
168            'title': title,
169            'formats': formats,
170            'subtitles': subtitles,
171            'automatic_captions': automatic_captions,
172        }
173
174
175class LecturioCourseIE(LecturioBaseIE):
176    _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
177    _TESTS = [{
178        'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
179        'info_dict': {
180            'id': 'microbiology-introduction',
181            'title': 'Microbiology: Introduction',
182            'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
183        },
184        'playlist_count': 45,
185        'skip': 'Requires lecturio account credentials',
186    }, {
187        'url': 'https://app.lecturio.com/#/course/c/6434',
188        'only_matching': True,
189    }]
190
191    def _real_extract(self, url):
192        nt, course_id = self._match_valid_url(url).groups()
193        display_id = nt or course_id
194        api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
195        course = self._download_json(
196            self._API_BASE_URL + api_path, display_id)
197        entries = []
198        for lecture in course.get('lectures', []):
199            lecture_id = str_or_none(lecture.get('id'))
200            lecture_url = lecture.get('url')
201            if lecture_url:
202                lecture_url = urljoin(url, lecture_url)
203            else:
204                lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
205            entries.append(self.url_result(
206                lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
207        return self.playlist_result(
208            entries, display_id, course.get('title'),
209            clean_html(course.get('description')))
210
211
212class LecturioDeCourseIE(LecturioBaseIE):
213    _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
214    _TEST = {
215        'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
216        'only_matching': True,
217    }
218
219    def _real_extract(self, url):
220        display_id = self._match_id(url)
221
222        webpage = self._download_webpage(url, display_id)
223
224        entries = []
225        for mobj in re.finditer(
226                r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
227                webpage):
228            lecture_url = urljoin(url, mobj.group('url'))
229            lecture_id = mobj.group('id')
230            entries.append(self.url_result(
231                lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
232
233        title = self._search_regex(
234            r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
235
236        return self.playlist_result(entries, display_id, title)