PageRenderTime 54ms CodeModel.GetById 33ms app.highlight 18ms RepoModel.GetById 1ms app.codeStats 0ms

/yt_dlp/extractor/dailymotion.py

https://gitlab.com/vitalii.dr/yt-dlp
Python | 389 lines | 378 code | 10 blank | 1 comment | 16 complexity | 16b1aae33b7731466ac5f292e0b512d0 MD5 | raw file
  1# coding: utf-8
  2from __future__ import unicode_literals
  3
  4import functools
  5import json
  6import re
  7
  8from .common import InfoExtractor
  9from ..compat import compat_HTTPError
 10from ..utils import (
 11    age_restricted,
 12    clean_html,
 13    ExtractorError,
 14    int_or_none,
 15    OnDemandPagedList,
 16    try_get,
 17    unescapeHTML,
 18    urlencode_postdata,
 19)
 20
 21
 22class DailymotionBaseInfoExtractor(InfoExtractor):
 23    _FAMILY_FILTER = None
 24    _HEADERS = {
 25        'Content-Type': 'application/json',
 26        'Origin': 'https://www.dailymotion.com',
 27    }
 28    _NETRC_MACHINE = 'dailymotion'
 29
 30    def _get_dailymotion_cookies(self):
 31        return self._get_cookies('https://www.dailymotion.com/')
 32
 33    @staticmethod
 34    def _get_cookie_value(cookies, name):
 35        cookie = cookies.get(name)
 36        if cookie:
 37            return cookie.value
 38
 39    def _set_dailymotion_cookie(self, name, value):
 40        self._set_cookie('www.dailymotion.com', name, value)
 41
 42    def _real_initialize(self):
 43        cookies = self._get_dailymotion_cookies()
 44        ff = self._get_cookie_value(cookies, 'ff')
 45        self._FAMILY_FILTER = ff == 'on' if ff else age_restricted(18, self.get_param('age_limit'))
 46        self._set_dailymotion_cookie('ff', 'on' if self._FAMILY_FILTER else 'off')
 47
 48    def _call_api(self, object_type, xid, object_fields, note, filter_extra=None):
 49        if not self._HEADERS.get('Authorization'):
 50            cookies = self._get_dailymotion_cookies()
 51            token = self._get_cookie_value(cookies, 'access_token') or self._get_cookie_value(cookies, 'client_token')
 52            if not token:
 53                data = {
 54                    'client_id': 'f1a362d288c1b98099c7',
 55                    'client_secret': 'eea605b96e01c796ff369935357eca920c5da4c5',
 56                }
 57                username, password = self._get_login_info()
 58                if username:
 59                    data.update({
 60                        'grant_type': 'password',
 61                        'password': password,
 62                        'username': username,
 63                    })
 64                else:
 65                    data['grant_type'] = 'client_credentials'
 66                try:
 67                    token = self._download_json(
 68                        'https://graphql.api.dailymotion.com/oauth/token',
 69                        None, 'Downloading Access Token',
 70                        data=urlencode_postdata(data))['access_token']
 71                except ExtractorError as e:
 72                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
 73                        raise ExtractorError(self._parse_json(
 74                            e.cause.read().decode(), xid)['error_description'], expected=True)
 75                    raise
 76                self._set_dailymotion_cookie('access_token' if username else 'client_token', token)
 77            self._HEADERS['Authorization'] = 'Bearer ' + token
 78
 79        resp = self._download_json(
 80            'https://graphql.api.dailymotion.com/', xid, note, data=json.dumps({
 81                'query': '''{
 82  %s(xid: "%s"%s) {
 83    %s
 84  }
 85}''' % (object_type, xid, ', ' + filter_extra if filter_extra else '', object_fields),
 86            }).encode(), headers=self._HEADERS)
 87        obj = resp['data'][object_type]
 88        if not obj:
 89            raise ExtractorError(resp['errors'][0]['message'], expected=True)
 90        return obj
 91
 92
 93class DailymotionIE(DailymotionBaseInfoExtractor):
 94    _VALID_URL = r'''(?ix)
 95                    https?://
 96                        (?:
 97                            (?:(?:www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:(?:embed|swf|\#)/)?video|swf)|
 98                            (?:www\.)?lequipe\.fr/video
 99                        )
100                        /(?P<id>[^/?_]+)(?:.+?\bplaylist=(?P<playlist_id>x[0-9a-z]+))?
101                    '''
102    IE_NAME = 'dailymotion'
103    _TESTS = [{
104        'url': 'http://www.dailymotion.com/video/x5kesuj_office-christmas-party-review-jason-bateman-olivia-munn-t-j-miller_news',
105        'md5': '074b95bdee76b9e3654137aee9c79dfe',
106        'info_dict': {
107            'id': 'x5kesuj',
108            'ext': 'mp4',
109            'title': 'Office Christmas Party Review –  Jason Bateman, Olivia Munn, T.J. Miller',
110            'description': 'Office Christmas Party Review -  Jason Bateman, Olivia Munn, T.J. Miller',
111            'duration': 187,
112            'timestamp': 1493651285,
113            'upload_date': '20170501',
114            'uploader': 'Deadline',
115            'uploader_id': 'x1xm8ri',
116            'age_limit': 0,
117        },
118    }, {
119        'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
120        'md5': '2137c41a8e78554bb09225b8eb322406',
121        'info_dict': {
122            'id': 'x2iuewm',
123            'ext': 'mp4',
124            'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
125            'description': 'Several come bundled with the Steam Controller.',
126            'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
127            'duration': 74,
128            'timestamp': 1425657362,
129            'upload_date': '20150306',
130            'uploader': 'IGN',
131            'uploader_id': 'xijv66',
132            'age_limit': 0,
133            'view_count': int,
134        },
135        'skip': 'video gone',
136    }, {
137        # Vevo video
138        'url': 'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
139        'info_dict': {
140            'title': 'Roar (Official)',
141            'id': 'USUV71301934',
142            'ext': 'mp4',
143            'uploader': 'Katy Perry',
144            'upload_date': '20130905',
145        },
146        'params': {
147            'skip_download': True,
148        },
149        'skip': 'VEVO is only available in some countries',
150    }, {
151        # age-restricted video
152        'url': 'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
153        'md5': '0d667a7b9cebecc3c89ee93099c4159d',
154        'info_dict': {
155            'id': 'xyh2zz',
156            'ext': 'mp4',
157            'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
158            'uploader': 'HotWaves1012',
159            'age_limit': 18,
160        },
161        'skip': 'video gone',
162    }, {
163        # geo-restricted, player v5
164        'url': 'http://www.dailymotion.com/video/xhza0o',
165        'only_matching': True,
166    }, {
167        # with subtitles
168        'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
169        'only_matching': True,
170    }, {
171        'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
172        'only_matching': True,
173    }, {
174        'url': 'http://www.dailymotion.com/swf/x3ss1m_funny-magic-trick-barry-and-stuart_fun',
175        'only_matching': True,
176    }, {
177        'url': 'https://www.lequipe.fr/video/x791mem',
178        'only_matching': True,
179    }, {
180        'url': 'https://www.lequipe.fr/video/k7MtHciueyTcrFtFKA2',
181        'only_matching': True,
182    }, {
183        'url': 'https://www.dailymotion.com/video/x3z49k?playlist=xv4bw',
184        'only_matching': True,
185    }]
186    _GEO_BYPASS = False
187    _COMMON_MEDIA_FIELDS = '''description
188      geoblockedCountries {
189        allowed
190      }
191      xid'''
192
193    @staticmethod
194    def _extract_urls(webpage):
195        urls = []
196        # Look for embedded Dailymotion player
197        # https://developer.dailymotion.com/player#player-parameters
198        for mobj in re.finditer(
199                r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage):
200            urls.append(unescapeHTML(mobj.group('url')))
201        for mobj in re.finditer(
202                r'(?s)DM\.player\([^,]+,\s*{.*?video[\'"]?\s*:\s*["\']?(?P<id>[0-9a-zA-Z]+).+?}\s*\);', webpage):
203            urls.append('https://www.dailymotion.com/embed/video/' + mobj.group('id'))
204        return urls
205
206    def _real_extract(self, url):
207        video_id, playlist_id = self._match_valid_url(url).groups()
208
209        if playlist_id:
210            if self._yes_playlist(playlist_id, video_id):
211                return self.url_result(
212                    'http://www.dailymotion.com/playlist/' + playlist_id,
213                    'DailymotionPlaylist', playlist_id)
214
215        password = self.get_param('videopassword')
216        media = self._call_api(
217            'media', video_id, '''... on Video {
218      %s
219      stats {
220        likes {
221          total
222        }
223        views {
224          total
225        }
226      }
227    }
228    ... on Live {
229      %s
230      audienceCount
231      isOnAir
232    }''' % (self._COMMON_MEDIA_FIELDS, self._COMMON_MEDIA_FIELDS), 'Downloading media JSON metadata',
233            'password: "%s"' % self.get_param('videopassword') if password else None)
234        xid = media['xid']
235
236        metadata = self._download_json(
237            'https://www.dailymotion.com/player/metadata/video/' + xid,
238            xid, 'Downloading metadata JSON',
239            query={'app': 'com.dailymotion.neon'})
240
241        error = metadata.get('error')
242        if error:
243            title = error.get('title') or error['raw_message']
244            # See https://developer.dailymotion.com/api#access-error
245            if error.get('code') == 'DM007':
246                allowed_countries = try_get(media, lambda x: x['geoblockedCountries']['allowed'], list)
247                self.raise_geo_restricted(msg=title, countries=allowed_countries)
248            raise ExtractorError(
249                '%s said: %s' % (self.IE_NAME, title), expected=True)
250
251        title = metadata['title']
252        is_live = media.get('isOnAir')
253        formats = []
254        for quality, media_list in metadata['qualities'].items():
255            for m in media_list:
256                media_url = m.get('url')
257                media_type = m.get('type')
258                if not media_url or media_type == 'application/vnd.lumberjack.manifest':
259                    continue
260                if media_type == 'application/x-mpegURL':
261                    formats.extend(self._extract_m3u8_formats(
262                        media_url, video_id, 'mp4', live=is_live, m3u8_id='hls', fatal=False))
263                else:
264                    f = {
265                        'url': media_url,
266                        'format_id': 'http-' + quality,
267                    }
268                    m = re.search(r'/H264-(\d+)x(\d+)(?:-(60)/)?', media_url)
269                    if m:
270                        width, height, fps = map(int_or_none, m.groups())
271                        f.update({
272                            'fps': fps,
273                            'height': height,
274                            'width': width,
275                        })
276                    formats.append(f)
277        for f in formats:
278            f['url'] = f['url'].split('#')[0]
279            if not f.get('fps') and f['format_id'].endswith('@60'):
280                f['fps'] = 60
281        self._sort_formats(formats)
282
283        subtitles = {}
284        subtitles_data = try_get(metadata, lambda x: x['subtitles']['data'], dict) or {}
285        for subtitle_lang, subtitle in subtitles_data.items():
286            subtitles[subtitle_lang] = [{
287                'url': subtitle_url,
288            } for subtitle_url in subtitle.get('urls', [])]
289
290        thumbnails = []
291        for height, poster_url in metadata.get('posters', {}).items():
292            thumbnails.append({
293                'height': int_or_none(height),
294                'id': height,
295                'url': poster_url,
296            })
297
298        owner = metadata.get('owner') or {}
299        stats = media.get('stats') or {}
300        get_count = lambda x: int_or_none(try_get(stats, lambda y: y[x + 's']['total']))
301
302        return {
303            'id': video_id,
304            'title': title,
305            'description': clean_html(media.get('description')),
306            'thumbnails': thumbnails,
307            'duration': int_or_none(metadata.get('duration')) or None,
308            'timestamp': int_or_none(metadata.get('created_time')),
309            'uploader': owner.get('screenname'),
310            'uploader_id': owner.get('id') or metadata.get('screenname'),
311            'age_limit': 18 if metadata.get('explicit') else 0,
312            'tags': metadata.get('tags'),
313            'view_count': get_count('view') or int_or_none(media.get('audienceCount')),
314            'like_count': get_count('like'),
315            'formats': formats,
316            'subtitles': subtitles,
317            'is_live': is_live,
318        }
319
320
321class DailymotionPlaylistBaseIE(DailymotionBaseInfoExtractor):
322    _PAGE_SIZE = 100
323
324    def _fetch_page(self, playlist_id, page):
325        page += 1
326        videos = self._call_api(
327            self._OBJECT_TYPE, playlist_id,
328            '''videos(allowExplicit: %s, first: %d, page: %d) {
329      edges {
330        node {
331          xid
332          url
333        }
334      }
335    }''' % ('false' if self._FAMILY_FILTER else 'true', self._PAGE_SIZE, page),
336            'Downloading page %d' % page)['videos']
337        for edge in videos['edges']:
338            node = edge['node']
339            yield self.url_result(
340                node['url'], DailymotionIE.ie_key(), node['xid'])
341
342    def _real_extract(self, url):
343        playlist_id = self._match_id(url)
344        entries = OnDemandPagedList(functools.partial(
345            self._fetch_page, playlist_id), self._PAGE_SIZE)
346        return self.playlist_result(
347            entries, playlist_id)
348
349
350class DailymotionPlaylistIE(DailymotionPlaylistBaseIE):
351    IE_NAME = 'dailymotion:playlist'
352    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>x[0-9a-z]+)'
353    _TESTS = [{
354        'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
355        'info_dict': {
356            'id': 'xv4bw',
357        },
358        'playlist_mincount': 20,
359    }]
360    _OBJECT_TYPE = 'collection'
361
362
363class DailymotionUserIE(DailymotionPlaylistBaseIE):
364    IE_NAME = 'dailymotion:user'
365    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P<id>[^/]+)'
366    _TESTS = [{
367        'url': 'https://www.dailymotion.com/user/nqtv',
368        'info_dict': {
369            'id': 'nqtv',
370        },
371        'playlist_mincount': 152,
372    }, {
373        'url': 'http://www.dailymotion.com/user/UnderProject',
374        'info_dict': {
375            'id': 'UnderProject',
376        },
377        'playlist_mincount': 1000,
378        'skip': 'Takes too long time',
379    }, {
380        'url': 'https://www.dailymotion.com/user/nqtv',
381        'info_dict': {
382            'id': 'nqtv',
383        },
384        'playlist_mincount': 148,
385        'params': {
386            'age_limit': 0,
387        },
388    }]
389    _OBJECT_TYPE = 'channel'