cola /cola/core/extractor/readability.py

Language Python Lines 368
MD5 Hash 2a9ff82f91f677813e5427fc4eba8ea5
Repository https://github.com/Flowerowl/cola.git View Raw File
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
Copyright (c) 2013 Qin Xuye <qin@qinxuye.me>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Created on 2013-7-15

@author: Chine
'''

import re

from cola.core.logs import get_logger
from cola.core.errors import DependencyNotInstalledError
from cola.core.utils import beautiful_soup

try:
    from bs4 import NavigableString
except ImportError:
    raise DependencyNotInstalledError("BeautifulSoup4")

from cola.core.extractor.preprocess import PreProcessor

__all__ = ['Extractor']

REGEXES = { 
    'unlikelyCandidatesRe': re.compile('combx|comment|disqus|foot|header|menu|meta|nav|rss|shoutbox|sidebar|aside|sponsor',re.I),
    'okMaybeItsACandidateRe': re.compile('and|article|body|column|main',re.I),
    'positiveRe': re.compile('article|body|content|entry|hentry|page|pagination|post|text',re.I),
    'negativeRe': re.compile('combx|comment|contact|foot|footer|footnote|link|media|meta|promo|related|scroll|shoutbox|sponsor|tags|widget',re.I),
    'divToPElementsRe': re.compile('<(a|blockquote|dl|div|img|ol|p|pre|table|ul)',re.I),
    'replaceBrsRe': re.compile('(<br[^>]*>[ \n\r\t]*){2,}',re.I),
    'replaceFontsRe': re.compile('<(\/?)font[^>]*>',re.I),
    'trimRe': re.compile('^\s+|\s+$/'),
    'normalizeRe': re.compile('\s{2,}/'),
    'killBreaksRe': re.compile('(<br\s*\/?>(\s|&nbsp;?)*){1,}/'),
    'videoRe': re.compile('http:\/\/(www\.)?(youtube|vimeo)\.com', re.I),
}

class HashableElement():
    def __init__(self, node):
        self.node = node
        self._path = None

    def _get_path(self):
        if self._path is None:
            reverse_path = []
            node = self.node
            while node:
                node_id = (node.name, tuple(node.attrs), node.string)
                reverse_path.append(node_id)
                node = node.parent
            self._path = tuple(reverse_path)
        return self._path
    path = property(_get_path)

    def __hash__(self):
        return hash(self.path)

    def __eq__(self, other):
        return self.path == other.path

    def __getattr__(self, name):
        return getattr(self.node, name)

class Extractor(object):
    TEXT_LENGTH_THRESHOLD = 25
    RETRY_LENGTH = 250
    
    def __init__(self, content, base_url=None, logger=None, debug=False, **options):
        self._content = content
        self.logger = logger
        self.base_url = base_url
        if self.logger is None:
            self.logger = get_logger('cola_extractor')
        self.on_debug = debug
        self.debug = self.logger.info if debug else (lambda s: None)
        self.options = options
            
        self._title = None
        self._html = None
            
    def preprocess(self, force=False):
        if force is True or self._html is None:
            preprocessor = PreProcessor(self._content, base_url=self.base_url)
            self._title, self._html = preprocessor.process()
            
    def title(self, force=False):
        self.preprocess(force=force)
        return self._title
    
    def content(self, force=False):
        self.preprocess(force=force)
        return self._html
    
    def _tags(self, node, *tag_names):
        for tag_name in tag_names:
            for n in node.find_all(tag_name):
                yield n
                
    def _text(self, node):
        return ''.join(node.find_all(text=True))
    
    def _describe(self, node):
        if not hasattr(node, 'name'):
            return "[text]"
        return "%s#%s.%s" % (
            node.name, node.get('id', ''), node.get('class',''))
                
    def _remove_unlikely_candidates(self):
        for elem in self._html.find_all():
            s = '%s%s%s' % (
                elem.name, elem.get('class', ''), elem.get('id', '')
            )
            if REGEXES['unlikelyCandidatesRe'].search(s) and \
                (not REGEXES['okMaybeItsACandidateRe'].search(s)) and \
                elem.name != 'body':
                self.debug("Removing unlikely candidate - %s" % (s,))
                elem.extract()
                
    def _transform_misused_divs_into_p(self):
        for elem in self._html.find_all('div'):
            if not REGEXES['divToPElementsRe'].search(''.join(map(unicode, elem.contents))):
                self.debug("Altering div(#%s.%s) to p" % (elem.get('id', ''), elem.get('class', '')))
                elem.name = 'p'
                
    def _get_link_density(self, node):
        link_length = len("".join([i.text or "" for i in node.find_all("a")]))
        text_length = len(self._text(node))
        return float(link_length) / max(text_length, 1)
                
    def _weight_node(self, node):
        weight = 0
        if node.get('class', None):
            cls = ''.join(node['class'])
            
            if REGEXES['negativeRe'].search(cls):
                weight -= 25

            if REGEXES['positiveRe'].search(cls):
                weight += 25

        if node.get('id', None):
            if REGEXES['negativeRe'].search(node['id']):
                weight -= 25

            if REGEXES['positiveRe'].search(node['id']):
                weight += 25

        return weight
                
    def _score_node(self, node):
        content_score = self._weight_node(node)
        name = node.name.lower()
        if name in ("div", "article"):
            content_score += 5
        elif name == "blockquote":
            content_score += 3
        elif name == "form":
            content_score -= 3
        elif name == "th":
            content_score -= 5
        return { 'content_score': content_score, 'elem': node }
                
    def _score_paragraphs(self, min_text_length=None):
        if min_text_length is None:
            min_text_length = self.TEXT_LENGTH_THRESHOLD
            
        candidates = {}
        elems = self._tags(self._html, 'p', 'td')
        
        for elem in elems:
            parent_node = elem.parent
            grand_parent_node = parent_node.parent
            parent_key = HashableElement(parent_node)
            grand_parent_key = HashableElement(grand_parent_node)

            inner_text = self._text(elem)
            
            # If this paragraph is less than 25 characters, don't even count it.
            if (not inner_text) or len(inner_text) < min_text_length:
                continue
            
            if parent_key not in candidates:
                candidates[parent_key] = self._score_node(parent_node)
            if grand_parent_node and grand_parent_key not in candidates:
                candidates[grand_parent_key] = self._score_node(grand_parent_node)
                
            content_score = 1
            content_score += len(re.split(ur',|,', inner_text))
            content_score += min([(len(inner_text) / 100), 3])

            candidates[parent_key]['content_score'] += content_score
            if grand_parent_node:
                candidates[grand_parent_key]['content_score'] += content_score / 2.0
                
        # Scale the final candidates score based on link density. Good content should have a
        # relatively small link density (5% or less) and be mostly unaffected by this operation.
        for elem, candidate in candidates.items():
            candidate['content_score'] *= (1 - self._get_link_density(elem))
            self.debug("candidate %s scored %s" % (self._describe(elem), candidate['content_score']))

        return candidates
    
    def _select_best_candidate(self, candidates):
        sorted_candidates = sorted(candidates.values(), 
                                   key=lambda x: x['content_score'], 
                                   reverse=True)
        self.debug("Top 5 candidates:")
        for candidate in sorted_candidates[:5]:
            elem = candidate['elem']
            self.debug("Candidate %s with score %s" % \
                       (self._describe(elem), candidate['content_score']))

        if len(sorted_candidates) == 0:
            return None
        best_candidate = sorted_candidates[0]
        self.debug("Best candidate %s with score %s" % \
                   (self._describe(best_candidate['elem']), best_candidate['content_score']))
        return best_candidate
    
    def _get_article(self, candidates, best_candidate):
        # Now that we have the top candidate, look through its siblings for content that might also be related.
        # Things like preambles, content split by ads that we removed, etc.
        
        sibling_score_threshold = max([10, best_candidate['content_score'] * 0.2])
        output = beautiful_soup("<div/>")
        for sibling in best_candidate['elem'].parent.contents:
            if isinstance(sibling, NavigableString): continue
            append = False
            if sibling is best_candidate['elem']:
                append = True
            sibling_key = HashableElement(sibling)
            if sibling_key in candidates and \
                candidates[sibling_key]['content_score'] >= sibling_score_threshold:
                append = True

            if sibling.name == "p":
                link_density = self._get_link_density(sibling)
                node_content = sibling.string or ""
                node_length = len(node_content)

                if node_length > 80 and link_density < 0.25:
                    append = True
                elif node_length < 80 and link_density == 0 and re.search('\.( |$)', node_content):
                    append = True

            if append:
                output.div.append(sibling)
                
        return output
    
    def _sanitize(self, node, candidates):
        for header in self._tags(node, "h1", "h2", "h3", "h4", "h5", "h6"):
            if self._weight_node(header) < 0 or \
                self._get_link_density(header) > 0.33: 
                header.extract()

        for elem in self._tags(node, "form", "iframe"):
            elem.extract()

        # Conditionally clean <table>s, <ul>s, and <div>s
        for el in self._tags(node, "table", "ul", "div"):
            weight = self._weight_node(el)
            el_key = HashableElement(el)
            if el_key in candidates:
                content_score = candidates[el_key]['content_score']
            else:
                content_score = 0
            name = el.name

            if weight + content_score < 0:
                el.extract()
                self.debug("Conditionally cleaned %s with weight %s and content score %s because score + content score was less than zero." %
                    (self._describe(el), weight, content_score))
            elif len(re.split(ur',|,', self._text(el))) < 10:
                counts = {}
                for kind in ['p', 'img', 'li', 'a', 'embed', 'input']:
                    counts[kind] = len(el.find_all(kind))
                counts["li"] -= 100

                content_length = len(self._text(el)) # Count the text length excluding any surrounding whitespace
                link_density = self._get_link_density(el)
                to_remove = False
                reason = ""

                if counts["img"] > counts["p"]:
                    reason = "too many images"
                    to_remove = True
                elif counts["li"] > counts["p"] and name != "ul" and name != "ol":
                    reason = "more <li>s than <p>s"
                    to_remove = True
                elif counts["input"] > (counts["p"] / 3):
                    reason = "less than 3x <p>s than <input>s"
                    to_remove = True
                elif content_length < (self.options.get('min_text_length', self.TEXT_LENGTH_THRESHOLD)) and (counts["img"] == 0 or counts["img"] > 2):
                    reason = "too short a content length without a single image"
                    to_remove = True
                elif weight < 25 and link_density > 0.2:
                    reason = "too many links for its weight (#{weight})"
                    to_remove = True
                elif weight >= 25 and link_density > 0.5:
                    reason = "too many links for its weight (#{weight})"
                    to_remove = True
                elif (counts["embed"] == 1 and content_length < 75) or counts["embed"] > 1:
                    reason = "<embed>s with too short a content length, or too many <embed>s"
                    to_remove = True

                if to_remove:
                    self.debug("Conditionally cleaned %s#%s.%s with weight %s and content score %s because it has %s." %
                        (el.name, el.get('id',''), el.get('class', ''), weight, content_score, reason))
                    el.extract()

        for el in ([node] + node.find_all()):
            if not (self.options.get('attributes')):
                el.attrMap = {}

        return unicode(node)
            
    def extract(self):
        try:
            ruthless = True
            while True:
                self.preprocess(force=True)
                for tag in self._tags(self._html, 'script', 'style'):
                    tag.extract()
                    
                if ruthless:
                    self._remove_unlikely_candidates()
                self._transform_misused_divs_into_p()
                candidates = self._score_paragraphs(self.options.get('min_text_length'))
                best_candidate = self._select_best_candidate(candidates)
                if best_candidate:
                    article = self._get_article(candidates, best_candidate)
                else:
                    if ruthless:
                        ruthless = False
                        self.debug("ended up stripping too much - going for a safer parse")
                        # try again
                        continue
                    else:
                        article = self._html.find('body') or self._html
                        
                cleaned_article = self._sanitize(article, candidates)
                retry_length = self.options.get('retry_length') or self.RETRY_LENGTH
                of_acceptable_length = len(cleaned_article or '') >= retry_length
                if ruthless and not of_acceptable_length:
                    ruthless = False
                    continue # try again
                else:
                    return cleaned_article
                
        except Exception, e:
            self.logger.exception(e)
            if self.on_debug:
                raise e
Back to Top