/lib/feedparser.py
Python | 3687 lines | 3466 code | 83 blank | 138 comment | 204 complexity | 5a92a2cf4b2970a80bdb0ef15777ae7b MD5 | raw file
Large files files are truncated, but you can click here to view the full file
- #!/usr/bin/env python
- """Universal feed parser
- Handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds
- Visit http://feedparser.org/ for the latest version
- Visit http://feedparser.org/docs/ for the latest documentation
- Required: Python 2.1 or later
- Recommended: Python 2.3 or later
- Recommended: CJKCodecs and iconv_codec <http://cjkpython.i18n.org/>
- """
- __version__ = "4.2-pre-" + "$Revision: 285 $"[11:14] + "-svn"
- __license__ = """Copyright (c) 2002-2008, Mark Pilgrim, All rights reserved.
- Redistribution and use in source and binary forms, with or without modification,
- are permitted provided that the following conditions are met:
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS'
- AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- POSSIBILITY OF SUCH DAMAGE."""
- __author__ = "Mark Pilgrim <http://diveintomark.org/>"
- __contributors__ = ["Jason Diamond <http://injektilo.org/>",
- "John Beimler <http://john.beimler.org/>",
- "Fazal Majid <http://www.majid.info/mylos/weblog/>",
- "Aaron Swartz <http://aaronsw.com/>",
- "Kevin Marks <http://epeus.blogspot.com/>",
- "Sam Ruby <http://intertwingly.net/>"]
- _debug = 0
- # HTTP "User-Agent" header to send to servers when downloading feeds.
- # If you are embedding feedparser in a larger application, you should
- # change this to your application name and URL.
- USER_AGENT = "UniversalFeedParser/%s +http://feedparser.org/" % __version__
- # HTTP "Accept" header to send to servers when downloading feeds. If you don't
- # want to send an Accept header, set this to None.
- ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1"
- # List of preferred XML parsers, by SAX driver name. These will be tried first,
- # but if they're not installed, Python will keep searching through its own list
- # of pre-installed parsers until it finds one that supports everything we need.
- PREFERRED_XML_PARSERS = ["drv_libxml2"]
- # If you want feedparser to automatically run HTML markup through HTML Tidy, set
- # this to 1. Requires mxTidy <http://www.egenix.com/files/python/mxTidy.html>
- # or utidylib <http://utidylib.berlios.de/>.
- TIDY_MARKUP = 0
- # List of Python interfaces for HTML Tidy, in order of preference. Only useful
- # if TIDY_MARKUP = 1
- PREFERRED_TIDY_INTERFACES = ["uTidy", "mxTidy"]
- # If you want feedparser to automatically resolve all relative URIs, set this
- # to 1.
- RESOLVE_RELATIVE_URIS = 1
- # If you want feedparser to automatically sanitize all potentially unsafe
- # HTML content, set this to 1.
- SANITIZE_HTML = 1
- # ---------- required modules (should come with any Python distribution) ----------
- import sgmllib, re, sys, copy, urlparse, time, rfc822, types, cgi, urllib, urllib2
- try:
- from cStringIO import StringIO as _StringIO
- except:
- from StringIO import StringIO as _StringIO
- # ---------- optional modules (feedparser will work without these, but with reduced functionality) ----------
- # gzip is included with most Python distributions, but may not be available if you compiled your own
- try:
- import gzip
- except:
- gzip = None
- try:
- import zlib
- except:
- zlib = None
- # If a real XML parser is available, feedparser will attempt to use it. feedparser has
- # been tested with the built-in SAX parser, PyXML, and libxml2. On platforms where the
- # Python distribution does not come with an XML parser (such as Mac OS X 10.2 and some
- # versions of FreeBSD), feedparser will quietly fall back on regex-based parsing.
- try:
- import xml.sax
- xml.sax.make_parser(PREFERRED_XML_PARSERS) # test for valid parsers
- from xml.sax.saxutils import escape as _xmlescape
- _XML_AVAILABLE = 1
- except:
- _XML_AVAILABLE = 0
- def _xmlescape(data,entities={}):
- data = data.replace('&', '&')
- data = data.replace('>', '>')
- data = data.replace('<', '<')
- for char, entity in entities:
- data = data.replace(char, entity)
- return data
- # base64 support for Atom feeds that contain embedded binary data
- try:
- import base64, binascii
- except:
- base64 = binascii = None
- # cjkcodecs and iconv_codec provide support for more character encodings.
- # Both are available from http://cjkpython.i18n.org/
- try:
- import cjkcodecs.aliases
- except:
- pass
- try:
- import iconv_codec
- except:
- pass
- # chardet library auto-detects character encodings
- # Download from http://chardet.feedparser.org/
- try:
- import chardet
- if _debug:
- import chardet.constants
- chardet.constants._debug = 1
- except:
- chardet = None
- # reversable htmlentitydefs mappings for Python 2.2
- try:
- from htmlentitydefs import name2codepoint, codepoint2name
- except:
- import htmlentitydefs
- name2codepoint={}
- codepoint2name={}
- for (name,codepoint) in htmlentitydefs.entitydefs.iteritems():
- if codepoint.startswith('&#'): codepoint=unichr(int(codepoint[2:-1]))
- name2codepoint[name]=ord(codepoint)
- codepoint2name[ord(codepoint)]=name
- # BeautifulSoup parser used for parsing microformats from embedded HTML content
- # http://www.crummy.com/software/BeautifulSoup/
- # feedparser is tested with BeautifulSoup 3.0.x, but it might work with the
- # older 2.x series. If it doesn't, and you can figure out why, I'll accept a
- # patch and modify the compatibility statement accordingly.
- try:
- import BeautifulSoup
- except:
- BeautifulSoup = None
- # ---------- don't touch these ----------
- class ThingsNobodyCaresAboutButMe(Exception): pass
- class CharacterEncodingOverride(ThingsNobodyCaresAboutButMe): pass
- class CharacterEncodingUnknown(ThingsNobodyCaresAboutButMe): pass
- class NonXMLContentType(ThingsNobodyCaresAboutButMe): pass
- class UndeclaredNamespace(Exception): pass
- sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*')
- sgmllib.special = re.compile('<!')
- sgmllib.charref = re.compile('&#(\d+|x[0-9a-fA-F]+);')
- if sgmllib.endbracket.search(' <').start(0):
- class EndBracketMatch:
- endbracket = re.compile('''([^'"<>]|"[^"]*"(?=>|/|\s|\w+=)|'[^']*'(?=>|/|\s|\w+=))*(?=[<>])|.*?(?=[<>])''')
- def search(self,string,index=0):
- self.match = self.endbracket.match(string,index)
- if self.match: return self
- def start(self,n):
- return self.match.end(n)
- sgmllib.endbracket = EndBracketMatch()
- SUPPORTED_VERSIONS = {'': 'unknown',
- 'rss090': 'RSS 0.90',
- 'rss091n': 'RSS 0.91 (Netscape)',
- 'rss091u': 'RSS 0.91 (Userland)',
- 'rss092': 'RSS 0.92',
- 'rss093': 'RSS 0.93',
- 'rss094': 'RSS 0.94',
- 'rss20': 'RSS 2.0',
- 'rss10': 'RSS 1.0',
- 'rss': 'RSS (unknown version)',
- 'atom01': 'Atom 0.1',
- 'atom02': 'Atom 0.2',
- 'atom03': 'Atom 0.3',
- 'atom10': 'Atom 1.0',
- 'atom': 'Atom (unknown version)',
- 'cdf': 'CDF',
- 'hotrss': 'Hot RSS'
- }
- try:
- UserDict = dict
- except NameError:
- # Python 2.1 does not have dict
- from UserDict import UserDict
- def dict(aList):
- rc = {}
- for k, v in aList:
- rc[k] = v
- return rc
- class FeedParserDict(UserDict):
- keymap = {'channel': 'feed',
- 'items': 'entries',
- 'guid': 'id',
- 'date': 'updated',
- 'date_parsed': 'updated_parsed',
- 'description': ['subtitle', 'summary'],
- 'url': ['href'],
- 'modified': 'updated',
- 'modified_parsed': 'updated_parsed',
- 'issued': 'published',
- 'issued_parsed': 'published_parsed',
- 'copyright': 'rights',
- 'copyright_detail': 'rights_detail',
- 'tagline': 'subtitle',
- 'tagline_detail': 'subtitle_detail'}
- def __getitem__(self, key):
- if key == 'category':
- return UserDict.__getitem__(self, 'tags')[0]['term']
- if key == 'enclosures':
- norel = lambda link: FeedParserDict([(name,value) for (name,value) in link.items() if name!='rel'])
- return [norel(link) for link in UserDict.__getitem__(self, 'links') if link['rel']=='enclosure']
- if key == 'license':
- for link in UserDict.__getitem__(self, 'links'):
- if link['rel']=='license' and link.has_key('href'):
- return link['href']
- if key == 'categories':
- return [(tag['scheme'], tag['term']) for tag in UserDict.__getitem__(self, 'tags')]
- realkey = self.keymap.get(key, key)
- if type(realkey) == types.ListType:
- for k in realkey:
- if UserDict.has_key(self, k):
- return UserDict.__getitem__(self, k)
- if UserDict.has_key(self, key):
- return UserDict.__getitem__(self, key)
- return UserDict.__getitem__(self, realkey)
- def __setitem__(self, key, value):
- for k in self.keymap.keys():
- if key == k:
- key = self.keymap[k]
- if type(key) == types.ListType:
- key = key[0]
- return UserDict.__setitem__(self, key, value)
- def get(self, key, default=None):
- if self.has_key(key):
- return self[key]
- else:
- return default
- def setdefault(self, key, value):
- if not self.has_key(key):
- self[key] = value
- return self[key]
-
- def has_key(self, key):
- try:
- return hasattr(self, key) or UserDict.has_key(self, key)
- except AttributeError:
- return False
-
- def __getattr__(self, key):
- try:
- return self.__dict__[key]
- except KeyError:
- pass
- try:
- assert not key.startswith('_')
- return self.__getitem__(key)
- except:
- raise AttributeError, "object has no attribute '%s'" % key
- def __setattr__(self, key, value):
- if key.startswith('_') or key == 'data':
- self.__dict__[key] = value
- else:
- return self.__setitem__(key, value)
- def __contains__(self, key):
- return self.has_key(key)
- def zopeCompatibilityHack():
- global FeedParserDict
- del FeedParserDict
- def FeedParserDict(aDict=None):
- rc = {}
- if aDict:
- rc.update(aDict)
- return rc
- _ebcdic_to_ascii_map = None
- def _ebcdic_to_ascii(s):
- global _ebcdic_to_ascii_map
- if not _ebcdic_to_ascii_map:
- emap = (
- 0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
- 16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
- 128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
- 144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
- 32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
- 38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
- 45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
- 186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
- 195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,201,
- 202,106,107,108,109,110,111,112,113,114,203,204,205,206,207,208,
- 209,126,115,116,117,118,119,120,121,122,210,211,212,213,214,215,
- 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,
- 123,65,66,67,68,69,70,71,72,73,232,233,234,235,236,237,
- 125,74,75,76,77,78,79,80,81,82,238,239,240,241,242,243,
- 92,159,83,84,85,86,87,88,89,90,244,245,246,247,248,249,
- 48,49,50,51,52,53,54,55,56,57,250,251,252,253,254,255
- )
- import string
- _ebcdic_to_ascii_map = string.maketrans( \
- ''.join(map(chr, range(256))), ''.join(map(chr, emap)))
- return s.translate(_ebcdic_to_ascii_map)
-
- _cp1252 = {
- unichr(128): unichr(8364), # euro sign
- unichr(130): unichr(8218), # single low-9 quotation mark
- unichr(131): unichr( 402), # latin small letter f with hook
- unichr(132): unichr(8222), # double low-9 quotation mark
- unichr(133): unichr(8230), # horizontal ellipsis
- unichr(134): unichr(8224), # dagger
- unichr(135): unichr(8225), # double dagger
- unichr(136): unichr( 710), # modifier letter circumflex accent
- unichr(137): unichr(8240), # per mille sign
- unichr(138): unichr( 352), # latin capital letter s with caron
- unichr(139): unichr(8249), # single left-pointing angle quotation mark
- unichr(140): unichr( 338), # latin capital ligature oe
- unichr(142): unichr( 381), # latin capital letter z with caron
- unichr(145): unichr(8216), # left single quotation mark
- unichr(146): unichr(8217), # right single quotation mark
- unichr(147): unichr(8220), # left double quotation mark
- unichr(148): unichr(8221), # right double quotation mark
- unichr(149): unichr(8226), # bullet
- unichr(150): unichr(8211), # en dash
- unichr(151): unichr(8212), # em dash
- unichr(152): unichr( 732), # small tilde
- unichr(153): unichr(8482), # trade mark sign
- unichr(154): unichr( 353), # latin small letter s with caron
- unichr(155): unichr(8250), # single right-pointing angle quotation mark
- unichr(156): unichr( 339), # latin small ligature oe
- unichr(158): unichr( 382), # latin small letter z with caron
- unichr(159): unichr( 376)} # latin capital letter y with diaeresis
- _urifixer = re.compile('^([A-Za-z][A-Za-z0-9+-.]*://)(/*)(.*?)')
- def _urljoin(base, uri):
- uri = _urifixer.sub(r'\1\3', uri)
- try:
- return urlparse.urljoin(base, uri)
- except:
- uri = urlparse.urlunparse([urllib.quote(part) for part in urlparse.urlparse(uri)])
- return urlparse.urljoin(base, uri)
- class _FeedParserMixin:
- namespaces = {'': '',
- 'http://backend.userland.com/rss': '',
- 'http://blogs.law.harvard.edu/tech/rss': '',
- 'http://purl.org/rss/1.0/': '',
- 'http://my.netscape.com/rdf/simple/0.9/': '',
- 'http://example.com/newformat#': '',
- 'http://example.com/necho': '',
- 'http://purl.org/echo/': '',
- 'uri/of/echo/namespace#': '',
- 'http://purl.org/pie/': '',
- 'http://purl.org/atom/ns#': '',
- 'http://www.w3.org/2005/Atom': '',
- 'http://purl.org/rss/1.0/modules/rss091#': '',
-
- 'http://webns.net/mvcb/': 'admin',
- 'http://purl.org/rss/1.0/modules/aggregation/': 'ag',
- 'http://purl.org/rss/1.0/modules/annotate/': 'annotate',
- 'http://media.tangent.org/rss/1.0/': 'audio',
- 'http://backend.userland.com/blogChannelModule': 'blogChannel',
- 'http://web.resource.org/cc/': 'cc',
- 'http://backend.userland.com/creativeCommonsRssModule': 'creativeCommons',
- 'http://purl.org/rss/1.0/modules/company': 'co',
- 'http://purl.org/rss/1.0/modules/content/': 'content',
- 'http://my.theinfo.org/changed/1.0/rss/': 'cp',
- 'http://purl.org/dc/elements/1.1/': 'dc',
- 'http://purl.org/dc/terms/': 'dcterms',
- 'http://purl.org/rss/1.0/modules/email/': 'email',
- 'http://purl.org/rss/1.0/modules/event/': 'ev',
- 'http://rssnamespace.org/feedburner/ext/1.0': 'feedburner',
- 'http://freshmeat.net/rss/fm/': 'fm',
- 'http://xmlns.com/foaf/0.1/': 'foaf',
- 'http://www.w3.org/2003/01/geo/wgs84_pos#': 'geo',
- 'http://postneo.com/icbm/': 'icbm',
- 'http://purl.org/rss/1.0/modules/image/': 'image',
- 'http://www.itunes.com/DTDs/PodCast-1.0.dtd': 'itunes',
- 'http://example.com/DTDs/PodCast-1.0.dtd': 'itunes',
- 'http://purl.org/rss/1.0/modules/link/': 'l',
- 'http://search.yahoo.com/mrss': 'media',
- 'http://madskills.com/public/xml/rss/module/pingback/': 'pingback',
- 'http://prismstandard.org/namespaces/1.2/basic/': 'prism',
- 'http://www.w3.org/1999/02/22-rdf-syntax-ns#': 'rdf',
- 'http://www.w3.org/2000/01/rdf-schema#': 'rdfs',
- 'http://purl.org/rss/1.0/modules/reference/': 'ref',
- 'http://purl.org/rss/1.0/modules/richequiv/': 'reqv',
- 'http://purl.org/rss/1.0/modules/search/': 'search',
- 'http://purl.org/rss/1.0/modules/slash/': 'slash',
- 'http://schemas.xmlsoap.org/soap/envelope/': 'soap',
- 'http://purl.org/rss/1.0/modules/servicestatus/': 'ss',
- 'http://hacks.benhammersley.com/rss/streaming/': 'str',
- 'http://purl.org/rss/1.0/modules/subscription/': 'sub',
- 'http://purl.org/rss/1.0/modules/syndication/': 'sy',
- 'http://schemas.pocketsoap.com/rss/myDescModule/': 'szf',
- 'http://purl.org/rss/1.0/modules/taxonomy/': 'taxo',
- 'http://purl.org/rss/1.0/modules/threading/': 'thr',
- 'http://purl.org/rss/1.0/modules/textinput/': 'ti',
- 'http://madskills.com/public/xml/rss/module/trackback/':'trackback',
- 'http://wellformedweb.org/commentAPI/': 'wfw',
- 'http://purl.org/rss/1.0/modules/wiki/': 'wiki',
- 'http://www.w3.org/1999/xhtml': 'xhtml',
- 'http://www.w3.org/1999/xlink': 'xlink',
- 'http://www.w3.org/XML/1998/namespace': 'xml'
- }
- _matchnamespaces = {}
- can_be_relative_uri = ['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'href', 'comments', 'icon', 'logo']
- can_contain_relative_uris = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']
- can_contain_dangerous_markup = ['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']
- html_types = ['text/html', 'application/xhtml+xml']
-
- def __init__(self, baseuri=None, baselang=None, encoding='utf-8'):
- if _debug: sys.stderr.write('initializing FeedParser\n')
- if not self._matchnamespaces:
- for k, v in self.namespaces.items():
- self._matchnamespaces[k.lower()] = v
- self.feeddata = FeedParserDict() # feed-level data
- self.encoding = encoding # character encoding
- self.entries = [] # list of entry-level data
- self.version = '' # feed type/version, see SUPPORTED_VERSIONS
- self.namespacesInUse = {} # dictionary of namespaces defined by the feed
- # the following are used internally to track state;
- # this is really out of control and should be refactored
- self.infeed = 0
- self.inentry = 0
- self.incontent = 0
- self.intextinput = 0
- self.inimage = 0
- self.inauthor = 0
- self.incontributor = 0
- self.inpublisher = 0
- self.insource = 0
- self.sourcedata = FeedParserDict()
- self.contentparams = FeedParserDict()
- self._summaryKey = None
- self.namespacemap = {}
- self.elementstack = []
- self.basestack = []
- self.langstack = []
- self.baseuri = baseuri or ''
- self.lang = baselang or None
- self.svgOK = 0
- self.hasTitle = 0
- if baselang:
- self.feeddata['language'] = baselang.replace('_','-')
- def unknown_starttag(self, tag, attrs):
- if _debug: sys.stderr.write('start %s with %s\n' % (tag, attrs))
- # normalize attrs
- attrs = [(k.lower(), v) for k, v in attrs]
- attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs]
-
- # track xml:base and xml:lang
- attrsD = dict(attrs)
- baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri
- if type(baseuri) != type(u''):
- try:
- baseuri = unicode(baseuri, self.encoding)
- except:
- baseuri = unicode(baseuri, 'iso-8859-1')
- self.baseuri = _urljoin(self.baseuri, baseuri)
- lang = attrsD.get('xml:lang', attrsD.get('lang'))
- if lang == '':
- # xml:lang could be explicitly set to '', we need to capture that
- lang = None
- elif lang is None:
- # if no xml:lang is specified, use parent lang
- lang = self.lang
- if lang:
- if tag in ('feed', 'rss', 'rdf:RDF'):
- self.feeddata['language'] = lang.replace('_','-')
- self.lang = lang
- self.basestack.append(self.baseuri)
- self.langstack.append(lang)
-
- # track namespaces
- for prefix, uri in attrs:
- if prefix.startswith('xmlns:'):
- self.trackNamespace(prefix[6:], uri)
- elif prefix == 'xmlns':
- self.trackNamespace(None, uri)
- # track inline content
- if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'):
- if tag in ['xhtml:div', 'div']: return # typepad does this 10/2007
- # element declared itself as escaped markup, but it isn't really
- self.contentparams['type'] = 'application/xhtml+xml'
- if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml':
- if tag.find(':') <> -1:
- prefix, tag = tag.split(':', 1)
- namespace = self.namespacesInUse.get(prefix, '')
- if tag=='math' and namespace=='http://www.w3.org/1998/Math/MathML':
- attrs.append(('xmlns',namespace))
- if tag=='svg' and namespace=='http://www.w3.org/2000/svg':
- attrs.append(('xmlns',namespace))
- if tag == 'svg': self.svgOK = 1
- return self.handle_data('<%s%s>' % (tag, self.strattrs(attrs)), escape=0)
- # match namespaces
- if tag.find(':') <> -1:
- prefix, suffix = tag.split(':', 1)
- else:
- prefix, suffix = '', tag
- prefix = self.namespacemap.get(prefix, prefix)
- if prefix:
- prefix = prefix + '_'
- # special hack for better tracking of empty textinput/image elements in illformed feeds
- if (not prefix) and tag not in ('title', 'link', 'description', 'name'):
- self.intextinput = 0
- if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'):
- self.inimage = 0
-
- # call special handler (if defined) or default handler
- methodname = '_start_' + prefix + suffix
- try:
- method = getattr(self, methodname)
- return method(attrsD)
- except AttributeError:
- return self.push(prefix + suffix, 1)
- def unknown_endtag(self, tag):
- if _debug: sys.stderr.write('end %s\n' % tag)
- # match namespaces
- if tag.find(':') <> -1:
- prefix, suffix = tag.split(':', 1)
- else:
- prefix, suffix = '', tag
- prefix = self.namespacemap.get(prefix, prefix)
- if prefix:
- prefix = prefix + '_'
- if suffix == 'svg': self.svgOK = 0
- # call special handler (if defined) or default handler
- methodname = '_end_' + prefix + suffix
- try:
- if self.svgOK: raise AttributeError()
- method = getattr(self, methodname)
- method()
- except AttributeError:
- self.pop(prefix + suffix)
- # track inline content
- if self.incontent and self.contentparams.has_key('type') and not self.contentparams.get('type', 'xml').endswith('xml'):
- # element declared itself as escaped markup, but it isn't really
- if tag in ['xhtml:div', 'div']: return # typepad does this 10/2007
- self.contentparams['type'] = 'application/xhtml+xml'
- if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml':
- tag = tag.split(':')[-1]
- self.handle_data('</%s>' % tag, escape=0)
- # track xml:base and xml:lang going out of scope
- if self.basestack:
- self.basestack.pop()
- if self.basestack and self.basestack[-1]:
- self.baseuri = self.basestack[-1]
- if self.langstack:
- self.langstack.pop()
- if self.langstack: # and (self.langstack[-1] is not None):
- self.lang = self.langstack[-1]
- def handle_charref(self, ref):
- # called for each character reference, e.g. for ' ', ref will be '160'
- if not self.elementstack: return
- ref = ref.lower()
- if ref in ('34', '38', '39', '60', '62', 'x22', 'x26', 'x27', 'x3c', 'x3e'):
- text = '&#%s;' % ref
- else:
- if ref[0] == 'x':
- c = int(ref[1:], 16)
- else:
- c = int(ref)
- text = unichr(c).encode('utf-8')
- self.elementstack[-1][2].append(text)
- def handle_entityref(self, ref):
- # called for each entity reference, e.g. for '©', ref will be 'copy'
- if not self.elementstack: return
- if _debug: sys.stderr.write('entering handle_entityref with %s\n' % ref)
- if ref in ('lt', 'gt', 'quot', 'amp', 'apos'):
- text = '&%s;' % ref
- elif ref in self.entities.keys():
- text = self.entities[ref]
- if text.startswith('&#') and text.endswith(';'):
- return self.handle_entityref(text)
- else:
- try: name2codepoint[ref]
- except KeyError: text = '&%s;' % ref
- else: text = unichr(name2codepoint[ref]).encode('utf-8')
- self.elementstack[-1][2].append(text)
- def handle_data(self, text, escape=1):
- # called for each block of plain text, i.e. outside of any tag and
- # not containing any character or entity references
- if not self.elementstack: return
- if escape and self.contentparams.get('type') == 'application/xhtml+xml':
- text = _xmlescape(text)
- self.elementstack[-1][2].append(text)
- def handle_comment(self, text):
- # called for each comment, e.g. <!-- insert message here -->
- pass
- def handle_pi(self, text):
- # called for each processing instruction, e.g. <?instruction>
- pass
- def handle_decl(self, text):
- pass
- def parse_declaration(self, i):
- # override internal declaration handler to handle CDATA blocks
- if _debug: sys.stderr.write('entering parse_declaration\n')
- if self.rawdata[i:i+9] == '<![CDATA[':
- k = self.rawdata.find(']]>', i)
- if k == -1: k = len(self.rawdata)
- self.handle_data(_xmlescape(self.rawdata[i+9:k]), 0)
- return k+3
- else:
- k = self.rawdata.find('>', i)
- return k+1
- def mapContentType(self, contentType):
- contentType = contentType.lower()
- if contentType == 'text':
- contentType = 'text/plain'
- elif contentType == 'html':
- contentType = 'text/html'
- elif contentType == 'xhtml':
- contentType = 'application/xhtml+xml'
- return contentType
-
- def trackNamespace(self, prefix, uri):
- loweruri = uri.lower()
- if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/') and not self.version:
- self.version = 'rss090'
- if loweruri == 'http://purl.org/rss/1.0/' and not self.version:
- self.version = 'rss10'
- if loweruri == 'http://www.w3.org/2005/atom' and not self.version:
- self.version = 'atom10'
- if loweruri.find('backend.userland.com/rss') <> -1:
- # match any backend.userland.com namespace
- uri = 'http://backend.userland.com/rss'
- loweruri = uri
- if self._matchnamespaces.has_key(loweruri):
- self.namespacemap[prefix] = self._matchnamespaces[loweruri]
- self.namespacesInUse[self._matchnamespaces[loweruri]] = uri
- else:
- self.namespacesInUse[prefix or ''] = uri
- def resolveURI(self, uri):
- return _urljoin(self.baseuri or '', uri)
-
- def decodeEntities(self, element, data):
- return data
- def strattrs(self, attrs):
- return ''.join([' %s="%s"' % (t[0],_xmlescape(t[1],{'"':'"'})) for t in attrs])
- def push(self, element, expectingText):
- self.elementstack.append([element, expectingText, []])
- def pop(self, element, stripWhitespace=1):
- if not self.elementstack: return
- if self.elementstack[-1][0] != element: return
-
- element, expectingText, pieces = self.elementstack.pop()
- if self.version == 'atom10' and self.contentparams.get('type','text') == 'application/xhtml+xml':
- # remove enclosing child element, but only if it is a <div> and
- # only if all the remaining content is nested underneath it.
- # This means that the divs would be retained in the following:
- # <div>foo</div><div>bar</div>
- while pieces and len(pieces)>1 and not pieces[-1].strip():
- del pieces[-1]
- while pieces and len(pieces)>1 and not pieces[0].strip():
- del pieces[0]
- if pieces and (pieces[0] == '<div>' or pieces[0].startswith('<div ')) and pieces[-1]=='</div>':
- depth = 0
- for piece in pieces[:-1]:
- if piece.startswith('</'):
- depth -= 1
- if depth == 0: break
- elif piece.startswith('<') and not piece.endswith('/>'):
- depth += 1
- else:
- pieces = pieces[1:-1]
- output = ''.join(pieces)
- if stripWhitespace:
- output = output.strip()
- if not expectingText: return output
- # decode base64 content
- if base64 and self.contentparams.get('base64', 0):
- try:
- output = base64.decodestring(output)
- except binascii.Error:
- pass
- except binascii.Incomplete:
- pass
-
- # resolve relative URIs
- if (element in self.can_be_relative_uri) and output:
- output = self.resolveURI(output)
-
- # decode entities within embedded markup
- if not self.contentparams.get('base64', 0):
- output = self.decodeEntities(element, output)
- if self.lookslikehtml(output):
- self.contentparams['type']='text/html'
- # remove temporary cruft from contentparams
- try:
- del self.contentparams['mode']
- except KeyError:
- pass
- try:
- del self.contentparams['base64']
- except KeyError:
- pass
- is_htmlish = self.mapContentType(self.contentparams.get('type', 'text/html')) in self.html_types
- # resolve relative URIs within embedded markup
- if is_htmlish and RESOLVE_RELATIVE_URIS:
- if element in self.can_contain_relative_uris:
- output = _resolveRelativeURIs(output, self.baseuri, self.encoding, self.contentparams.get('type', 'text/html'))
-
- # parse microformats
- # (must do this before sanitizing because some microformats
- # rely on elements that we sanitize)
- if is_htmlish and element in ['content', 'description', 'summary']:
- mfresults = _parseMicroformats(output, self.baseuri, self.encoding)
- if mfresults:
- for tag in mfresults.get('tags', []):
- self._addTag(tag['term'], tag['scheme'], tag['label'])
- for enclosure in mfresults.get('enclosures', []):
- self._start_enclosure(enclosure)
- for xfn in mfresults.get('xfn', []):
- self._addXFN(xfn['relationships'], xfn['href'], xfn['name'])
- vcard = mfresults.get('vcard')
- if vcard:
- self._getContext()['vcard'] = vcard
-
- # sanitize embedded markup
- if is_htmlish and SANITIZE_HTML:
- if element in self.can_contain_dangerous_markup:
- output = _sanitizeHTML(output, self.encoding, self.contentparams.get('type', 'text/html'))
- if self.encoding and type(output) != type(u''):
- try:
- output = unicode(output, self.encoding)
- except:
- pass
- # address common error where people take data that is already
- # utf-8, presume that it is iso-8859-1, and re-encode it.
- if self.encoding=='utf-8' and type(output) == type(u''):
- try:
- output = unicode(output.encode('iso-8859-1'), 'utf-8')
- except:
- pass
- # map win-1252 extensions to the proper code points
- if type(output) == type(u''):
- output = u''.join([c in _cp1252.keys() and _cp1252[c] or c for c in output])
- # categories/tags/keywords/whatever are handled in _end_category
- if element == 'category':
- return output
- if element == 'title' and self.hasTitle:
- return output
-
- # store output in appropriate place(s)
- if self.inentry and not self.insource:
- if element == 'content':
- self.entries[-1].setdefault(element, [])
- contentparams = copy.deepcopy(self.contentparams)
- contentparams['value'] = output
- self.entries[-1][element].append(contentparams)
- elif element == 'link':
- self.entries[-1][element] = output
- if output:
- self.entries[-1]['links'][-1]['href'] = output
- else:
- if element == 'description':
- element = 'summary'
- self.entries[-1][element] = output
- if self.incontent:
- contentparams = copy.deepcopy(self.contentparams)
- contentparams['value'] = output
- self.entries[-1][element + '_detail'] = contentparams
- elif (self.infeed or self.insource):# and (not self.intextinput) and (not self.inimage):
- context = self._getContext()
- if element == 'description':
- element = 'subtitle'
- context[element] = output
- if element == 'link':
- context['links'][-1]['href'] = output
- elif self.incontent:
- contentparams = copy.deepcopy(self.contentparams)
- contentparams['value'] = output
- context[element + '_detail'] = contentparams
- return output
- def pushContent(self, tag, attrsD, defaultContentType, expectingText):
- self.incontent += 1
- if self.lang: self.lang=self.lang.replace('_','-')
- self.contentparams = FeedParserDict({
- 'type': self.mapContentType(attrsD.get('type', defaultContentType)),
- 'language': self.lang,
- 'base': self.baseuri})
- self.contentparams['base64'] = self._isBase64(attrsD, self.contentparams)
- self.push(tag, expectingText)
- def popContent(self, tag):
- value = self.pop(tag)
- self.incontent -= 1
- self.contentparams.clear()
- return value
-
- # a number of elements in a number of RSS variants are nominally plain
- # text, but this is routinely ignored. This is an attempt to detect
- # the most common cases. As false positives often result in silent
- # data loss, this function errs on the conservative side.
- def lookslikehtml(self, str):
- if self.version.startswith('atom'): return
- if self.contentparams.get('type','text/html') != 'text/plain': return
- # must have a close tag or a entity reference to qualify
- if not (re.search(r'</(\w+)>',str) or re.search("&#?\w+;",str)): return
- # all tags must be in a restricted subset of valid HTML tags
- if filter(lambda t: t.lower() not in _HTMLSanitizer.acceptable_elements,
- re.findall(r'</?(\w+)',str)): return
- # all entities must have been defined as valid HTML entities
- from htmlentitydefs import entitydefs
- if filter(lambda e: e not in entitydefs.keys(),
- re.findall(r'&(\w+);',str)): return
- return 1
- def _mapToStandardPrefix(self, name):
- colonpos = name.find(':')
- if colonpos <> -1:
- prefix = name[:colonpos]
- suffix = name[colonpos+1:]
- prefix = self.namespacemap.get(prefix, prefix)
- name = prefix + ':' + suffix
- return name
-
- def _getAttribute(self, attrsD, name):
- return attrsD.get(self._mapToStandardPrefix(name))
- def _isBase64(self, attrsD, contentparams):
- if attrsD.get('mode', '') == 'base64':
- return 1
- if self.contentparams['type'].startswith('text/'):
- return 0
- if self.contentparams['type'].endswith('+xml'):
- return 0
- if self.contentparams['type'].endswith('/xml'):
- return 0
- return 1
- def _itsAnHrefDamnIt(self, attrsD):
- href = attrsD.get('url', attrsD.get('uri', attrsD.get('href', None)))
- if href:
- try:
- del attrsD['url']
- except KeyError:
- pass
- try:
- del attrsD['uri']
- except KeyError:
- pass
- attrsD['href'] = href
- return attrsD
-
- def _save(self, key, value):
- context = self._getContext()
- context.setdefault(key, value)
- def _start_rss(self, attrsD):
- versionmap = {'0.91': 'rss091u',
- '0.92': 'rss092',
- '0.93': 'rss093',
- '0.94': 'rss094'}
- if not self.version:
- attr_version = attrsD.get('version', '')
- version = versionmap.get(attr_version)
- if version:
- self.version = version
- elif attr_version.startswith('2.'):
- self.version = 'rss20'
- else:
- self.version = 'rss'
-
- def _start_dlhottitles(self, attrsD):
- self.version = 'hotrss'
- def _start_channel(self, attrsD):
- self.infeed = 1
- self._cdf_common(attrsD)
- _start_feedinfo = _start_channel
- def _cdf_common(self, attrsD):
- if attrsD.has_key('lastmod'):
- self._start_modified({})
- self.elementstack[-1][-1] = attrsD['lastmod']
- self._end_modified()
- if attrsD.has_key('href'):
- self._start_link({})
- self.elementstack[-1][-1] = attrsD['href']
- self._end_link()
-
- def _start_feed(self, attrsD):
- self.infeed = 1
- versionmap = {'0.1': 'atom01',
- '0.2': 'atom02',
- '0.3': 'atom03'}
- if not self.version:
- attr_version = attrsD.get('version')
- version = versionmap.get(attr_version)
- if version:
- self.version = version
- else:
- self.version = 'atom'
- def _end_channel(self):
- self.infeed = 0
- _end_feed = _end_channel
-
- def _start_image(self, attrsD):
- context = self._getContext()
- context.setdefault('image', FeedParserDict())
- self.inimage = 1
- self.hasTitle = 0
- self.push('image', 0)
-
- def _end_image(self):
- self.pop('image')
- self.inimage = 0
- def _start_textinput(self, attrsD):
- context = self._getContext()
- context.setdefault('textinput', FeedParserDict())
- self.intextinput = 1
- self.hasTitle = 0
- self.push('textinput', 0)
- _start_textInput = _start_textinput
-
- def _end_textinput(self):
- self.pop('textinput')
- self.intextinput = 0
- _end_textInput = _end_textinput
- def _start_author(self, attrsD):
- self.inauthor = 1
- self.push('author', 1)
- _start_managingeditor = _start_author
- _start_dc_author = _start_author
- _start_dc_creator = _start_author
- _start_itunes_author = _start_author
- def _end_author(self):
- self.pop('author')
- self.inauthor = 0
- self._sync_author_detail()
- _end_managingeditor = _end_author
- _end_dc_author = _end_author
- _end_dc_creator = _end_author
- _end_itunes_author = _end_author
- def _start_itunes_owner(self, attrsD):
- self.inpublisher = 1
- self.push('publisher', 0)
- def _end_itunes_owner(self):
- self.pop('publisher')
- self.inpublisher = 0
- self._sync_author_detail('publisher')
- def _start_contributor(self, attrsD):
- self.incontributor = 1
- context = self._getContext()
- context.setdefault('contributors', [])
- context['contributors'].append(FeedParserDict())
- self.push('contributor', 0)
- def _end_contributor(self):
- self.pop('contributor')
- self.incontributor = 0
- def _start_dc_contributor(self, attrsD):
- self.incontributor = 1
- context = self._getContext()
- context.setdefault('contributors', [])
- context['contributors'].append(FeedParserDict())
- self.push('name', 0)
- def _end_dc_contributor(self):
- self._end_name()
- self.incontributor = 0
- def _start_name(self, attrsD):
- self.push('name', 0)
- _start_itunes_name = _start_name
- def _end_name(self):
- value = self.pop('name')
- if self.inpublisher:
- self._save_author('name', value, 'publisher')
- elif self.inauthor:
- self._save_author('name', value)
- elif self.incontributor:
- self._save_contributor('name', value)
- elif self.intextinput:
- context = self._getContext()
- context['name'] = value
- _end_itunes_name = _end_name
- def _start_width(self, attrsD):
- self.push('width', 0)
- def _end_width(self):
- value = self.pop('width')
- try:
- value = int(value)
- except:
- value = 0
- if self.inimage:
- context = self._getContext()
- context['width'] = value
- def _start_height(self, attrsD):
- self.push('height', 0)
- def _end_height(self):
- value = self.pop('height')
- try:
- value = int(value)
- except:
- value = 0
- if self.inimage:
- context = self._getContext()
- context['height'] = value
- def _start_url(self, attrsD):
- self.push('href', 1)
- _start_homepage = _start_url
- _start_uri = _start_url
- def _end_url(self):
- value = self.pop('href')
- if self.inauthor:
- self._save_author('href', value)
- elif self.incontributor:
- self._save_contributor('href', value)
- _end_homepage = _end_url
- _end_uri = _end_url
- def _start_email(self, attrsD):
- self.push('email', 0)
- _start_itunes_email = _start_email
- def _end_email(self):
- value = self.pop('email')
- if self.inpublisher:
- self._save_author('email', value, 'publisher')
- elif self.inauthor:
- self._save_author('email', value)
- elif self.incontributor:
- self._save_contributor('email', value)
- _end_itunes_email = _end_email
- def _getContext(self):
- if self.insource:
- context = self.sourcedata
- elif self.inimage:
- context = self.feeddata['image']
- elif self.intextinput:
- context = self.feeddata['textinput']
- elif self.inentry:
- context = self.entries[-1]
- else:
- context = self.feeddata
- return context
- def _save_author(self, key, value, prefix='author'):
- context = self._getContext()
- context.setdefault(prefix + '_detail', FeedParserDict())
- context[prefix + '_detail'][key] = value
- self._sync_author_detail()
- def _save_contributor(self, key, value):
- context = self._getContext()
- context.setdefault('contributors', [FeedParserDict()])
- context['contributors'][-1][key] = value
- def _sync_author_detail(self, key='author'):
- context = self._getContext()
- detail = context.get('%s_detail' % key)
- if detail:
- name = detail.get('name')
- email = detail.get('email')
- if name and email:
- context[key] = '%s (%s)' % (name, email)
- elif name:
- context[key] = name
- elif email:
- context[key] = email
- else:
- author, email = context.get(key), None
- if not author: return
- emailmatch = re.search(r'''(([a-zA-Z0-9\_\-\.\+]+)@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.)|(([a-zA-Z0-9\-]+\.)+))([a-zA-Z]{2,4}|[0-9]{1,3})(\]?))(\?subject=\S+)?''', author)
- if emailmatch:
- email = emailmatch.group(0)
- # probably a better way to do the following, but it passes all the tests
- author = author.replace(email, '')
- author = author.replace('()', '')
- author = author.replace('<>', '')
- author = author.replace('<>', '')
- author = author.strip()
- if author and (author[0] == '('):
- author = author[1:]
- if author and (author[-1] == ')'):
- author = author[:-1]
- author = author.strip()
- if author or email:
- context.setdefault('%s_detail' % key, FeedParserDict())
- if author:
- context['%s_detail' % key]['name'] = author
- if email:
- context['%s_detail' % key]['email'] = email
- def _start_subtitle(self, attrsD):
- self.pushContent('subtitle', attrsD, 'text/plain', 1)
- _start_tagline = _start_subtitle
- _start_itunes_subtitle = _start_subtitle
- def _end_subtitle(self):
- self.popContent('subtitle')
- _end_tagline = _end_subtitle
- _end_itunes_subtitle = _end_subtitle
-
- def _start_rights(self, attrsD):
- self.pushContent('rights', attrsD, 'text/plain', 1)
- _start_dc_rights = _start_rights
- _start_copyright = _start_rights
- def _end_rights(self):
- self.popContent('rights')
- _end_dc_rights = _end_rights
- _end_copyright = _end_rights
- def _start_item(self, attrsD):
- self.entries.append(FeedParserDict())
- self.push('item', 0)
- self.inentry = 1
- self.guidislink = 0
- self.hasTitle = 0
- id = self._getAttribute(attrsD, 'rdf:about')
- if id:
- context = self._getContext()
- context['id'] = id
- self._cdf_common(attrsD)
- _start_entry = _start_item
- _start_product = _start_item
- def _end_item(self):
- self.pop('item')
- self.inentry = 0
- _end_entry = _end_item
- def _start_dc_language(self, attrsD):
- self.push('language', 1)
- _start_language = _start_dc_language
- def _end_dc_language(self):
- self.lang = self.pop('language')
- _end_language = _end_dc_language
- def _start_dc_publisher(self, attrsD):
- self.push('publisher', 1)
- _start_webmaster = _start_dc_publisher
- def _end_dc_publisher(self):
- self.pop('publisher')
- self._sync_author_detail('publisher')
- _end_webmaster = _end_dc_publisher
- def _start_published(self, attrsD):
- self.push('published', 1)
- _start_dcterms_issued = _start_published
- _start_issued = _start_published
- def _end_published(self):
- value = self.pop('published')
- self._save('published_parsed', _parse_date(value))
- _end_dcterms_issued = _end_published
- _end_issued = _end_published
- def _start_updated(self, attrsD):
- self.push('updated', 1)
- _start_modified = _start_update…
Large files files are truncated, but you can click here to view the full file