PageRenderTime 468ms CodeModel.GetById 222ms app.highlight 73ms RepoModel.GetById 168ms app.codeStats 0ms

/Lib/email/charset.py

http://unladen-swallow.googlecode.com/
Python | 391 lines | 334 code | 14 blank | 43 comment | 3 complexity | 7e618fcb0b062f2deb396f7bdbdd1177 MD5 | raw file
  1# Copyright (C) 2001-2006 Python Software Foundation
  2# Author: Ben Gertzfield, Barry Warsaw
  3# Contact: email-sig@python.org
  4
  5__all__ = [
  6    'Charset',
  7    'add_alias',
  8    'add_charset',
  9    'add_codec',
 10    ]
 11
 12import email.base64mime
 13import email.quoprimime
 14
 15from email import errors
 16from email.encoders import encode_7or8bit
 17
 18
 19
 20# Flags for types of header encodings
 21QP          = 1 # Quoted-Printable
 22BASE64      = 2 # Base64
 23SHORTEST    = 3 # the shorter of QP and base64, but only for headers
 24
 25# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
 26MISC_LEN = 7
 27
 28DEFAULT_CHARSET = 'us-ascii'
 29
 30
 31
 32# Defaults
 33CHARSETS = {
 34    # input        header enc  body enc output conv
 35    'iso-8859-1':  (QP,        QP,      None),
 36    'iso-8859-2':  (QP,        QP,      None),
 37    'iso-8859-3':  (QP,        QP,      None),
 38    'iso-8859-4':  (QP,        QP,      None),
 39    # iso-8859-5 is Cyrillic, and not especially used
 40    # iso-8859-6 is Arabic, also not particularly used
 41    # iso-8859-7 is Greek, QP will not make it readable
 42    # iso-8859-8 is Hebrew, QP will not make it readable
 43    'iso-8859-9':  (QP,        QP,      None),
 44    'iso-8859-10': (QP,        QP,      None),
 45    # iso-8859-11 is Thai, QP will not make it readable
 46    'iso-8859-13': (QP,        QP,      None),
 47    'iso-8859-14': (QP,        QP,      None),
 48    'iso-8859-15': (QP,        QP,      None),
 49    'iso-8859-16': (QP,        QP,      None),
 50    'windows-1252':(QP,        QP,      None),
 51    'viscii':      (QP,        QP,      None),
 52    'us-ascii':    (None,      None,    None),
 53    'big5':        (BASE64,    BASE64,  None),
 54    'gb2312':      (BASE64,    BASE64,  None),
 55    'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
 56    'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
 57    'iso-2022-jp': (BASE64,    None,    None),
 58    'koi8-r':      (BASE64,    BASE64,  None),
 59    'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
 60    # We're making this one up to represent raw unencoded 8-bit
 61    '8bit':        (None,      BASE64, 'utf-8'),
 62    }
 63
 64# Aliases for other commonly-used names for character sets.  Map
 65# them to the real ones used in email.
 66ALIASES = {
 67    'latin_1': 'iso-8859-1',
 68    'latin-1': 'iso-8859-1',
 69    'latin_2': 'iso-8859-2',
 70    'latin-2': 'iso-8859-2',
 71    'latin_3': 'iso-8859-3',
 72    'latin-3': 'iso-8859-3',
 73    'latin_4': 'iso-8859-4',
 74    'latin-4': 'iso-8859-4',
 75    'latin_5': 'iso-8859-9',
 76    'latin-5': 'iso-8859-9',
 77    'latin_6': 'iso-8859-10',
 78    'latin-6': 'iso-8859-10',
 79    'latin_7': 'iso-8859-13',
 80    'latin-7': 'iso-8859-13',
 81    'latin_8': 'iso-8859-14',
 82    'latin-8': 'iso-8859-14',
 83    'latin_9': 'iso-8859-15',
 84    'latin-9': 'iso-8859-15',
 85    'latin_10':'iso-8859-16',
 86    'latin-10':'iso-8859-16',
 87    'cp949':   'ks_c_5601-1987',
 88    'euc_jp':  'euc-jp',
 89    'euc_kr':  'euc-kr',
 90    'ascii':   'us-ascii',
 91    }
 92
 93
 94# Map charsets to their Unicode codec strings.
 95CODEC_MAP = {
 96    'gb2312':      'eucgb2312_cn',
 97    'big5':        'big5_tw',
 98    # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
 99    # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
100    # Let that stuff pass through without conversion to/from Unicode.
101    'us-ascii':    None,
102    }
103
104
105
106# Convenience functions for extending the above mappings
107def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
108    """Add character set properties to the global registry.
109
110    charset is the input character set, and must be the canonical name of a
111    character set.
112
113    Optional header_enc and body_enc is either Charset.QP for
114    quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
115    the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
116    is only valid for header_enc.  It describes how message headers and
117    message bodies in the input charset are to be encoded.  Default is no
118    encoding.
119
120    Optional output_charset is the character set that the output should be
121    in.  Conversions will proceed from input charset, to Unicode, to the
122    output charset when the method Charset.convert() is called.  The default
123    is to output in the same character set as the input.
124
125    Both input_charset and output_charset must have Unicode codec entries in
126    the module's charset-to-codec mapping; use add_codec(charset, codecname)
127    to add codecs the module does not know about.  See the codecs module's
128    documentation for more information.
129    """
130    if body_enc == SHORTEST:
131        raise ValueError('SHORTEST not allowed for body_enc')
132    CHARSETS[charset] = (header_enc, body_enc, output_charset)
133
134
135def add_alias(alias, canonical):
136    """Add a character set alias.
137
138    alias is the alias name, e.g. latin-1
139    canonical is the character set's canonical name, e.g. iso-8859-1
140    """
141    ALIASES[alias] = canonical
142
143
144def add_codec(charset, codecname):
145    """Add a codec that map characters in the given charset to/from Unicode.
146
147    charset is the canonical name of a character set.  codecname is the name
148    of a Python codec, as appropriate for the second argument to the unicode()
149    built-in, or to the encode() method of a Unicode string.
150    """
151    CODEC_MAP[charset] = codecname
152
153
154
155class Charset:
156    """Map character sets to their email properties.
157
158    This class provides information about the requirements imposed on email
159    for a specific character set.  It also provides convenience routines for
160    converting between character sets, given the availability of the
161    applicable codecs.  Given a character set, it will do its best to provide
162    information on how to use that character set in an email in an
163    RFC-compliant way.
164
165    Certain character sets must be encoded with quoted-printable or base64
166    when used in email headers or bodies.  Certain character sets must be
167    converted outright, and are not allowed in email.  Instances of this
168    module expose the following information about a character set:
169
170    input_charset: The initial character set specified.  Common aliases
171                   are converted to their `official' email names (e.g. latin_1
172                   is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
173
174    header_encoding: If the character set must be encoded before it can be
175                     used in an email header, this attribute will be set to
176                     Charset.QP (for quoted-printable), Charset.BASE64 (for
177                     base64 encoding), or Charset.SHORTEST for the shortest of
178                     QP or BASE64 encoding.  Otherwise, it will be None.
179
180    body_encoding: Same as header_encoding, but describes the encoding for the
181                   mail message's body, which indeed may be different than the
182                   header encoding.  Charset.SHORTEST is not allowed for
183                   body_encoding.
184
185    output_charset: Some character sets must be converted before the can be
186                    used in email headers or bodies.  If the input_charset is
187                    one of them, this attribute will contain the name of the
188                    charset output will be converted to.  Otherwise, it will
189                    be None.
190
191    input_codec: The name of the Python codec used to convert the
192                 input_charset to Unicode.  If no conversion codec is
193                 necessary, this attribute will be None.
194
195    output_codec: The name of the Python codec used to convert Unicode
196                  to the output_charset.  If no conversion codec is necessary,
197                  this attribute will have the same value as the input_codec.
198    """
199    def __init__(self, input_charset=DEFAULT_CHARSET):
200        # RFC 2046, $4.1.2 says charsets are not case sensitive.  We coerce to
201        # unicode because its .lower() is locale insensitive.  If the argument
202        # is already a unicode, we leave it at that, but ensure that the
203        # charset is ASCII, as the standard (RFC XXX) requires.
204        try:
205            if isinstance(input_charset, unicode):
206                input_charset.encode('ascii')
207            else:
208                input_charset = unicode(input_charset, 'ascii')
209        except UnicodeError:
210            raise errors.CharsetError(input_charset)
211        input_charset = input_charset.lower()
212        # Set the input charset after filtering through the aliases
213        self.input_charset = ALIASES.get(input_charset, input_charset)
214        # We can try to guess which encoding and conversion to use by the
215        # charset_map dictionary.  Try that first, but let the user override
216        # it.
217        henc, benc, conv = CHARSETS.get(self.input_charset,
218                                        (SHORTEST, BASE64, None))
219        if not conv:
220            conv = self.input_charset
221        # Set the attributes, allowing the arguments to override the default.
222        self.header_encoding = henc
223        self.body_encoding = benc
224        self.output_charset = ALIASES.get(conv, conv)
225        # Now set the codecs.  If one isn't defined for input_charset,
226        # guess and try a Unicode codec with the same name as input_codec.
227        self.input_codec = CODEC_MAP.get(self.input_charset,
228                                         self.input_charset)
229        self.output_codec = CODEC_MAP.get(self.output_charset,
230                                          self.output_charset)
231
232    def __str__(self):
233        return self.input_charset.lower()
234
235    __repr__ = __str__
236
237    def __eq__(self, other):
238        return str(self) == str(other).lower()
239
240    def __ne__(self, other):
241        return not self.__eq__(other)
242
243    def get_body_encoding(self):
244        """Return the content-transfer-encoding used for body encoding.
245
246        This is either the string `quoted-printable' or `base64' depending on
247        the encoding used, or it is a function in which case you should call
248        the function with a single argument, the Message object being
249        encoded.  The function should then set the Content-Transfer-Encoding
250        header itself to whatever is appropriate.
251
252        Returns "quoted-printable" if self.body_encoding is QP.
253        Returns "base64" if self.body_encoding is BASE64.
254        Returns "7bit" otherwise.
255        """
256        assert self.body_encoding != SHORTEST
257        if self.body_encoding == QP:
258            return 'quoted-printable'
259        elif self.body_encoding == BASE64:
260            return 'base64'
261        else:
262            return encode_7or8bit
263
264    def convert(self, s):
265        """Convert a string from the input_codec to the output_codec."""
266        if self.input_codec != self.output_codec:
267            return unicode(s, self.input_codec).encode(self.output_codec)
268        else:
269            return s
270
271    def to_splittable(self, s):
272        """Convert a possibly multibyte string to a safely splittable format.
273
274        Uses the input_codec to try and convert the string to Unicode, so it
275        can be safely split on character boundaries (even for multibyte
276        characters).
277
278        Returns the string as-is if it isn't known how to convert it to
279        Unicode with the input_charset.
280
281        Characters that could not be converted to Unicode will be replaced
282        with the Unicode replacement character U+FFFD.
283        """
284        if isinstance(s, unicode) or self.input_codec is None:
285            return s
286        try:
287            return unicode(s, self.input_codec, 'replace')
288        except LookupError:
289            # Input codec not installed on system, so return the original
290            # string unchanged.
291            return s
292
293    def from_splittable(self, ustr, to_output=True):
294        """Convert a splittable string back into an encoded string.
295
296        Uses the proper codec to try and convert the string from Unicode back
297        into an encoded format.  Return the string as-is if it is not Unicode,
298        or if it could not be converted from Unicode.
299
300        Characters that could not be converted from Unicode will be replaced
301        with an appropriate character (usually '?').
302
303        If to_output is True (the default), uses output_codec to convert to an
304        encoded format.  If to_output is False, uses input_codec.
305        """
306        if to_output:
307            codec = self.output_codec
308        else:
309            codec = self.input_codec
310        if not isinstance(ustr, unicode) or codec is None:
311            return ustr
312        try:
313            return ustr.encode(codec, 'replace')
314        except LookupError:
315            # Output codec not installed
316            return ustr
317
318    def get_output_charset(self):
319        """Return the output character set.
320
321        This is self.output_charset if that is not None, otherwise it is
322        self.input_charset.
323        """
324        return self.output_charset or self.input_charset
325
326    def encoded_header_len(self, s):
327        """Return the length of the encoded header string."""
328        cset = self.get_output_charset()
329        # The len(s) of a 7bit encoding is len(s)
330        if self.header_encoding == BASE64:
331            return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
332        elif self.header_encoding == QP:
333            return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
334        elif self.header_encoding == SHORTEST:
335            lenb64 = email.base64mime.base64_len(s)
336            lenqp = email.quoprimime.header_quopri_len(s)
337            return min(lenb64, lenqp) + len(cset) + MISC_LEN
338        else:
339            return len(s)
340
341    def header_encode(self, s, convert=False):
342        """Header-encode a string, optionally converting it to output_charset.
343
344        If convert is True, the string will be converted from the input
345        charset to the output charset automatically.  This is not useful for
346        multibyte character sets, which have line length issues (multibyte
347        characters must be split on a character, not a byte boundary); use the
348        high-level Header class to deal with these issues.  convert defaults
349        to False.
350
351        The type of encoding (base64 or quoted-printable) will be based on
352        self.header_encoding.
353        """
354        cset = self.get_output_charset()
355        if convert:
356            s = self.convert(s)
357        # 7bit/8bit encodings return the string unchanged (modulo conversions)
358        if self.header_encoding == BASE64:
359            return email.base64mime.header_encode(s, cset)
360        elif self.header_encoding == QP:
361            return email.quoprimime.header_encode(s, cset, maxlinelen=None)
362        elif self.header_encoding == SHORTEST:
363            lenb64 = email.base64mime.base64_len(s)
364            lenqp = email.quoprimime.header_quopri_len(s)
365            if lenb64 < lenqp:
366                return email.base64mime.header_encode(s, cset)
367            else:
368                return email.quoprimime.header_encode(s, cset, maxlinelen=None)
369        else:
370            return s
371
372    def body_encode(self, s, convert=True):
373        """Body-encode a string and convert it to output_charset.
374
375        If convert is True (the default), the string will be converted from
376        the input charset to output charset automatically.  Unlike
377        header_encode(), there are no issues with byte boundaries and
378        multibyte charsets in email bodies, so this is usually pretty safe.
379
380        The type of encoding (base64 or quoted-printable) will be based on
381        self.body_encoding.
382        """
383        if convert:
384            s = self.convert(s)
385        # 7bit/8bit encodings return the string unchanged (module conversions)
386        if self.body_encoding is BASE64:
387            return email.base64mime.body_encode(s)
388        elif self.body_encoding is QP:
389            return email.quoprimime.body_encode(s)
390        else:
391            return s