/Lib/email/charset.py
Python | 391 lines | 334 code | 14 blank | 43 comment | 3 complexity | 7e618fcb0b062f2deb396f7bdbdd1177 MD5 | raw file
1# Copyright (C) 2001-2006 Python Software Foundation 2# Author: Ben Gertzfield, Barry Warsaw 3# Contact: email-sig@python.org 4 5__all__ = [ 6 'Charset', 7 'add_alias', 8 'add_charset', 9 'add_codec', 10 ] 11 12import email.base64mime 13import email.quoprimime 14 15from email import errors 16from email.encoders import encode_7or8bit 17 18 19 20# Flags for types of header encodings 21QP = 1 # Quoted-Printable 22BASE64 = 2 # Base64 23SHORTEST = 3 # the shorter of QP and base64, but only for headers 24 25# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 26MISC_LEN = 7 27 28DEFAULT_CHARSET = 'us-ascii' 29 30 31 32# Defaults 33CHARSETS = { 34 # input header enc body enc output conv 35 'iso-8859-1': (QP, QP, None), 36 'iso-8859-2': (QP, QP, None), 37 'iso-8859-3': (QP, QP, None), 38 'iso-8859-4': (QP, QP, None), 39 # iso-8859-5 is Cyrillic, and not especially used 40 # iso-8859-6 is Arabic, also not particularly used 41 # iso-8859-7 is Greek, QP will not make it readable 42 # iso-8859-8 is Hebrew, QP will not make it readable 43 'iso-8859-9': (QP, QP, None), 44 'iso-8859-10': (QP, QP, None), 45 # iso-8859-11 is Thai, QP will not make it readable 46 'iso-8859-13': (QP, QP, None), 47 'iso-8859-14': (QP, QP, None), 48 'iso-8859-15': (QP, QP, None), 49 'iso-8859-16': (QP, QP, None), 50 'windows-1252':(QP, QP, None), 51 'viscii': (QP, QP, None), 52 'us-ascii': (None, None, None), 53 'big5': (BASE64, BASE64, None), 54 'gb2312': (BASE64, BASE64, None), 55 'euc-jp': (BASE64, None, 'iso-2022-jp'), 56 'shift_jis': (BASE64, None, 'iso-2022-jp'), 57 'iso-2022-jp': (BASE64, None, None), 58 'koi8-r': (BASE64, BASE64, None), 59 'utf-8': (SHORTEST, BASE64, 'utf-8'), 60 # We're making this one up to represent raw unencoded 8-bit 61 '8bit': (None, BASE64, 'utf-8'), 62 } 63 64# Aliases for other commonly-used names for character sets. Map 65# them to the real ones used in email. 66ALIASES = { 67 'latin_1': 'iso-8859-1', 68 'latin-1': 'iso-8859-1', 69 'latin_2': 'iso-8859-2', 70 'latin-2': 'iso-8859-2', 71 'latin_3': 'iso-8859-3', 72 'latin-3': 'iso-8859-3', 73 'latin_4': 'iso-8859-4', 74 'latin-4': 'iso-8859-4', 75 'latin_5': 'iso-8859-9', 76 'latin-5': 'iso-8859-9', 77 'latin_6': 'iso-8859-10', 78 'latin-6': 'iso-8859-10', 79 'latin_7': 'iso-8859-13', 80 'latin-7': 'iso-8859-13', 81 'latin_8': 'iso-8859-14', 82 'latin-8': 'iso-8859-14', 83 'latin_9': 'iso-8859-15', 84 'latin-9': 'iso-8859-15', 85 'latin_10':'iso-8859-16', 86 'latin-10':'iso-8859-16', 87 'cp949': 'ks_c_5601-1987', 88 'euc_jp': 'euc-jp', 89 'euc_kr': 'euc-kr', 90 'ascii': 'us-ascii', 91 } 92 93 94# Map charsets to their Unicode codec strings. 95CODEC_MAP = { 96 'gb2312': 'eucgb2312_cn', 97 'big5': 'big5_tw', 98 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all 99 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. 100 # Let that stuff pass through without conversion to/from Unicode. 101 'us-ascii': None, 102 } 103 104 105 106# Convenience functions for extending the above mappings 107def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): 108 """Add character set properties to the global registry. 109 110 charset is the input character set, and must be the canonical name of a 111 character set. 112 113 Optional header_enc and body_enc is either Charset.QP for 114 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for 115 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST 116 is only valid for header_enc. It describes how message headers and 117 message bodies in the input charset are to be encoded. Default is no 118 encoding. 119 120 Optional output_charset is the character set that the output should be 121 in. Conversions will proceed from input charset, to Unicode, to the 122 output charset when the method Charset.convert() is called. The default 123 is to output in the same character set as the input. 124 125 Both input_charset and output_charset must have Unicode codec entries in 126 the module's charset-to-codec mapping; use add_codec(charset, codecname) 127 to add codecs the module does not know about. See the codecs module's 128 documentation for more information. 129 """ 130 if body_enc == SHORTEST: 131 raise ValueError('SHORTEST not allowed for body_enc') 132 CHARSETS[charset] = (header_enc, body_enc, output_charset) 133 134 135def add_alias(alias, canonical): 136 """Add a character set alias. 137 138 alias is the alias name, e.g. latin-1 139 canonical is the character set's canonical name, e.g. iso-8859-1 140 """ 141 ALIASES[alias] = canonical 142 143 144def add_codec(charset, codecname): 145 """Add a codec that map characters in the given charset to/from Unicode. 146 147 charset is the canonical name of a character set. codecname is the name 148 of a Python codec, as appropriate for the second argument to the unicode() 149 built-in, or to the encode() method of a Unicode string. 150 """ 151 CODEC_MAP[charset] = codecname 152 153 154 155class Charset: 156 """Map character sets to their email properties. 157 158 This class provides information about the requirements imposed on email 159 for a specific character set. It also provides convenience routines for 160 converting between character sets, given the availability of the 161 applicable codecs. Given a character set, it will do its best to provide 162 information on how to use that character set in an email in an 163 RFC-compliant way. 164 165 Certain character sets must be encoded with quoted-printable or base64 166 when used in email headers or bodies. Certain character sets must be 167 converted outright, and are not allowed in email. Instances of this 168 module expose the following information about a character set: 169 170 input_charset: The initial character set specified. Common aliases 171 are converted to their `official' email names (e.g. latin_1 172 is converted to iso-8859-1). Defaults to 7-bit us-ascii. 173 174 header_encoding: If the character set must be encoded before it can be 175 used in an email header, this attribute will be set to 176 Charset.QP (for quoted-printable), Charset.BASE64 (for 177 base64 encoding), or Charset.SHORTEST for the shortest of 178 QP or BASE64 encoding. Otherwise, it will be None. 179 180 body_encoding: Same as header_encoding, but describes the encoding for the 181 mail message's body, which indeed may be different than the 182 header encoding. Charset.SHORTEST is not allowed for 183 body_encoding. 184 185 output_charset: Some character sets must be converted before the can be 186 used in email headers or bodies. If the input_charset is 187 one of them, this attribute will contain the name of the 188 charset output will be converted to. Otherwise, it will 189 be None. 190 191 input_codec: The name of the Python codec used to convert the 192 input_charset to Unicode. If no conversion codec is 193 necessary, this attribute will be None. 194 195 output_codec: The name of the Python codec used to convert Unicode 196 to the output_charset. If no conversion codec is necessary, 197 this attribute will have the same value as the input_codec. 198 """ 199 def __init__(self, input_charset=DEFAULT_CHARSET): 200 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to 201 # unicode because its .lower() is locale insensitive. If the argument 202 # is already a unicode, we leave it at that, but ensure that the 203 # charset is ASCII, as the standard (RFC XXX) requires. 204 try: 205 if isinstance(input_charset, unicode): 206 input_charset.encode('ascii') 207 else: 208 input_charset = unicode(input_charset, 'ascii') 209 except UnicodeError: 210 raise errors.CharsetError(input_charset) 211 input_charset = input_charset.lower() 212 # Set the input charset after filtering through the aliases 213 self.input_charset = ALIASES.get(input_charset, input_charset) 214 # We can try to guess which encoding and conversion to use by the 215 # charset_map dictionary. Try that first, but let the user override 216 # it. 217 henc, benc, conv = CHARSETS.get(self.input_charset, 218 (SHORTEST, BASE64, None)) 219 if not conv: 220 conv = self.input_charset 221 # Set the attributes, allowing the arguments to override the default. 222 self.header_encoding = henc 223 self.body_encoding = benc 224 self.output_charset = ALIASES.get(conv, conv) 225 # Now set the codecs. If one isn't defined for input_charset, 226 # guess and try a Unicode codec with the same name as input_codec. 227 self.input_codec = CODEC_MAP.get(self.input_charset, 228 self.input_charset) 229 self.output_codec = CODEC_MAP.get(self.output_charset, 230 self.output_charset) 231 232 def __str__(self): 233 return self.input_charset.lower() 234 235 __repr__ = __str__ 236 237 def __eq__(self, other): 238 return str(self) == str(other).lower() 239 240 def __ne__(self, other): 241 return not self.__eq__(other) 242 243 def get_body_encoding(self): 244 """Return the content-transfer-encoding used for body encoding. 245 246 This is either the string `quoted-printable' or `base64' depending on 247 the encoding used, or it is a function in which case you should call 248 the function with a single argument, the Message object being 249 encoded. The function should then set the Content-Transfer-Encoding 250 header itself to whatever is appropriate. 251 252 Returns "quoted-printable" if self.body_encoding is QP. 253 Returns "base64" if self.body_encoding is BASE64. 254 Returns "7bit" otherwise. 255 """ 256 assert self.body_encoding != SHORTEST 257 if self.body_encoding == QP: 258 return 'quoted-printable' 259 elif self.body_encoding == BASE64: 260 return 'base64' 261 else: 262 return encode_7or8bit 263 264 def convert(self, s): 265 """Convert a string from the input_codec to the output_codec.""" 266 if self.input_codec != self.output_codec: 267 return unicode(s, self.input_codec).encode(self.output_codec) 268 else: 269 return s 270 271 def to_splittable(self, s): 272 """Convert a possibly multibyte string to a safely splittable format. 273 274 Uses the input_codec to try and convert the string to Unicode, so it 275 can be safely split on character boundaries (even for multibyte 276 characters). 277 278 Returns the string as-is if it isn't known how to convert it to 279 Unicode with the input_charset. 280 281 Characters that could not be converted to Unicode will be replaced 282 with the Unicode replacement character U+FFFD. 283 """ 284 if isinstance(s, unicode) or self.input_codec is None: 285 return s 286 try: 287 return unicode(s, self.input_codec, 'replace') 288 except LookupError: 289 # Input codec not installed on system, so return the original 290 # string unchanged. 291 return s 292 293 def from_splittable(self, ustr, to_output=True): 294 """Convert a splittable string back into an encoded string. 295 296 Uses the proper codec to try and convert the string from Unicode back 297 into an encoded format. Return the string as-is if it is not Unicode, 298 or if it could not be converted from Unicode. 299 300 Characters that could not be converted from Unicode will be replaced 301 with an appropriate character (usually '?'). 302 303 If to_output is True (the default), uses output_codec to convert to an 304 encoded format. If to_output is False, uses input_codec. 305 """ 306 if to_output: 307 codec = self.output_codec 308 else: 309 codec = self.input_codec 310 if not isinstance(ustr, unicode) or codec is None: 311 return ustr 312 try: 313 return ustr.encode(codec, 'replace') 314 except LookupError: 315 # Output codec not installed 316 return ustr 317 318 def get_output_charset(self): 319 """Return the output character set. 320 321 This is self.output_charset if that is not None, otherwise it is 322 self.input_charset. 323 """ 324 return self.output_charset or self.input_charset 325 326 def encoded_header_len(self, s): 327 """Return the length of the encoded header string.""" 328 cset = self.get_output_charset() 329 # The len(s) of a 7bit encoding is len(s) 330 if self.header_encoding == BASE64: 331 return email.base64mime.base64_len(s) + len(cset) + MISC_LEN 332 elif self.header_encoding == QP: 333 return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN 334 elif self.header_encoding == SHORTEST: 335 lenb64 = email.base64mime.base64_len(s) 336 lenqp = email.quoprimime.header_quopri_len(s) 337 return min(lenb64, lenqp) + len(cset) + MISC_LEN 338 else: 339 return len(s) 340 341 def header_encode(self, s, convert=False): 342 """Header-encode a string, optionally converting it to output_charset. 343 344 If convert is True, the string will be converted from the input 345 charset to the output charset automatically. This is not useful for 346 multibyte character sets, which have line length issues (multibyte 347 characters must be split on a character, not a byte boundary); use the 348 high-level Header class to deal with these issues. convert defaults 349 to False. 350 351 The type of encoding (base64 or quoted-printable) will be based on 352 self.header_encoding. 353 """ 354 cset = self.get_output_charset() 355 if convert: 356 s = self.convert(s) 357 # 7bit/8bit encodings return the string unchanged (modulo conversions) 358 if self.header_encoding == BASE64: 359 return email.base64mime.header_encode(s, cset) 360 elif self.header_encoding == QP: 361 return email.quoprimime.header_encode(s, cset, maxlinelen=None) 362 elif self.header_encoding == SHORTEST: 363 lenb64 = email.base64mime.base64_len(s) 364 lenqp = email.quoprimime.header_quopri_len(s) 365 if lenb64 < lenqp: 366 return email.base64mime.header_encode(s, cset) 367 else: 368 return email.quoprimime.header_encode(s, cset, maxlinelen=None) 369 else: 370 return s 371 372 def body_encode(self, s, convert=True): 373 """Body-encode a string and convert it to output_charset. 374 375 If convert is True (the default), the string will be converted from 376 the input charset to output charset automatically. Unlike 377 header_encode(), there are no issues with byte boundaries and 378 multibyte charsets in email bodies, so this is usually pretty safe. 379 380 The type of encoding (base64 or quoted-printable) will be based on 381 self.body_encoding. 382 """ 383 if convert: 384 s = self.convert(s) 385 # 7bit/8bit encodings return the string unchanged (module conversions) 386 if self.body_encoding is BASE64: 387 return email.base64mime.body_encode(s) 388 elif self.body_encoding is QP: 389 return email.quoprimime.body_encode(s) 390 else: 391 return s