/Lib/mailbox.py

http://unladen-swallow.googlecode.com/ · Python · 2119 lines · 1954 code · 93 blank · 72 comment · 163 complexity · 156d685014555e10485133be4fa899d2 MD5 · raw file

Large files are truncated click here to view the full file

  1. #! /usr/bin/env python
  2. """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
  3. # Notes for authors of new mailbox subclasses:
  4. #
  5. # Remember to fsync() changes to disk before closing a modified file
  6. # or returning from a flush() method. See functions _sync_flush() and
  7. # _sync_close().
  8. import sys
  9. import os
  10. import time
  11. import calendar
  12. import socket
  13. import errno
  14. import copy
  15. import email
  16. import email.message
  17. import email.generator
  18. import rfc822
  19. import StringIO
  20. try:
  21. if sys.platform == 'os2emx':
  22. # OS/2 EMX fcntl() not adequate
  23. raise ImportError
  24. import fcntl
  25. except ImportError:
  26. fcntl = None
  27. __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  28. 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  29. 'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  30. 'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  31. class Mailbox:
  32. """A group of messages in a particular place."""
  33. def __init__(self, path, factory=None, create=True):
  34. """Initialize a Mailbox instance."""
  35. self._path = os.path.abspath(os.path.expanduser(path))
  36. self._factory = factory
  37. def add(self, message):
  38. """Add message and return assigned key."""
  39. raise NotImplementedError('Method must be implemented by subclass')
  40. def remove(self, key):
  41. """Remove the keyed message; raise KeyError if it doesn't exist."""
  42. raise NotImplementedError('Method must be implemented by subclass')
  43. def __delitem__(self, key):
  44. self.remove(key)
  45. def discard(self, key):
  46. """If the keyed message exists, remove it."""
  47. try:
  48. self.remove(key)
  49. except KeyError:
  50. pass
  51. def __setitem__(self, key, message):
  52. """Replace the keyed message; raise KeyError if it doesn't exist."""
  53. raise NotImplementedError('Method must be implemented by subclass')
  54. def get(self, key, default=None):
  55. """Return the keyed message, or default if it doesn't exist."""
  56. try:
  57. return self.__getitem__(key)
  58. except KeyError:
  59. return default
  60. def __getitem__(self, key):
  61. """Return the keyed message; raise KeyError if it doesn't exist."""
  62. if not self._factory:
  63. return self.get_message(key)
  64. else:
  65. return self._factory(self.get_file(key))
  66. def get_message(self, key):
  67. """Return a Message representation or raise a KeyError."""
  68. raise NotImplementedError('Method must be implemented by subclass')
  69. def get_string(self, key):
  70. """Return a string representation or raise a KeyError."""
  71. raise NotImplementedError('Method must be implemented by subclass')
  72. def get_file(self, key):
  73. """Return a file-like representation or raise a KeyError."""
  74. raise NotImplementedError('Method must be implemented by subclass')
  75. def iterkeys(self):
  76. """Return an iterator over keys."""
  77. raise NotImplementedError('Method must be implemented by subclass')
  78. def keys(self):
  79. """Return a list of keys."""
  80. return list(self.iterkeys())
  81. def itervalues(self):
  82. """Return an iterator over all messages."""
  83. for key in self.iterkeys():
  84. try:
  85. value = self[key]
  86. except KeyError:
  87. continue
  88. yield value
  89. def __iter__(self):
  90. return self.itervalues()
  91. def values(self):
  92. """Return a list of messages. Memory intensive."""
  93. return list(self.itervalues())
  94. def iteritems(self):
  95. """Return an iterator over (key, message) tuples."""
  96. for key in self.iterkeys():
  97. try:
  98. value = self[key]
  99. except KeyError:
  100. continue
  101. yield (key, value)
  102. def items(self):
  103. """Return a list of (key, message) tuples. Memory intensive."""
  104. return list(self.iteritems())
  105. def has_key(self, key):
  106. """Return True if the keyed message exists, False otherwise."""
  107. raise NotImplementedError('Method must be implemented by subclass')
  108. def __contains__(self, key):
  109. return self.has_key(key)
  110. def __len__(self):
  111. """Return a count of messages in the mailbox."""
  112. raise NotImplementedError('Method must be implemented by subclass')
  113. def clear(self):
  114. """Delete all messages."""
  115. for key in self.iterkeys():
  116. self.discard(key)
  117. def pop(self, key, default=None):
  118. """Delete the keyed message and return it, or default."""
  119. try:
  120. result = self[key]
  121. except KeyError:
  122. return default
  123. self.discard(key)
  124. return result
  125. def popitem(self):
  126. """Delete an arbitrary (key, message) pair and return it."""
  127. for key in self.iterkeys():
  128. return (key, self.pop(key)) # This is only run once.
  129. else:
  130. raise KeyError('No messages in mailbox')
  131. def update(self, arg=None):
  132. """Change the messages that correspond to certain keys."""
  133. if hasattr(arg, 'iteritems'):
  134. source = arg.iteritems()
  135. elif hasattr(arg, 'items'):
  136. source = arg.items()
  137. else:
  138. source = arg
  139. bad_key = False
  140. for key, message in source:
  141. try:
  142. self[key] = message
  143. except KeyError:
  144. bad_key = True
  145. if bad_key:
  146. raise KeyError('No message with key(s)')
  147. def flush(self):
  148. """Write any pending changes to the disk."""
  149. raise NotImplementedError('Method must be implemented by subclass')
  150. def lock(self):
  151. """Lock the mailbox."""
  152. raise NotImplementedError('Method must be implemented by subclass')
  153. def unlock(self):
  154. """Unlock the mailbox if it is locked."""
  155. raise NotImplementedError('Method must be implemented by subclass')
  156. def close(self):
  157. """Flush and close the mailbox."""
  158. raise NotImplementedError('Method must be implemented by subclass')
  159. def _dump_message(self, message, target, mangle_from_=False):
  160. # Most files are opened in binary mode to allow predictable seeking.
  161. # To get native line endings on disk, the user-friendly \n line endings
  162. # used in strings and by email.Message are translated here.
  163. """Dump message contents to target file."""
  164. if isinstance(message, email.message.Message):
  165. buffer = StringIO.StringIO()
  166. gen = email.generator.Generator(buffer, mangle_from_, 0)
  167. gen.flatten(message)
  168. buffer.seek(0)
  169. target.write(buffer.read().replace('\n', os.linesep))
  170. elif isinstance(message, str):
  171. if mangle_from_:
  172. message = message.replace('\nFrom ', '\n>From ')
  173. message = message.replace('\n', os.linesep)
  174. target.write(message)
  175. elif hasattr(message, 'read'):
  176. while True:
  177. line = message.readline()
  178. if line == '':
  179. break
  180. if mangle_from_ and line.startswith('From '):
  181. line = '>From ' + line[5:]
  182. line = line.replace('\n', os.linesep)
  183. target.write(line)
  184. else:
  185. raise TypeError('Invalid message type: %s' % type(message))
  186. class Maildir(Mailbox):
  187. """A qmail-style Maildir mailbox."""
  188. colon = ':'
  189. def __init__(self, dirname, factory=rfc822.Message, create=True):
  190. """Initialize a Maildir instance."""
  191. Mailbox.__init__(self, dirname, factory, create)
  192. if not os.path.exists(self._path):
  193. if create:
  194. os.mkdir(self._path, 0700)
  195. os.mkdir(os.path.join(self._path, 'tmp'), 0700)
  196. os.mkdir(os.path.join(self._path, 'new'), 0700)
  197. os.mkdir(os.path.join(self._path, 'cur'), 0700)
  198. else:
  199. raise NoSuchMailboxError(self._path)
  200. self._toc = {}
  201. def add(self, message):
  202. """Add message and return assigned key."""
  203. tmp_file = self._create_tmp()
  204. try:
  205. self._dump_message(message, tmp_file)
  206. finally:
  207. _sync_close(tmp_file)
  208. if isinstance(message, MaildirMessage):
  209. subdir = message.get_subdir()
  210. suffix = self.colon + message.get_info()
  211. if suffix == self.colon:
  212. suffix = ''
  213. else:
  214. subdir = 'new'
  215. suffix = ''
  216. uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
  217. dest = os.path.join(self._path, subdir, uniq + suffix)
  218. try:
  219. if hasattr(os, 'link'):
  220. os.link(tmp_file.name, dest)
  221. os.remove(tmp_file.name)
  222. else:
  223. os.rename(tmp_file.name, dest)
  224. except OSError, e:
  225. os.remove(tmp_file.name)
  226. if e.errno == errno.EEXIST:
  227. raise ExternalClashError('Name clash with existing message: %s'
  228. % dest)
  229. else:
  230. raise
  231. if isinstance(message, MaildirMessage):
  232. os.utime(dest, (os.path.getatime(dest), message.get_date()))
  233. return uniq
  234. def remove(self, key):
  235. """Remove the keyed message; raise KeyError if it doesn't exist."""
  236. os.remove(os.path.join(self._path, self._lookup(key)))
  237. def discard(self, key):
  238. """If the keyed message exists, remove it."""
  239. # This overrides an inapplicable implementation in the superclass.
  240. try:
  241. self.remove(key)
  242. except KeyError:
  243. pass
  244. except OSError, e:
  245. if e.errno != errno.ENOENT:
  246. raise
  247. def __setitem__(self, key, message):
  248. """Replace the keyed message; raise KeyError if it doesn't exist."""
  249. old_subpath = self._lookup(key)
  250. temp_key = self.add(message)
  251. temp_subpath = self._lookup(temp_key)
  252. if isinstance(message, MaildirMessage):
  253. # temp's subdir and suffix were specified by message.
  254. dominant_subpath = temp_subpath
  255. else:
  256. # temp's subdir and suffix were defaults from add().
  257. dominant_subpath = old_subpath
  258. subdir = os.path.dirname(dominant_subpath)
  259. if self.colon in dominant_subpath:
  260. suffix = self.colon + dominant_subpath.split(self.colon)[-1]
  261. else:
  262. suffix = ''
  263. self.discard(key)
  264. new_path = os.path.join(self._path, subdir, key + suffix)
  265. os.rename(os.path.join(self._path, temp_subpath), new_path)
  266. if isinstance(message, MaildirMessage):
  267. os.utime(new_path, (os.path.getatime(new_path),
  268. message.get_date()))
  269. def get_message(self, key):
  270. """Return a Message representation or raise a KeyError."""
  271. subpath = self._lookup(key)
  272. f = open(os.path.join(self._path, subpath), 'r')
  273. try:
  274. if self._factory:
  275. msg = self._factory(f)
  276. else:
  277. msg = MaildirMessage(f)
  278. finally:
  279. f.close()
  280. subdir, name = os.path.split(subpath)
  281. msg.set_subdir(subdir)
  282. if self.colon in name:
  283. msg.set_info(name.split(self.colon)[-1])
  284. msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
  285. return msg
  286. def get_string(self, key):
  287. """Return a string representation or raise a KeyError."""
  288. f = open(os.path.join(self._path, self._lookup(key)), 'r')
  289. try:
  290. return f.read()
  291. finally:
  292. f.close()
  293. def get_file(self, key):
  294. """Return a file-like representation or raise a KeyError."""
  295. f = open(os.path.join(self._path, self._lookup(key)), 'rb')
  296. return _ProxyFile(f)
  297. def iterkeys(self):
  298. """Return an iterator over keys."""
  299. self._refresh()
  300. for key in self._toc:
  301. try:
  302. self._lookup(key)
  303. except KeyError:
  304. continue
  305. yield key
  306. def has_key(self, key):
  307. """Return True if the keyed message exists, False otherwise."""
  308. self._refresh()
  309. return key in self._toc
  310. def __len__(self):
  311. """Return a count of messages in the mailbox."""
  312. self._refresh()
  313. return len(self._toc)
  314. def flush(self):
  315. """Write any pending changes to disk."""
  316. return # Maildir changes are always written immediately.
  317. def lock(self):
  318. """Lock the mailbox."""
  319. return
  320. def unlock(self):
  321. """Unlock the mailbox if it is locked."""
  322. return
  323. def close(self):
  324. """Flush and close the mailbox."""
  325. return
  326. def list_folders(self):
  327. """Return a list of folder names."""
  328. result = []
  329. for entry in os.listdir(self._path):
  330. if len(entry) > 1 and entry[0] == '.' and \
  331. os.path.isdir(os.path.join(self._path, entry)):
  332. result.append(entry[1:])
  333. return result
  334. def get_folder(self, folder):
  335. """Return a Maildir instance for the named folder."""
  336. return Maildir(os.path.join(self._path, '.' + folder),
  337. factory=self._factory,
  338. create=False)
  339. def add_folder(self, folder):
  340. """Create a folder and return a Maildir instance representing it."""
  341. path = os.path.join(self._path, '.' + folder)
  342. result = Maildir(path, factory=self._factory)
  343. maildirfolder_path = os.path.join(path, 'maildirfolder')
  344. if not os.path.exists(maildirfolder_path):
  345. os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY,
  346. 0666))
  347. return result
  348. def remove_folder(self, folder):
  349. """Delete the named folder, which must be empty."""
  350. path = os.path.join(self._path, '.' + folder)
  351. for entry in os.listdir(os.path.join(path, 'new')) + \
  352. os.listdir(os.path.join(path, 'cur')):
  353. if len(entry) < 1 or entry[0] != '.':
  354. raise NotEmptyError('Folder contains message(s): %s' % folder)
  355. for entry in os.listdir(path):
  356. if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
  357. os.path.isdir(os.path.join(path, entry)):
  358. raise NotEmptyError("Folder contains subdirectory '%s': %s" %
  359. (folder, entry))
  360. for root, dirs, files in os.walk(path, topdown=False):
  361. for entry in files:
  362. os.remove(os.path.join(root, entry))
  363. for entry in dirs:
  364. os.rmdir(os.path.join(root, entry))
  365. os.rmdir(path)
  366. def clean(self):
  367. """Delete old files in "tmp"."""
  368. now = time.time()
  369. for entry in os.listdir(os.path.join(self._path, 'tmp')):
  370. path = os.path.join(self._path, 'tmp', entry)
  371. if now - os.path.getatime(path) > 129600: # 60 * 60 * 36
  372. os.remove(path)
  373. _count = 1 # This is used to generate unique file names.
  374. def _create_tmp(self):
  375. """Create a file in the tmp subdirectory and open and return it."""
  376. now = time.time()
  377. hostname = socket.gethostname()
  378. if '/' in hostname:
  379. hostname = hostname.replace('/', r'\057')
  380. if ':' in hostname:
  381. hostname = hostname.replace(':', r'\072')
  382. uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
  383. Maildir._count, hostname)
  384. path = os.path.join(self._path, 'tmp', uniq)
  385. try:
  386. os.stat(path)
  387. except OSError, e:
  388. if e.errno == errno.ENOENT:
  389. Maildir._count += 1
  390. try:
  391. return _create_carefully(path)
  392. except OSError, e:
  393. if e.errno != errno.EEXIST:
  394. raise
  395. else:
  396. raise
  397. # Fall through to here if stat succeeded or open raised EEXIST.
  398. raise ExternalClashError('Name clash prevented file creation: %s' %
  399. path)
  400. def _refresh(self):
  401. """Update table of contents mapping."""
  402. self._toc = {}
  403. for subdir in ('new', 'cur'):
  404. subdir_path = os.path.join(self._path, subdir)
  405. for entry in os.listdir(subdir_path):
  406. p = os.path.join(subdir_path, entry)
  407. if os.path.isdir(p):
  408. continue
  409. uniq = entry.split(self.colon)[0]
  410. self._toc[uniq] = os.path.join(subdir, entry)
  411. def _lookup(self, key):
  412. """Use TOC to return subpath for given key, or raise a KeyError."""
  413. try:
  414. if os.path.exists(os.path.join(self._path, self._toc[key])):
  415. return self._toc[key]
  416. except KeyError:
  417. pass
  418. self._refresh()
  419. try:
  420. return self._toc[key]
  421. except KeyError:
  422. raise KeyError('No message with key: %s' % key)
  423. # This method is for backward compatibility only.
  424. def next(self):
  425. """Return the next message in a one-time iteration."""
  426. if not hasattr(self, '_onetime_keys'):
  427. self._onetime_keys = self.iterkeys()
  428. while True:
  429. try:
  430. return self[self._onetime_keys.next()]
  431. except StopIteration:
  432. return None
  433. except KeyError:
  434. continue
  435. class _singlefileMailbox(Mailbox):
  436. """A single-file mailbox."""
  437. def __init__(self, path, factory=None, create=True):
  438. """Initialize a single-file mailbox."""
  439. Mailbox.__init__(self, path, factory, create)
  440. try:
  441. f = open(self._path, 'rb+')
  442. except IOError, e:
  443. if e.errno == errno.ENOENT:
  444. if create:
  445. f = open(self._path, 'wb+')
  446. else:
  447. raise NoSuchMailboxError(self._path)
  448. elif e.errno == errno.EACCES:
  449. f = open(self._path, 'rb')
  450. else:
  451. raise
  452. self._file = f
  453. self._toc = None
  454. self._next_key = 0
  455. self._pending = False # No changes require rewriting the file.
  456. self._locked = False
  457. self._file_length = None # Used to record mailbox size
  458. def add(self, message):
  459. """Add message and return assigned key."""
  460. self._lookup()
  461. self._toc[self._next_key] = self._append_message(message)
  462. self._next_key += 1
  463. self._pending = True
  464. return self._next_key - 1
  465. def remove(self, key):
  466. """Remove the keyed message; raise KeyError if it doesn't exist."""
  467. self._lookup(key)
  468. del self._toc[key]
  469. self._pending = True
  470. def __setitem__(self, key, message):
  471. """Replace the keyed message; raise KeyError if it doesn't exist."""
  472. self._lookup(key)
  473. self._toc[key] = self._append_message(message)
  474. self._pending = True
  475. def iterkeys(self):
  476. """Return an iterator over keys."""
  477. self._lookup()
  478. for key in self._toc.keys():
  479. yield key
  480. def has_key(self, key):
  481. """Return True if the keyed message exists, False otherwise."""
  482. self._lookup()
  483. return key in self._toc
  484. def __len__(self):
  485. """Return a count of messages in the mailbox."""
  486. self._lookup()
  487. return len(self._toc)
  488. def lock(self):
  489. """Lock the mailbox."""
  490. if not self._locked:
  491. _lock_file(self._file)
  492. self._locked = True
  493. def unlock(self):
  494. """Unlock the mailbox if it is locked."""
  495. if self._locked:
  496. _unlock_file(self._file)
  497. self._locked = False
  498. def flush(self):
  499. """Write any pending changes to disk."""
  500. if not self._pending:
  501. return
  502. # In order to be writing anything out at all, self._toc must
  503. # already have been generated (and presumably has been modified
  504. # by adding or deleting an item).
  505. assert self._toc is not None
  506. # Check length of self._file; if it's changed, some other process
  507. # has modified the mailbox since we scanned it.
  508. self._file.seek(0, 2)
  509. cur_len = self._file.tell()
  510. if cur_len != self._file_length:
  511. raise ExternalClashError('Size of mailbox file changed '
  512. '(expected %i, found %i)' %
  513. (self._file_length, cur_len))
  514. new_file = _create_temporary(self._path)
  515. try:
  516. new_toc = {}
  517. self._pre_mailbox_hook(new_file)
  518. for key in sorted(self._toc.keys()):
  519. start, stop = self._toc[key]
  520. self._file.seek(start)
  521. self._pre_message_hook(new_file)
  522. new_start = new_file.tell()
  523. while True:
  524. buffer = self._file.read(min(4096,
  525. stop - self._file.tell()))
  526. if buffer == '':
  527. break
  528. new_file.write(buffer)
  529. new_toc[key] = (new_start, new_file.tell())
  530. self._post_message_hook(new_file)
  531. except:
  532. new_file.close()
  533. os.remove(new_file.name)
  534. raise
  535. _sync_close(new_file)
  536. # self._file is about to get replaced, so no need to sync.
  537. self._file.close()
  538. try:
  539. os.rename(new_file.name, self._path)
  540. except OSError, e:
  541. if e.errno == errno.EEXIST or \
  542. (os.name == 'os2' and e.errno == errno.EACCES):
  543. os.remove(self._path)
  544. os.rename(new_file.name, self._path)
  545. else:
  546. raise
  547. self._file = open(self._path, 'rb+')
  548. self._toc = new_toc
  549. self._pending = False
  550. if self._locked:
  551. _lock_file(self._file, dotlock=False)
  552. def _pre_mailbox_hook(self, f):
  553. """Called before writing the mailbox to file f."""
  554. return
  555. def _pre_message_hook(self, f):
  556. """Called before writing each message to file f."""
  557. return
  558. def _post_message_hook(self, f):
  559. """Called after writing each message to file f."""
  560. return
  561. def close(self):
  562. """Flush and close the mailbox."""
  563. self.flush()
  564. if self._locked:
  565. self.unlock()
  566. self._file.close() # Sync has been done by self.flush() above.
  567. def _lookup(self, key=None):
  568. """Return (start, stop) or raise KeyError."""
  569. if self._toc is None:
  570. self._generate_toc()
  571. if key is not None:
  572. try:
  573. return self._toc[key]
  574. except KeyError:
  575. raise KeyError('No message with key: %s' % key)
  576. def _append_message(self, message):
  577. """Append message to mailbox and return (start, stop) offsets."""
  578. self._file.seek(0, 2)
  579. self._pre_message_hook(self._file)
  580. offsets = self._install_message(message)
  581. self._post_message_hook(self._file)
  582. self._file.flush()
  583. self._file_length = self._file.tell() # Record current length of mailbox
  584. return offsets
  585. class _mboxMMDF(_singlefileMailbox):
  586. """An mbox or MMDF mailbox."""
  587. _mangle_from_ = True
  588. def get_message(self, key):
  589. """Return a Message representation or raise a KeyError."""
  590. start, stop = self._lookup(key)
  591. self._file.seek(start)
  592. from_line = self._file.readline().replace(os.linesep, '')
  593. string = self._file.read(stop - self._file.tell())
  594. msg = self._message_factory(string.replace(os.linesep, '\n'))
  595. msg.set_from(from_line[5:])
  596. return msg
  597. def get_string(self, key, from_=False):
  598. """Return a string representation or raise a KeyError."""
  599. start, stop = self._lookup(key)
  600. self._file.seek(start)
  601. if not from_:
  602. self._file.readline()
  603. string = self._file.read(stop - self._file.tell())
  604. return string.replace(os.linesep, '\n')
  605. def get_file(self, key, from_=False):
  606. """Return a file-like representation or raise a KeyError."""
  607. start, stop = self._lookup(key)
  608. self._file.seek(start)
  609. if not from_:
  610. self._file.readline()
  611. return _PartialFile(self._file, self._file.tell(), stop)
  612. def _install_message(self, message):
  613. """Format a message and blindly write to self._file."""
  614. from_line = None
  615. if isinstance(message, str) and message.startswith('From '):
  616. newline = message.find('\n')
  617. if newline != -1:
  618. from_line = message[:newline]
  619. message = message[newline + 1:]
  620. else:
  621. from_line = message
  622. message = ''
  623. elif isinstance(message, _mboxMMDFMessage):
  624. from_line = 'From ' + message.get_from()
  625. elif isinstance(message, email.message.Message):
  626. from_line = message.get_unixfrom() # May be None.
  627. if from_line is None:
  628. from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
  629. start = self._file.tell()
  630. self._file.write(from_line + os.linesep)
  631. self._dump_message(message, self._file, self._mangle_from_)
  632. stop = self._file.tell()
  633. return (start, stop)
  634. class mbox(_mboxMMDF):
  635. """A classic mbox mailbox."""
  636. _mangle_from_ = True
  637. def __init__(self, path, factory=None, create=True):
  638. """Initialize an mbox mailbox."""
  639. self._message_factory = mboxMessage
  640. _mboxMMDF.__init__(self, path, factory, create)
  641. def _pre_message_hook(self, f):
  642. """Called before writing each message to file f."""
  643. if f.tell() != 0:
  644. f.write(os.linesep)
  645. def _generate_toc(self):
  646. """Generate key-to-(start, stop) table of contents."""
  647. starts, stops = [], []
  648. self._file.seek(0)
  649. while True:
  650. line_pos = self._file.tell()
  651. line = self._file.readline()
  652. if line.startswith('From '):
  653. if len(stops) < len(starts):
  654. stops.append(line_pos - len(os.linesep))
  655. starts.append(line_pos)
  656. elif line == '':
  657. stops.append(line_pos)
  658. break
  659. self._toc = dict(enumerate(zip(starts, stops)))
  660. self._next_key = len(self._toc)
  661. self._file_length = self._file.tell()
  662. class MMDF(_mboxMMDF):
  663. """An MMDF mailbox."""
  664. def __init__(self, path, factory=None, create=True):
  665. """Initialize an MMDF mailbox."""
  666. self._message_factory = MMDFMessage
  667. _mboxMMDF.__init__(self, path, factory, create)
  668. def _pre_message_hook(self, f):
  669. """Called before writing each message to file f."""
  670. f.write('\001\001\001\001' + os.linesep)
  671. def _post_message_hook(self, f):
  672. """Called after writing each message to file f."""
  673. f.write(os.linesep + '\001\001\001\001' + os.linesep)
  674. def _generate_toc(self):
  675. """Generate key-to-(start, stop) table of contents."""
  676. starts, stops = [], []
  677. self._file.seek(0)
  678. next_pos = 0
  679. while True:
  680. line_pos = next_pos
  681. line = self._file.readline()
  682. next_pos = self._file.tell()
  683. if line.startswith('\001\001\001\001' + os.linesep):
  684. starts.append(next_pos)
  685. while True:
  686. line_pos = next_pos
  687. line = self._file.readline()
  688. next_pos = self._file.tell()
  689. if line == '\001\001\001\001' + os.linesep:
  690. stops.append(line_pos - len(os.linesep))
  691. break
  692. elif line == '':
  693. stops.append(line_pos)
  694. break
  695. elif line == '':
  696. break
  697. self._toc = dict(enumerate(zip(starts, stops)))
  698. self._next_key = len(self._toc)
  699. self._file.seek(0, 2)
  700. self._file_length = self._file.tell()
  701. class MH(Mailbox):
  702. """An MH mailbox."""
  703. def __init__(self, path, factory=None, create=True):
  704. """Initialize an MH instance."""
  705. Mailbox.__init__(self, path, factory, create)
  706. if not os.path.exists(self._path):
  707. if create:
  708. os.mkdir(self._path, 0700)
  709. os.close(os.open(os.path.join(self._path, '.mh_sequences'),
  710. os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
  711. else:
  712. raise NoSuchMailboxError(self._path)
  713. self._locked = False
  714. def add(self, message):
  715. """Add message and return assigned key."""
  716. keys = self.keys()
  717. if len(keys) == 0:
  718. new_key = 1
  719. else:
  720. new_key = max(keys) + 1
  721. new_path = os.path.join(self._path, str(new_key))
  722. f = _create_carefully(new_path)
  723. try:
  724. if self._locked:
  725. _lock_file(f)
  726. try:
  727. self._dump_message(message, f)
  728. if isinstance(message, MHMessage):
  729. self._dump_sequences(message, new_key)
  730. finally:
  731. if self._locked:
  732. _unlock_file(f)
  733. finally:
  734. _sync_close(f)
  735. return new_key
  736. def remove(self, key):
  737. """Remove the keyed message; raise KeyError if it doesn't exist."""
  738. path = os.path.join(self._path, str(key))
  739. try:
  740. f = open(path, 'rb+')
  741. except IOError, e:
  742. if e.errno == errno.ENOENT:
  743. raise KeyError('No message with key: %s' % key)
  744. else:
  745. raise
  746. try:
  747. if self._locked:
  748. _lock_file(f)
  749. try:
  750. f.close()
  751. os.remove(os.path.join(self._path, str(key)))
  752. finally:
  753. if self._locked:
  754. _unlock_file(f)
  755. finally:
  756. f.close()
  757. def __setitem__(self, key, message):
  758. """Replace the keyed message; raise KeyError if it doesn't exist."""
  759. path = os.path.join(self._path, str(key))
  760. try:
  761. f = open(path, 'rb+')
  762. except IOError, e:
  763. if e.errno == errno.ENOENT:
  764. raise KeyError('No message with key: %s' % key)
  765. else:
  766. raise
  767. try:
  768. if self._locked:
  769. _lock_file(f)
  770. try:
  771. os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
  772. self._dump_message(message, f)
  773. if isinstance(message, MHMessage):
  774. self._dump_sequences(message, key)
  775. finally:
  776. if self._locked:
  777. _unlock_file(f)
  778. finally:
  779. _sync_close(f)
  780. def get_message(self, key):
  781. """Return a Message representation or raise a KeyError."""
  782. try:
  783. if self._locked:
  784. f = open(os.path.join(self._path, str(key)), 'r+')
  785. else:
  786. f = open(os.path.join(self._path, str(key)), 'r')
  787. except IOError, e:
  788. if e.errno == errno.ENOENT:
  789. raise KeyError('No message with key: %s' % key)
  790. else:
  791. raise
  792. try:
  793. if self._locked:
  794. _lock_file(f)
  795. try:
  796. msg = MHMessage(f)
  797. finally:
  798. if self._locked:
  799. _unlock_file(f)
  800. finally:
  801. f.close()
  802. for name, key_list in self.get_sequences().iteritems():
  803. if key in key_list:
  804. msg.add_sequence(name)
  805. return msg
  806. def get_string(self, key):
  807. """Return a string representation or raise a KeyError."""
  808. try:
  809. if self._locked:
  810. f = open(os.path.join(self._path, str(key)), 'r+')
  811. else:
  812. f = open(os.path.join(self._path, str(key)), 'r')
  813. except IOError, e:
  814. if e.errno == errno.ENOENT:
  815. raise KeyError('No message with key: %s' % key)
  816. else:
  817. raise
  818. try:
  819. if self._locked:
  820. _lock_file(f)
  821. try:
  822. return f.read()
  823. finally:
  824. if self._locked:
  825. _unlock_file(f)
  826. finally:
  827. f.close()
  828. def get_file(self, key):
  829. """Return a file-like representation or raise a KeyError."""
  830. try:
  831. f = open(os.path.join(self._path, str(key)), 'rb')
  832. except IOError, e:
  833. if e.errno == errno.ENOENT:
  834. raise KeyError('No message with key: %s' % key)
  835. else:
  836. raise
  837. return _ProxyFile(f)
  838. def iterkeys(self):
  839. """Return an iterator over keys."""
  840. return iter(sorted(int(entry) for entry in os.listdir(self._path)
  841. if entry.isdigit()))
  842. def has_key(self, key):
  843. """Return True if the keyed message exists, False otherwise."""
  844. return os.path.exists(os.path.join(self._path, str(key)))
  845. def __len__(self):
  846. """Return a count of messages in the mailbox."""
  847. return len(list(self.iterkeys()))
  848. def lock(self):
  849. """Lock the mailbox."""
  850. if not self._locked:
  851. self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
  852. _lock_file(self._file)
  853. self._locked = True
  854. def unlock(self):
  855. """Unlock the mailbox if it is locked."""
  856. if self._locked:
  857. _unlock_file(self._file)
  858. _sync_close(self._file)
  859. del self._file
  860. self._locked = False
  861. def flush(self):
  862. """Write any pending changes to the disk."""
  863. return
  864. def close(self):
  865. """Flush and close the mailbox."""
  866. if self._locked:
  867. self.unlock()
  868. def list_folders(self):
  869. """Return a list of folder names."""
  870. result = []
  871. for entry in os.listdir(self._path):
  872. if os.path.isdir(os.path.join(self._path, entry)):
  873. result.append(entry)
  874. return result
  875. def get_folder(self, folder):
  876. """Return an MH instance for the named folder."""
  877. return MH(os.path.join(self._path, folder),
  878. factory=self._factory, create=False)
  879. def add_folder(self, folder):
  880. """Create a folder and return an MH instance representing it."""
  881. return MH(os.path.join(self._path, folder),
  882. factory=self._factory)
  883. def remove_folder(self, folder):
  884. """Delete the named folder, which must be empty."""
  885. path = os.path.join(self._path, folder)
  886. entries = os.listdir(path)
  887. if entries == ['.mh_sequences']:
  888. os.remove(os.path.join(path, '.mh_sequences'))
  889. elif entries == []:
  890. pass
  891. else:
  892. raise NotEmptyError('Folder not empty: %s' % self._path)
  893. os.rmdir(path)
  894. def get_sequences(self):
  895. """Return a name-to-key-list dictionary to define each sequence."""
  896. results = {}
  897. f = open(os.path.join(self._path, '.mh_sequences'), 'r')
  898. try:
  899. all_keys = set(self.keys())
  900. for line in f:
  901. try:
  902. name, contents = line.split(':')
  903. keys = set()
  904. for spec in contents.split():
  905. if spec.isdigit():
  906. keys.add(int(spec))
  907. else:
  908. start, stop = (int(x) for x in spec.split('-'))
  909. keys.update(range(start, stop + 1))
  910. results[name] = [key for key in sorted(keys) \
  911. if key in all_keys]
  912. if len(results[name]) == 0:
  913. del results[name]
  914. except ValueError:
  915. raise FormatError('Invalid sequence specification: %s' %
  916. line.rstrip())
  917. finally:
  918. f.close()
  919. return results
  920. def set_sequences(self, sequences):
  921. """Set sequences using the given name-to-key-list dictionary."""
  922. f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
  923. try:
  924. os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
  925. for name, keys in sequences.iteritems():
  926. if len(keys) == 0:
  927. continue
  928. f.write('%s:' % name)
  929. prev = None
  930. completing = False
  931. for key in sorted(set(keys)):
  932. if key - 1 == prev:
  933. if not completing:
  934. completing = True
  935. f.write('-')
  936. elif completing:
  937. completing = False
  938. f.write('%s %s' % (prev, key))
  939. else:
  940. f.write(' %s' % key)
  941. prev = key
  942. if completing:
  943. f.write(str(prev) + '\n')
  944. else:
  945. f.write('\n')
  946. finally:
  947. _sync_close(f)
  948. def pack(self):
  949. """Re-name messages to eliminate numbering gaps. Invalidates keys."""
  950. sequences = self.get_sequences()
  951. prev = 0
  952. changes = []
  953. for key in self.iterkeys():
  954. if key - 1 != prev:
  955. changes.append((key, prev + 1))
  956. if hasattr(os, 'link'):
  957. os.link(os.path.join(self._path, str(key)),
  958. os.path.join(self._path, str(prev + 1)))
  959. os.unlink(os.path.join(self._path, str(key)))
  960. else:
  961. os.rename(os.path.join(self._path, str(key)),
  962. os.path.join(self._path, str(prev + 1)))
  963. prev += 1
  964. self._next_key = prev + 1
  965. if len(changes) == 0:
  966. return
  967. for name, key_list in sequences.items():
  968. for old, new in changes:
  969. if old in key_list:
  970. key_list[key_list.index(old)] = new
  971. self.set_sequences(sequences)
  972. def _dump_sequences(self, message, key):
  973. """Inspect a new MHMessage and update sequences appropriately."""
  974. pending_sequences = message.get_sequences()
  975. all_sequences = self.get_sequences()
  976. for name, key_list in all_sequences.iteritems():
  977. if name in pending_sequences:
  978. key_list.append(key)
  979. elif key in key_list:
  980. del key_list[key_list.index(key)]
  981. for sequence in pending_sequences:
  982. if sequence not in all_sequences:
  983. all_sequences[sequence] = [key]
  984. self.set_sequences(all_sequences)
  985. class Babyl(_singlefileMailbox):
  986. """An Rmail-style Babyl mailbox."""
  987. _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
  988. 'forwarded', 'edited', 'resent'))
  989. def __init__(self, path, factory=None, create=True):
  990. """Initialize a Babyl mailbox."""
  991. _singlefileMailbox.__init__(self, path, factory, create)
  992. self._labels = {}
  993. def add(self, message):
  994. """Add message and return assigned key."""
  995. key = _singlefileMailbox.add(self, message)
  996. if isinstance(message, BabylMessage):
  997. self._labels[key] = message.get_labels()
  998. return key
  999. def remove(self, key):
  1000. """Remove the keyed message; raise KeyError if it doesn't exist."""
  1001. _singlefileMailbox.remove(self, key)
  1002. if key in self._labels:
  1003. del self._labels[key]
  1004. def __setitem__(self, key, message):
  1005. """Replace the keyed message; raise KeyError if it doesn't exist."""
  1006. _singlefileMailbox.__setitem__(self, key, message)
  1007. if isinstance(message, BabylMessage):
  1008. self._labels[key] = message.get_labels()
  1009. def get_message(self, key):
  1010. """Return a Message representation or raise a KeyError."""
  1011. start, stop = self._lookup(key)
  1012. self._file.seek(start)
  1013. self._file.readline() # Skip '1,' line specifying labels.
  1014. original_headers = StringIO.StringIO()
  1015. while True:
  1016. line = self._file.readline()
  1017. if line == '*** EOOH ***' + os.linesep or line == '':
  1018. break
  1019. original_headers.write(line.replace(os.linesep, '\n'))
  1020. visible_headers = StringIO.StringIO()
  1021. while True:
  1022. line = self._file.readline()
  1023. if line == os.linesep or line == '':
  1024. break
  1025. visible_headers.write(line.replace(os.linesep, '\n'))
  1026. body = self._file.read(stop - self._file.tell()).replace(os.linesep,
  1027. '\n')
  1028. msg = BabylMessage(original_headers.getvalue() + body)
  1029. msg.set_visible(visible_headers.getvalue())
  1030. if key in self._labels:
  1031. msg.set_labels(self._labels[key])
  1032. return msg
  1033. def get_string(self, key):
  1034. """Return a string representation or raise a KeyError."""
  1035. start, stop = self._lookup(key)
  1036. self._file.seek(start)
  1037. self._file.readline() # Skip '1,' line specifying labels.
  1038. original_headers = StringIO.StringIO()
  1039. while True:
  1040. line = self._file.readline()
  1041. if line == '*** EOOH ***' + os.linesep or line == '':
  1042. break
  1043. original_headers.write(line.replace(os.linesep, '\n'))
  1044. while True:
  1045. line = self._file.readline()
  1046. if line == os.linesep or line == '':
  1047. break
  1048. return original_headers.getvalue() + \
  1049. self._file.read(stop - self._file.tell()).replace(os.linesep,
  1050. '\n')
  1051. def get_file(self, key):
  1052. """Return a file-like representation or raise a KeyError."""
  1053. return StringIO.StringIO(self.get_string(key).replace('\n',
  1054. os.linesep))
  1055. def get_labels(self):
  1056. """Return a list of user-defined labels in the mailbox."""
  1057. self._lookup()
  1058. labels = set()
  1059. for label_list in self._labels.values():
  1060. labels.update(label_list)
  1061. labels.difference_update(self._special_labels)
  1062. return list(labels)
  1063. def _generate_toc(self):
  1064. """Generate key-to-(start, stop) table of contents."""
  1065. starts, stops = [], []
  1066. self._file.seek(0)
  1067. next_pos = 0
  1068. label_lists = []
  1069. while True:
  1070. line_pos = next_pos
  1071. line = self._file.readline()
  1072. next_pos = self._file.tell()
  1073. if line == '\037\014' + os.linesep:
  1074. if len(stops) < len(starts):
  1075. stops.append(line_pos - len(os.linesep))
  1076. starts.append(next_pos)
  1077. labels = [label.strip() for label
  1078. in self._file.readline()[1:].split(',')
  1079. if label.strip() != '']
  1080. label_lists.append(labels)
  1081. elif line == '\037' or line == '\037' + os.linesep:
  1082. if len(stops) < len(starts):
  1083. stops.append(line_pos - len(os.linesep))
  1084. elif line == '':
  1085. stops.append(line_pos - len(os.linesep))
  1086. break
  1087. self._toc = dict(enumerate(zip(starts, stops)))
  1088. self._labels = dict(enumerate(label_lists))
  1089. self._next_key = len(self._toc)
  1090. self._file.seek(0, 2)
  1091. self._file_length = self._file.tell()
  1092. def _pre_mailbox_hook(self, f):
  1093. """Called before writing the mailbox to file f."""
  1094. f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
  1095. (os.linesep, os.linesep, ','.join(self.get_labels()),
  1096. os.linesep))
  1097. def _pre_message_hook(self, f):
  1098. """Called before writing each message to file f."""
  1099. f.write('\014' + os.linesep)
  1100. def _post_message_hook(self, f):
  1101. """Called after writing each message to file f."""
  1102. f.write(os.linesep + '\037')
  1103. def _install_message(self, message):
  1104. """Write message contents and return (start, stop)."""
  1105. start = self._file.tell()
  1106. if isinstance(message, BabylMessage):
  1107. special_labels = []
  1108. labels = []
  1109. for label in message.get_labels():
  1110. if label in self._special_labels:
  1111. special_labels.append(label)
  1112. else:
  1113. labels.append(label)
  1114. self._file.write('1')
  1115. for label in special_labels:
  1116. self._file.write(', ' + label)
  1117. self._file.write(',,')
  1118. for label in labels:
  1119. self._file.write(' ' + label + ',')
  1120. self._file.write(os.linesep)
  1121. else:
  1122. self._file.write('1,,' + os.linesep)
  1123. if isinstance(message, email.message.Message):
  1124. orig_buffer = StringIO.StringIO()
  1125. orig_generator = email.generator.Generator(orig_buffer, False, 0)
  1126. orig_generator.flatten(message)
  1127. orig_buffer.seek(0)
  1128. while True:
  1129. line = orig_buffer.readline()
  1130. self._file.write(line.replace('\n', os.linesep))
  1131. if line == '\n' or line == '':
  1132. break
  1133. self._file.write('*** EOOH ***' + os.linesep)
  1134. if isinstance(message, BabylMessage):
  1135. vis_buffer = StringIO.StringIO()
  1136. vis_generator = email.generator.Generator(vis_buffer, False, 0)
  1137. vis_generator.flatten(message.get_visible())
  1138. while True:
  1139. line = vis_buffer.readline()
  1140. self._file.write(line.replace('\n', os.linesep))
  1141. if line == '\n' or line == '':
  1142. break
  1143. else:
  1144. orig_buffer.seek(0)
  1145. while True:
  1146. line = orig_buffer.readline()
  1147. self._file.write(line.replace('\n', os.linesep))
  1148. if line == '\n' or line == '':
  1149. break
  1150. while True:
  1151. buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
  1152. if buffer == '':
  1153. break
  1154. self._file.write(buffer.replace('\n', os.linesep))
  1155. elif isinstance(message, str):
  1156. body_start = message.find('\n\n') + 2
  1157. if body_start - 2 != -1:
  1158. self._file.write(message[:body_start].replace('\n',
  1159. os.linesep))
  1160. self._file.write('*** EOOH ***' + os.linesep)
  1161. self._file.write(message[:body_start].replace('\n',
  1162. os.linesep))
  1163. self._file.write(message[body_start:].replace('\n',
  1164. os.linesep))
  1165. else:
  1166. self._file.write('*** EOOH ***' + os.linesep + os.linesep)
  1167. self._file.write(message.replace('\n', os.linesep))
  1168. elif hasattr(message, 'readline'):
  1169. original_pos = message.tell()
  1170. first_pass = True
  1171. while True:
  1172. line = message.readline()
  1173. self._file.write(line.replace('\n', os.linesep))
  1174. if line == '\n' or line == '':
  1175. self._file.write('*** EOOH ***' + os.linesep)
  1176. if first_pass:
  1177. first_pass = False
  1178. message.seek(original_pos)
  1179. else:
  1180. break
  1181. while True:
  1182. buffer = message.read(4096) # Buffer size is arbitrary.
  1183. if buffer == '':
  1184. break
  1185. self._file.write(buffer.replace('\n', os.linesep))
  1186. else:
  1187. raise TypeError('Invalid message type: %s' % type(message))
  1188. stop = self._file.tell()
  1189. return (start, stop)
  1190. class Message(email.message.Message):
  1191. """Message with mailbox-format-specific properties."""
  1192. def __init__(self, message=None):
  1193. """Initialize a Message instance."""
  1194. if isinstance(message, email.message.Message):
  1195. self._become_message(copy.deepcopy(message))
  1196. if isinstance(message, Message):
  1197. message._explain_to(self)
  1198. elif isinstance(message, str):
  1199. self._become_message(email.message_from_string(message))
  1200. elif hasattr(message, "read"):
  1201. self._become_message(email.message_from_file(message))
  1202. elif message is None:
  1203. email.message.Message.__init__(self)
  1204. else:
  1205. raise TypeError('Invalid message type: %s' % type(message))
  1206. def _become_message(self, message):
  1207. """Assume the non-format-specific state of message."""
  1208. for name in ('_headers', '_unixfrom', '_payload', '_charset',
  1209. 'preamble', 'epilogue', 'defects', '_default_type'):
  1210. self.__dict__[name] = message.__dict__[name]
  1211. def _explain_to(self, message):
  1212. """Copy format-specific state to message insofar as possible."""
  1213. if isinstance(message, Message):
  1214. return # There's nothing format-specific to explain.
  1215. else:
  1216. raise TypeError('Cannot convert to spe…