/src/calibre/devices/apple/driver.py

Large files files are truncated, but you can click here to view the full file

# -*- coding: utf-8 -*-

__license__   = 'GPL v3'
__copyright__ = '2010, Gregory Riker'
__docformat__ = 'restructuredtext en'


import cStringIO, ctypes, datetime, os, re, shutil, subprocess, sys, tempfile, time
from calibre.constants import __appname__, __version__, DEBUG
from calibre import fit_image
from calibre.constants import isosx, iswindows
from calibre.devices.errors import UserFeedback
from calibre.devices.usbms.deviceconfig import DeviceConfig
from calibre.devices.interface import DevicePlugin
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.metadata import authors_to_string, MetaInformation, \
    title_sort
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.epub import set_metadata
from calibre.library.server.utils import strftime
from calibre.utils.config import config_dir, prefs
from calibre.utils.date import now, parse_date
from calibre.utils.logging import Log
from calibre.utils.zipfile import ZipFile

from PIL import Image as PILImage

if isosx:
    try:
        import appscript
        appscript
    except:
        # appscript fails to load on 10.4
        appscript = None

if iswindows:
    import pythoncom, win32com.client

class DriverBase(DeviceConfig, DevicePlugin):
    # Needed for config_widget to work
    FORMATS = ['epub', 'pdf']
    SUPPORTS_SUB_DIRS = True   # To enable second checkbox in customize widget

    @classmethod
    def _config_base_name(cls):
        return 'iTunes'

class ITUNES(DriverBase):
    '''
    Calling sequences:
    Initialization:
        can_handle() or can_handle_windows()
        reset()
        open()
        card_prefix()
        can_handle()
        set_progress_reporter()
        get_device_information()
        card_prefix()
        free_space()
        (Job 1 Get device information finishes)
        can_handle()
        set_progress_reporter()
        books() (once for each storage point)
        settings()
        settings()
        can_handle() (~1x per second OSX while idle)
    Delete:
        delete_books()
        remove_books_from_metadata()
        use_plugboard_ext()
        set_plugboard()
        sync_booklists()
        card_prefix()
        free_space()
    Upload:
        settings()
        set_progress_reporter()
        upload_books()
        add_books_to_metadata()
        use_plugboard_ext()
        set_plugboard()
        set_progress_reporter()
        sync_booklists()
        card_prefix()
        free_space()
    '''

    name = 'Apple device interface'
    gui_name = _('Apple device')
    icon = I('devices/ipad.png')
    description    = _('Communicate with iTunes/iBooks.')
    supported_platforms = ['osx','windows']
    author = 'GRiker'
    #: The version of this plugin as a 3-tuple (major, minor, revision)
    version        = (0,9,0)

    OPEN_FEEDBACK_MESSAGE = _(
        'Apple device detected, launching iTunes, please wait ...')
    BACKLOADING_ERROR_MESSAGE = _(
        "Cannot copy books directly from iDevice. "
        "Drag from iTunes Library to desktop, then add to calibre's Library window.")

    # Product IDs:
    #  0x1291   iPod Touch
    #  0x1293   iPod Touch 2G
    #  0x1299   iPod Touch 3G
    #  0x1292   iPhone 3G
    #  0x1294   iPhone 3GS
    #  0x1297   iPhone 4
    #  0x129a   iPad
    VENDOR_ID = [0x05ac]
    PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a]
    BCD = [0x01]

    # Plugboard ID
    DEVICE_PLUGBOARD_NAME = 'APPLE'

    # iTunes enumerations
    Audiobooks = [
        'Audible file',
        'MPEG audio file',
        'Protected AAC audio file'
        ]
    ArtworkFormat = [
        'Unknown',
        'JPEG',
        'PNG',
        'BMP'
        ]
    PlaylistKind = [
        'Unknown',
        'Library',
        'User',
        'CD',
        'Device',
        'Radio Tuner'
        ]
    PlaylistSpecialKind = [
        'Unknown',
        'Purchased Music',
        'Party Shuffle',
        'Podcasts',
        'Folder',
        'Video',
        'Music',
        'Movies',
        'TV Shows',
        'Books',
        ]
    SearchField = [
        'All',
        'Visible',
        'Artists',
        'Albums',
        'Composers',
        'SongNames',
        ]
    Sources = [
        'Unknown',
        'Library',
        'iPod',
        'AudioCD',
        'MP3CD',
        'Device',
        'RadioTuner',
        'SharedLibrary'
        ]

    # Cover art size limits
    MAX_COVER_WIDTH = 510
    MAX_COVER_HEIGHT = 680

    # Properties
    cached_books = {}
    cache_dir = os.path.join(config_dir, 'caches', 'itunes')
    calibre_library_path = prefs['library_path']
    archive_path = os.path.join(cache_dir, "thumbs.zip")
    description_prefix = "added by calibre"
    ejected = False
    iTunes= None
    iTunes_media = None
    library_orphans = None
    log = Log()
    manual_sync_mode = False
    path_template = 'iTunes/%s - %s.%s'
    plugboards = None
    plugboard_func = None
    problem_titles = []
    problem_msg = None
    report_progress = None
    update_list = []
    sources = None
    update_msg = None
    update_needed = False

    # Public methods
    def add_books_to_metadata(self, locations, metadata, booklists):
        '''
        Add locations to the booklists. This function must not communicate with
        the device.
        @param locations: Result of a call to L{upload_books}
        @param metadata: List of MetaInformation objects, same as for
        :method:`upload_books`.
        @param booklists: A tuple containing the result of calls to
                                (L{books}(oncard=None), L{books}(oncard='carda'),
                                L{books}(oncard='cardb')).
        '''
        if DEBUG:
            self.log.info("ITUNES.add_books_to_metadata()")

        task_count = float(len(self.update_list))

        # Delete any obsolete copies of the book from the booklist
        if self.update_list:
            if False:
                self._dump_booklist(booklists[0], header='before',indent=2)
                self._dump_update_list(header='before',indent=2)
                self._dump_cached_books(header='before',indent=2)

            for (j,p_book) in enumerate(self.update_list):
                if False:
                    if isosx:
                        self.log.info("  looking for '%s' by %s uuid:%s" %
                            (p_book['title'],p_book['author'], p_book['uuid']))
                    elif iswindows:
                        self.log.info(" looking for '%s' by %s (%s)" %
                                        (p_book['title'],p_book['author'], p_book['uuid']))

                # Purge the booklist, self.cached_books
                for i,bl_book in enumerate(booklists[0]):
                    if bl_book.uuid == p_book['uuid']:
                        # Remove from booklists[0]
                        booklists[0].pop(i)
                        if False:
                            if isosx:
                                self.log.info("  removing old %s %s from booklists[0]" %
                                    (p_book['title'], str(p_book['lib_book'])[-9:]))
                            elif iswindows:
                                self.log.info(" removing old '%s' from booklists[0]" %
                                                (p_book['title']))

                        # If >1 matching uuid, remove old title
                        matching_uuids = 0
                        for cb in self.cached_books:
                            if self.cached_books[cb]['uuid'] == p_book['uuid']:
                                matching_uuids += 1

                        if matching_uuids > 1:
                            for cb in self.cached_books:
                                if self.cached_books[cb]['uuid'] == p_book['uuid']:
                                    if self.cached_books[cb]['title'] == p_book['title'] and \
                                       self.cached_books[cb]['author'] == p_book['author']:
                                        if DEBUG:
                                            self._dump_cached_book(self.cached_books[cb],header="removing from self.cached_books:", indent=2)
                                        self.cached_books.pop(cb)
                                        break
                            break
                if self.report_progress is not None:
                    self.report_progress(j+1/task_count, _('Updating device metadata listing...'))

            if self.report_progress is not None:
                self.report_progress(1.0, _('Updating device metadata listing...'))

        # Add new books to booklists[0]
        # Charles thinks this should be
        # for new_book in metadata[0]:
        for new_book in locations[0]:
            if DEBUG:
                self.log.info("  adding '%s' by '%s' to booklists[0]" %
                    (new_book.title, new_book.author))
            booklists[0].append(new_book)

        if False:
            self._dump_booklist(booklists[0],header='after',indent=2)
            self._dump_cached_books(header='after',indent=2)

    def books(self, oncard=None, end_session=True):
        """
        Return a list of ebooks on the device.
        @param oncard:  If 'carda' or 'cardb' return a list of ebooks on the
                        specific storage card, otherwise return list of ebooks
                        in main memory of device. If a card is specified and no
                        books are on the card return empty list.
        @return: A BookList.

        Implementation notes:
        iTunes does not sync purchased books, they are only on the device.  They are visible, but
        they are not backed up to iTunes.  Since calibre can't manage them, don't show them in the
        list of device books.

        """
        if not oncard:
            if DEBUG:
                self.log.info("ITUNES:books():")
                if self.settings().use_subdirs:
                    self.log.info(" Cover fetching/caching enabled")
                else:
                    self.log.info(" Cover fetching/caching disabled")

            # Fetch a list of books from iPod device connected to iTunes
            if 'iPod' in self.sources:
                booklist = BookList(self.log)
                cached_books = {}

                if isosx:
                    library_books = self._get_library_books()
                    device_books = self._get_device_books()
                    book_count = float(len(device_books))
                    for (i,book) in enumerate(device_books):
                        this_book = Book(book.name(), book.artist())
                        format = 'pdf' if book.kind().startswith('PDF') else 'epub'
                        this_book.path = self.path_template % (book.name(), book.artist(),format)
                        try:
                            this_book.datetime = parse_date(str(book.date_added())).timetuple()
                        except:
                            this_book.datetime = time.gmtime()
                        this_book.db_id = None
                        this_book.device_collections = []
                        this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                        this_book.size = book.size()
                        this_book.uuid = book.composer()
                        # Hack to discover if we're running in GUI environment
                        if self.report_progress is not None:
                            this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
                        else:
                            this_book.thumbnail = None
                        booklist.add_book(this_book, False)

                        cached_books[this_book.path] = {
                         'title':book.name(),
                         'author':[book.artist()],
                         'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
                         'dev_book':book,
                         'uuid': book.composer()
                         }

                        if self.report_progress is not None:
                            self.report_progress(i+1/book_count, _('%d of %d') % (i+1, book_count))
                    self._purge_orphans(library_books, cached_books)

                elif iswindows:
                    try:
                        pythoncom.CoInitialize()
                        self.iTunes = win32com.client.Dispatch("iTunes.Application")
                        library_books = self._get_library_books()
                        device_books = self._get_device_books()
                        book_count = float(len(device_books))
                        for (i,book) in enumerate(device_books):
                            this_book = Book(book.Name, book.Artist)
                            format = 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
                            this_book.path = self.path_template % (book.Name, book.Artist,format)
                            try:
                                this_book.datetime = parse_date(str(book.DateAdded)).timetuple()
                            except:
                                this_book.datetime = time.gmtime()
                            this_book.db_id = None
                            this_book.device_collections = []
                            this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                            this_book.size = book.Size
                            # Hack to discover if we're running in GUI environment
                            if self.report_progress is not None:
                                this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
                            else:
                                this_book.thumbnail = None
                            booklist.add_book(this_book, False)

                            cached_books[this_book.path] = {
                             'title':book.Name,
                             'author':book.Artist,
                             'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
                             'uuid': book.Composer,
                             'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
                             }

                            if self.report_progress is not None:
                                self.report_progress(i+1/book_count,
                                        _('%d of %d') % (i+1, book_count))
                        self._purge_orphans(library_books, cached_books)

                    finally:
                        pythoncom.CoUninitialize()

                if self.report_progress is not None:
                    self.report_progress(1.0, _('finished'))
                self.cached_books = cached_books
                if DEBUG:
                    self._dump_booklist(booklist, 'returning from books()', indent=2)
                    self._dump_cached_books('returning from books()',indent=2)
                return booklist
        else:
            return BookList(self.log)

    def can_handle(self, device_info, debug=False):
        '''
        Unix version of :method:`can_handle_windows`

        :param device_info: Is a tupe of (vid, pid, bcd, manufacturer, product,
        serial number)

        Confirm that:
            - iTunes is running
            - there is an iDevice connected
        This gets called first when the device fingerprint is read, so it needs to
        instantiate iTunes if necessary
        This gets called ~1x/second while device fingerprint is sensed
        '''
        if appscript is None:
            return False

        if self.iTunes:
            # Check for connected book-capable device
            self.sources = self._get_sources()
            if 'iPod' in self.sources:
                #if DEBUG:
                    #sys.stdout.write('.')
                    #sys.stdout.flush()
                return True
            else:
                if DEBUG:
                    sys.stdout.write('-')
                    sys.stdout.flush()
                return False
        else:
            # Called at entry
            # We need to know if iTunes sees the iPad
            # It may have been ejected
            if DEBUG:
                self.log.info("ITUNES.can_handle()")

            self._launch_iTunes()
            self.sources = self._get_sources()
            if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
                attempts = 9
                while attempts:
                    # If iTunes was just launched, device may not be detected yet
                    self.sources = self._get_sources()
                    if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
                        attempts -= 1
                        time.sleep(0.5)
                        if DEBUG:
                            self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts))
                    else:
                        if DEBUG:
                            self.log.info(' found connected iPad')
                        break
                else:
                    # iTunes running, but not connected iPad
                    if DEBUG:
                        self.log.info(' self.ejected = True')
                    self.ejected = True
                    return False

            self._discover_manual_sync_mode(wait = 2 if self.initial_status == 'launched' else 0)
            return True

    def can_handle_windows(self, device_id, debug=False):
        '''
        Optional method to perform further checks on a device to see if this driver
        is capable of handling it. If it is not it should return False. This method
        is only called after the vendor, product ids and the bcd have matched, so
        it can do some relatively time intensive checks. The default implementation
        returns True. This method is called only on windows. See also
        :method:`can_handle`.

        :param device_info: On windows a device ID string. On Unix a tuple of
        ``(vendor_id, product_id, bcd)``.

        iPad implementation notes:
        It is necessary to use this method to check for the presence of a connected
        iPad, as we have to return True if we can handle device interaction, or False if not.

        '''
        if self.iTunes:
            # We've previously run, so the user probably ejected the device
            try:
                pythoncom.CoInitialize()
                self.sources = self._get_sources()
                if 'iPod' in self.sources:
                    if DEBUG:
                        sys.stdout.write('.')
                        sys.stdout.flush()
                    if DEBUG:
                        self.log.info('ITUNES.can_handle_windows:\n confirming connected iPad')
                    self.ejected = False
                    self._discover_manual_sync_mode()
                    return True
                else:
                    if DEBUG:
                        self.log.info("ITUNES.can_handle_windows():\n device ejected")
                    self.ejected = True
                    return False
            except:
                # iTunes connection failed, probably not running anymore

                self.log.error("ITUNES.can_handle_windows():\n lost connection to iTunes")
                return False
            finally:
                pythoncom.CoUninitialize()

        else:
            if DEBUG:
                self.log.info("ITUNES:can_handle_windows():\n Launching iTunes")

            try:
                pythoncom.CoInitialize()
                self._launch_iTunes()
                self.sources = self._get_sources()
                if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
                    attempts = 9
                    while attempts:
                        # If iTunes was just launched, device may not be detected yet
                        self.sources = self._get_sources()
                        if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
                            attempts -= 1
                            time.sleep(0.5)
                            if DEBUG:
                                self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts))
                        else:
                            if DEBUG:
                                self.log.info(' found connected iPad in iTunes')
                            break
                    else:
                        # iTunes running, but not connected iPad
                        if DEBUG:
                            self.log.info(' iDevice has been ejected')
                        self.ejected = True
                        return False

                self.log.info(' found connected iPad in sources')
                self._discover_manual_sync_mode(wait=1.0)

            finally:
                pythoncom.CoUninitialize()

            return True

    def card_prefix(self, end_session=True):
        '''
        Return a 2 element list of the prefix to paths on the cards.
        If no card is present None is set for the card's prefix.
        E.G.
        ('/place', '/place2')
        (None, 'place2')
        ('place', None)
        (None, None)
        '''
        return (None,None)

    @classmethod
    def config_widget(cls):
        '''
        Return a QWidget with settings for the device interface
        '''
        cw = DriverBase.config_widget()
        # Turn off the Save template
        cw.opt_save_template.setVisible(False)
        cw.label.setVisible(False)
        # Repurpose the metadata checkbox
        cw.opt_read_metadata.setText(_("Use Series as Category in iTunes/iBooks"))
        # Repurpose the use_subdirs checkbox
        cw.opt_use_subdirs.setText(_("Cache covers from iTunes/iBooks"))
        return cw

    def delete_books(self, paths, end_session=True):
        '''
        Delete books at paths on device.
        iTunes doesn't let us directly delete a book on the device.
        If the requested paths are deletable (i.e., it's in the Library|Books list),
        delete the paths from the library, then resync iPad

        '''
        self.problem_titles = []
        self.problem_msg = _("Some books not found in iTunes database.\n"
                              "Delete using the iBooks app.\n"
                              "Click 'Show Details' for a list.")
        self.log.info("ITUNES:delete_books()")
        for path in paths:
            if self.cached_books[path]['lib_book']:
                if DEBUG:
                    self.log.info(" Deleting '%s' from iTunes library" % (path))

                if isosx:
                    self._remove_from_iTunes(self.cached_books[path])
                    if self.manual_sync_mode:
                        self._remove_from_device(self.cached_books[path])
                elif iswindows:
                    try:
                        pythoncom.CoInitialize()
                        self.iTunes = win32com.client.Dispatch("iTunes.Application")
                        self._remove_from_iTunes(self.cached_books[path])
                        if self.manual_sync_mode:
                            self._remove_from_device(self.cached_books[path])
                    finally:
                        pythoncom.CoUninitialize()

                if not self.manual_sync_mode:
                    self.update_needed = True
                    self.update_msg = "Deleted books from device"
                else:
                    self.log.info(" skipping sync phase, manual_sync_mode: True")
            else:
                if self.manual_sync_mode:
                    metadata = MetaInformation(self.cached_books[path]['title'],
                                               [self.cached_books[path]['author']])
                    metadata.uuid = self.cached_books[path]['uuid']

                    if isosx:
                        self._remove_existing_copy(self.cached_books[path],metadata)
                    elif iswindows:
                        try:
                            pythoncom.CoInitialize()
                            self.iTunes = win32com.client.Dispatch("iTunes.Application")
                            self._remove_existing_copy(self.cached_books[path],metadata)
                        finally:
                            pythoncom.CoUninitialize()

                else:
                    self.problem_titles.append("'%s' by %s" %
                     (self.cached_books[path]['title'],self.cached_books[path]['author']))

    def eject(self):
        '''
        Un-mount / eject the device from the OS. This does not check if there
        are pending GUI jobs that need to communicate with the device.
        '''
        if DEBUG:
            self.log.info("ITUNES:eject(): ejecting '%s'" % self.sources['iPod'])
        if isosx:
            self.iTunes.eject(self.sources['iPod'])
        elif iswindows:
            if 'iPod' in self.sources:
                try:
                    pythoncom.CoInitialize()
                    self.iTunes = win32com.client.Dispatch("iTunes.Application")
                    self.iTunes.sources.ItemByName(self.sources['iPod']).EjectIPod()

                finally:
                    pythoncom.CoUninitialize()

        self.iTunes = None
        self.sources = None

    def free_space(self, end_session=True):
        """
        Get free space available on the mountpoints:
          1. Main memory
          2. Card A
          3. Card B

        @return: A 3 element list with free space in bytes of (1, 2, 3). If a
        particular device doesn't have any of these locations it should return -1.

        In Windows, a sync-in-progress blocks this call until sync is complete
        """
        if DEBUG:
            self.log.info("ITUNES:free_space()")

        free_space = 0
        if isosx:
            if 'iPod' in self.sources:
                connected_device = self.sources['iPod']
                free_space = self.iTunes.sources[connected_device].free_space()

        elif iswindows:
            if 'iPod' in self.sources:

                while True:
                    try:
                        try:
                            pythoncom.CoInitialize()
                            self.iTunes = win32com.client.Dispatch("iTunes.Application")
                            connected_device = self.sources['iPod']
                            free_space = self.iTunes.sources.ItemByName(connected_device).FreeSpace
                        finally:
                            pythoncom.CoUninitialize()
                            break
                    except:
                        self.log.error(' waiting for free_space() call to go through')

        return (free_space,-1,-1)

    def get_device_information(self, end_session=True):
        """
        Ask device for device information. See L{DeviceInfoQuery}.
        @return: (device name, device version, software version on device, mime type)
        """
        if DEBUG:
            self.log.info("ITUNES:get_device_information()")

        return (self.sources['iPod'],'hw v1.0','sw v1.0', 'mime type normally goes here')

    def get_file(self, path, outfile, end_session=True):
        '''
        Read the file at C{path} on the device and write it to outfile.
        @param outfile: file object like C{sys.stdout} or the result of an C{open} call
        '''
        if DEBUG:
            self.log.info("ITUNES.get_file(): exporting '%s'" % path)
        outfile.write(open(self.cached_books[path]['lib_book'].location().path).read())

    def open(self):
        '''
        Perform any device specific initialization. Called after the device is
        detected but before any other functions that communicate with the device.
        For example: For devices that present themselves as USB Mass storage
        devices, this method would be responsible for mounting the device or
        if the device has been automounted, for finding out where it has been
        mounted. The base class within USBMS device.py has a implementation of
        this function that should serve as a good example for USB Mass storage
        devices.

        Note that most of the initialization is necessarily performed in can_handle(), as
        we need to talk to iTunes to discover if there's a connected iPod
        '''
        if DEBUG:
            self.log.info("ITUNES.open()")

        # Confirm/create thumbs archive
        if not os.path.exists(self.cache_dir):
            if DEBUG:
                self.log.info(" creating thumb cache '%s'" % self.cache_dir)
            os.makedirs(self.cache_dir)

        if not os.path.exists(self.archive_path):
            self.log.info(" creating zip archive")
            zfw = ZipFile(self.archive_path, mode='w')
            zfw.writestr("iTunes Thumbs Archive",'')
            zfw.close()
        else:
            if DEBUG:
                self.log.info(" existing thumb cache at '%s'" % self.archive_path)

    def remove_books_from_metadata(self, paths, booklists):
        '''
        Remove books from the metadata list. This function must not communicate
        with the device.
        @param paths: paths to books on the device.
        @param booklists:  A tuple containing the result of calls to
                                (L{books}(oncard=None), L{books}(oncard='carda'),
                                L{books}(oncard='cardb')).

        NB: This will not find books that were added by a different installation of calibre
            as uuids are different
        '''
        if DEBUG:
            self.log.info("ITUNES.remove_books_from_metadata()")
        for path in paths:
            if DEBUG:
                self._dump_cached_book(self.cached_books[path], indent=2)
                self.log.info("  looking for '%s' by '%s' uuid:%s" %
                                (self.cached_books[path]['title'],
                                 self.cached_books[path]['author'],
                                 self.cached_books[path]['uuid']))

            # Purge the booklist, self.cached_books, thumb cache
            for i,bl_book in enumerate(booklists[0]):
                if False:
                    self.log.info(" evaluating '%s' by '%s' uuid:%s" %
                                  (bl_book.title, bl_book.author,bl_book.uuid))

                found = False
                if bl_book.uuid == self.cached_books[path]['uuid']:
                    if False:
                        self.log.info("  matched with uuid")
                    booklists[0].pop(i)
                    found = True
                elif bl_book.title == self.cached_books[path]['title'] and \
                     bl_book.author[0] == self.cached_books[path]['author']:
                    if False:
                        self.log.info("  matched with title + author")
                    booklists[0].pop(i)
                    found = True

                if found:
                    # Remove from self.cached_books
                    for cb in self.cached_books:
                        if self.cached_books[cb]['uuid'] == self.cached_books[path]['uuid']:
                            self.cached_books.pop(cb)
                            break

                    # Remove from thumb from thumb cache
                    thumb_path = path.rpartition('.')[0] + '.jpg'
                    zf = ZipFile(self.archive_path,'a')
                    fnames = zf.namelist()
                    try:
                        thumb = [x for x in fnames if thumb_path in x][0]
                    except:
                        thumb = None
                    if thumb:
                        if DEBUG:
                            self.log.info("  deleting '%s' from cover cache" % (thumb_path))
                            zf.delete(thumb_path)
                    else:
                        if DEBUG:
                            self.log.info("  '%s' not found in cover cache" % thumb_path)
                    zf.close()

                    break
            else:
                if DEBUG:
                    self.log.error("  unable to find '%s' by '%s' (%s)" %
                                    (bl_book.title, bl_book.author,bl_book.uuid))

        if False:
            self._dump_booklist(booklists[0], indent = 2)
            self._dump_cached_books(indent=2)

    def reset(self, key='-1', log_packets=False, report_progress=None,
            detected_device=None) :
        """
        :key: The key to unlock the device
        :log_packets: If true the packet stream to/from the device is logged
        :report_progress: Function that is called with a % progress
                                (number between 0 and 100) for various tasks
                                If it is called with -1 that means that the
                                task does not have any progress information
        :detected_device: Device information from the device scanner
        """
        if DEBUG:
            self.log.info("ITUNES.reset()")

    def set_progress_reporter(self, report_progress):
        '''
        @param report_progress: Function that is called with a % progress
                                (number between 0 and 100) for various tasks
                                If it is called with -1 that means that the
                                task does not have any progress information
        '''
        self.report_progress = report_progress

    def set_plugboards(self, plugboards, pb_func):
        # This method is called with the plugboard that matches the format
        # declared in use_plugboard_ext and a device name of ITUNES
        if DEBUG:
            self.log.info("ITUNES.set_plugboard()")
            #self.log.info('  using plugboard %s' % plugboards)
        self.plugboards = plugboards
        self.plugboard_func = pb_func

    def sync_booklists(self, booklists, end_session=True):
        '''
        Update metadata on device.
        @param booklists: A tuple containing the result of calls to
                                (L{books}(oncard=None), L{books}(oncard='carda'),
                                L{books}(oncard='cardb')).
        '''

        if DEBUG:
            self.log.info("ITUNES.sync_booklists()")

        if self.update_needed:
            if DEBUG:
                self.log.info(' calling _update_device')
            self._update_device(msg=self.update_msg, wait=False)
            self.update_needed = False

        # Inform user of any problem books
        if self.problem_titles:
            raise UserFeedback(self.problem_msg,
                                  details='\n'.join(self.problem_titles), level=UserFeedback.WARN)
        self.problem_titles = []
        self.problem_msg = None
        self.update_list = []

    def total_space(self, end_session=True):
        """
        Get total space available on the mountpoints:
            1. Main memory
            2. Memory Card A
            3. Memory Card B

        @return: A 3 element list with total space in bytes of (1, 2, 3). If a
        particular device doesn't have any of these locations it should return 0.
        """
        if DEBUG:
            self.log.info("ITUNES:total_space()")
        capacity = 0
        if isosx:
            if 'iPod' in self.sources:
                connected_device = self.sources['iPod']
                capacity = self.iTunes.sources[connected_device].capacity()

        return (capacity,-1,-1)

    def upload_books(self, files, names, on_card=None, end_session=True,
                     metadata=None):
        '''
        Upload a list of books to the device. If a file already
        exists on the device, it should be replaced.
        This method should raise a L{FreeSpaceError} if there is not enough
        free space on the device. The text of the FreeSpaceError must contain the
        word "card" if C{on_card} is not None otherwise it must contain the word "memory".
        :files: A list of paths and/or file-like objects.
        :names: A list of file names that the books should have
        once uploaded to the device. len(names) == len(files)
        :return: A list of 3-element tuples. The list is meant to be passed
        to L{add_books_to_metadata}.
        :metadata: If not None, it is a list of :class:`Metadata` objects.
        The idea is to use the metadata to determine where on the device to
        put the book. len(metadata) == len(files). Apart from the regular
        cover (path to cover), there may also be a thumbnail attribute, which should
        be used in preference. The thumbnail attribute is of the form
        (width, height, cover_data as jpeg).
        '''

        new_booklist = []
        self.update_list = []
        file_count = float(len(files))
        self.problem_titles = []
        self.problem_msg = _("Some cover art could not be converted.\n"
                                     "Click 'Show Details' for a list.")

        if DEBUG:
            self.log.info("ITUNES.upload_books()")
            self._dump_files(files, header='upload_books()',indent=2)
            self._dump_update_list(header='upload_books()',indent=2)

        if isosx:
            for (i,file) in enumerate(files):
                format = file.rpartition('.')[2].lower()
                path = self.path_template % (metadata[i].title, metadata[i].author[0],format)
                self._remove_existing_copy(path, metadata[i])
                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                db_added, lb_added = self._add_new_copy(fpath, metadata[i])
                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                new_booklist.append(this_book)
                self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)

                # Add new_book to self.cached_books
                if DEBUG:
                    self.log.info("ITUNES.upload_books()")
                    self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" %
                                  ( metadata[i].title, metadata[i].author, metadata[i].uuid))
                self.cached_books[this_book.path] = {
                   'author': metadata[i].author,
                 'dev_book': db_added,
                   'format': format,
                 'lib_book': lb_added,
                    'title': metadata[i].title,
                     'uuid': metadata[i].uuid }


                # Report progress
                if self.report_progress is not None:
                    self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))

        elif iswindows:
            try:
                pythoncom.CoInitialize()
                self.iTunes = win32com.client.Dispatch("iTunes.Application")

                for (i,file) in enumerate(files):
                    format = file.rpartition('.')[2].lower()
                    path = self.path_template % (metadata[i].title, metadata[i].author[0],format)
                    self._remove_existing_copy(path, metadata[i])
                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                    db_added, lb_added = self._add_new_copy(fpath, metadata[i])

                    if self.manual_sync_mode and not db_added:
                        # Problem finding added book, probably title/author change needing to be written to metadata
                        self.problem_msg = ("Title and/or author metadata mismatch with uploaded books.\n"
                                            "Click 'Show Details...' for affected books.")
                        self.problem_titles.append("'%s' by %s" % (metadata[i].title, metadata[i].author[0]))

                    thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                    new_booklist.append(this_book)
                    self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)

                    # Add new_book to self.cached_books
                    if DEBUG:
                        self.log.info("ITUNES.upload_books()")
                        self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" %
                                      ( metadata[i].title, metadata[i].author, metadata[i].uuid))
                    self.cached_books[this_book.path] = {
                       'author': metadata[i].author[0],
                     'dev_book': db_added,
                       'format': format,
                     'lib_book': lb_added,
                        'title': metadata[i].title,
                         'uuid': metadata[i].uuid}

                    # Report progress
                    if self.report_progress is not None:
                        self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))
            finally:
                pythoncom.CoUninitialize()

        if self.report_progress is not None:
            self.report_progress(1.0, _('finished'))

        # Tell sync_booklists we need a re-sync
        if not self.manual_sync_mode:
            self.update_needed = True
            self.update_msg = "Added books to device"

        if False:
            self._dump_booklist(new_booklist,header="after upload_books()",indent=2)
            self._dump_cached_books(header="after upload_books()",indent=2)
        return (new_booklist, [], [])

    # Private methods
    def _add_device_book(self,fpath, metadata):
        '''
        assumes pythoncom wrapper for windows
        '''
        self.log.info(" ITUNES._add_device_book()")
        if isosx:
            if 'iPod' in self.sources:
                connected_device = self.sources['iPod']
                device = self.iTunes.sources[connected_device]
                for pl in device.playlists():
                    if pl.special_kind() == appscript.k.Books:
                        break
                else:
                    if DEBUG:
                        self.log.error("  Device|Books playlist not found")

                # Add the passed book to the Device|Books playlist
                added = pl.add(appscript.mactypes.File(fpath),to=pl)
                if False:
                    self.log.info("  '%s' added to Device|Books" % metadata.title)
                return added

        elif iswindows:
            if 'iPod' in self.sources:
                connected_device = self.sources['iPod']
                device = self.iTunes.sources.ItemByName(connected_device)

                db_added = None
                for pl in device.Playlists:
                    if pl.Kind == self.PlaylistKind.index('User') and \
                       pl.SpecialKind == self.PlaylistSpecialKind.index('Books'):
                        break
                else:
                    if DEBUG:
                        self.log.info("  no Books playlist found")

                # Add the passed book to the Device|Books playlist
                if pl:
                    file_s = ctypes.c_char_p(fpath)
                    FileArray = ctypes.c_char_p * 1
                    fa = FileArray(file_s)
                    op_status = pl.AddFiles(fa)

                    if DEBUG:
                        sys.stdout.write("  uploading '%s' to Device|Books ..." % metadata.title)
                        sys.stdout.flush()

                    while op_status.InProgress:
                        time.sleep(0.5)
                        if DEBUG:
                            sys.stdout.write('.')
                            sys.stdout.flush()
                    if DEBUG:
                        sys.stdout.write("\n")
                        sys.stdout.flush()

                    if False:
                        '''
                        Preferred
                        Disabled because op_status.Tracks never returns a value after adding file
                        This would be the preferred approach (as under OSX)
                        It works in _add_library_book()
                        '''
                        if DEBUG:
                            sys.stdout.write("  waiting for handle to added '%s' ..." % metadata.title)
                            sys.stdout.flush()
                        while not op_status.Tracks:
                            time.sleep(0.5)
                            if DEBUG:
                                sys.stdout.write('.')
                                sys.stdout.flush()

                        if DEBUG:
                            print
                        added = op_status.Tracks[0]
                    else:
                        '''
                        Hackish
                        Search Library|Books for the book we just added
                        PDF file name is added title - need to search for base filename w/o extension
                        '''
                        format = fpath.rpartition('.')[2].lower()
                        base_fn = fpath.rpartition(os.sep)[2]
                        base_fn = base_fn.rpartition('.')[0]
                        db_added = self._find_device_book(
                            { 'title': base_fn if format == 'pdf' else metadata.title,
                             'author': metadata.authors[0],
                               'uuid': metadata.uuid,
                             'format': format})
                    return db_added

    def _add_library_book(self,file, metadata):
        '''
        windows assumes pythoncom wrapper
        '''
        self.log.info(" ITUNES._add_library_book()")
        if isosx:
            added = self.iTunes.add(appscript.mactypes.File(file))

        elif iswindows:
            lib = self.iTunes.LibraryPlaylist
            file_s = ctypes.c_char_p(file)
            FileArray = ctypes.c_char_p * 1
            fa = FileArray(file_s)
            op_status = lib.AddFiles(fa)
            if DEBUG:
                self.log.info("  file added to Library|Books")

            self.log.info("  iTunes adding '%s'" % file)

            if DEBUG:
                sys.stdout.write("  iTunes copying '%s' ..." % metadata.title)
                sys.stdout.flush()

            while op_status.InProgress:
                time.sleep(0.5)
                if DEBUG:
                    sys.stdout.write('.')
                    sys.stdout.flush()
            if DEBUG:
                sys.stdout.write("\n")
                sys.stdout.flush()

            if True:
                '''
                Preferable
                Originally disabled because op_status.Tracks never returned a value
                after adding file.  Seems to be working with iTunes 9.2.1.5 06 Aug 2010
                '''
                if DEBUG:
                    sys.stdout.write("  waiting for handle to added '%s' ..." % metadata.title)
                    sys.stdout.flush()
                while op_status.Tracks is None:
                    time.sleep(0.5)
                    if DEBUG:
                        sys.stdout.write('.')
                        sys.stdout.flush()
                if DEBUG:
                    print
                added = op_status.Tracks[0]
            else:
                '''
                Hackish
                Search Library|Books for the book we just added
                PDF file name is added title - need to search for base filename w/o extension
                '''
                format = file.rpartition('.')[2].lower()
                base_fn = file.rpartition(os.sep)[2]
                base_fn = base_fn.rpartition('.')[0]
                added = self._find_library_book(
                    { 'title': base_fn if format == 'pdf' else metadata.title,
                     'author': metadata.author[0],
                       'uuid': metadata.uuid,
                     'format': format})
        return added

    def _add_new_copy(self, fpath, metadata):
        '''
        '''
        if DEBUG:
            self.log.info(" ITUNES._add_new_copy()")

        db_added = None
        lb_added = None

        if self.manual_sync_mode:
            db_added = self._add_device_book(fpath, metadata)
            if not getattr(fpath, 'deleted_after_upload', False):
                lb_added = self._add_library_book(fpath, metadata)
                if lb_added:
                    if DEBUG:
                        self.log.info("  file added to Library|Books for iTunes<->iBooks tracking")
        else:
            lb_added = self._add_library_book(fpath, metadata)
            if DEBUG:
                self.log.info("  file added to Library|Books for pending sync")

        return db_added, lb_added

    def _cover_to_thumb(self, path, metadata, db_added, lb_added, format):
        '''
        assumes pythoncom wrapper for db_added
        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
        '''
        self.log.info(" ITUNES._cover_to_thumb()")

        thumb = None
        if metadata.cover:

            if format == 'epub':
                # Pre-shrink cover
                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
                try:
                    img = PILImage.open(metadata.cover)
                    width = img.size[0]
                    height = img.size[1]
                    scaled, nwidth, nheight = fit_image(width, …