/LMG/Tools/rss.py

"""

RSS support for BitTorrent.

This module provides RSS support for:

 1. reading feeds

 2. downloading .torrent file from feeds

 3. automatically download .torrent files using filters

 4. downloading .torrent files from indirect RSS articles

"""



__author__    = "Roee Shlomo"

__version__   = "2.1"

__date__      = "$2007/09/30$"

__license__   = 'MIT license'



import os

import wx

import pickle

import re



from threading import Thread, Event

from time import time, mktime

from feedparser import parse

from LMG.GUI.Base.guiupdate import SafeInvocation

from LMG.Utility.threadpool import ThreadPool, makeRequests

from LMG.Utility.helpers import existsAndIsReadable

from LMG.Utility.configreader import ConfigReader

from LMG.Utility.urlfinder import HtmlTorrentScanner



# Smart episode detection regex

matchesmodes = [re.compile("(?P<name>.*?)s(?P<f_s>[0-9]+)e(?P<f_e>[0-9]+)[\\-\\+]s(?P<l_s>[0-9]+)e(?P<l_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)(?P<f_s>[0-9]+)x(?P<f_e>[0-9]+)[\\-\\+](?P<l_s>[0-9]+)x(?P<l_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)(?P<f_s>[0-9]+)(?P<f_e>[0-9]{2})[\\-\\+](?P<l_s>[0-9]+)(?P<l_e>[0-9]{2}).*?"),

                re.compile("(?P<name>.*?)s(?P<f_s>[0-9]+)e(?P<f_e>[0-9]+)[\\-\\+]e(?P<l_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)s(?P<f_s>[0-9]+)e(?P<f_e>[0-9]+)[\\-\\+](?P<l_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)(?P<f_s>[0-9]+)x(?P<f_e>[0-9]+)[\\-\\+](?P<l_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)(?P<f_s>[0-9]+)(?P<f_e>[0-9]{2})[\\-\\+](?P<l_e>[0-9]{2}).*?"),

                re.compile("(?P<name>.*?)s(?P<f_s>[0-9]+)e(?P<f_e>[0-9]+).*?"),

                re.compile("(?P<name>.*?)(?P<f_s>[0-9]+)x(?P<f_e>[0-9]+).*?"),                

                re.compile("(?P<name>.*?)s(?P<f_s>[0-9]+).*?"),

                re.compile("(?P<name>.*?)season (?P<f_s>[0-9]+).*?"),

                ]



#################################################################################

class RSS(SafeInvocation):

    """

    Main RSS Class

    """

    def __init__(self, parent):



        # Initialization

        self.doneflag = Event()

        SafeInvocation.__init__(self)

        

        # Main management

        self.feeds = []

        self.rules = []

        self.history = {}

        

        # Set up config

        self.config = ConfigManager(self)



        # html scanner threads

        self.htmlscanners = {}



        # timer

        self.timer = wx.Timer(self, -1)

        self.timerCount = -1

        self.Bind(wx.EVT_TIMER, self.onTimer)



    #######################

    # Feeders

    #######################

    def addFeeder(self, args = [], kwargs = {}):

        """

        FEEDERS: Add new feeder

        """

        feed = RSSFeeder(self, RSSFeederOptions(*args, **kwargs))

        if feed:

            self.feeds.append(feed)

        

    def removeFeeder(self, index):

        """

        FEEDERS: Remove feeder by index

        """

        del self.feeds[index]



    def changeFeeder(self, index, optionsTuple):

        """

        FEEDERS: Changes feeder options

        """

        self.feeds[index].fromTuple(optionsTuple)



    def changeName(self, feeder, name):

        """

        FEEDERS: Change feeder name

        """

        more = 1

        newname = name

        names = [feed.options.name for feed in self.feeds if feed != feeder]

        while newname in names:

            more += 1

            newname = name + " " + str(more)    

        feeder.options.name = newname            

        self.config.save("feeds")



        # Update GUI

        self.updateFeederSelector()

        if utility.window.exists('rssfeeders'):

            utility.window['rssfeeders'].onRestore()

        

    def findFeeder(self, url = None, name = None):

        """

        FEEDERS: Find feeder by URL or name

        """

        for feed in self.feeds:

            if url and feed.options.url == url:

                return feed

            if name and feed.options.name == name:

                return feed

        return None



    #######################

    # Rules

    #######################

    def addRule(self, args = [], kwargs = {}):

        """

        RULES: Add new rule

        """

        if isinstance(args, Rule):

            rule = args

        else:

            rule = Rule(*args, **kwargs)

        self.rules.append(rule)

        self.config.save("rules")



    def removeRule(self, index):

        """

        RULES: Remove rule by index

        """

        del self.rules[index]

        self.config.save("rules")



    #######################

    # History

    #######################    

    def addToHistory(self, article):

        """

        HISTORY: Add article to history

        """

        self.history[article.url.lower()] = article.title

        self.config.save("history")

        self.updateList()

        

        match, episodes, name = self.GetEpisodes(article.title)

        if match:

            self.addToEpHistory(name, episodes)

        

    def addLinkToHistory(self, link):

        """

        HISTORY: Add link to history

        """

        self.history[link.lower()] = link

        self.config.save("history")

        self.updateList()

                        

    def removeFromHistory(self, url):

        """

        HISTORY: Remove article from history

        """

        del self.history[url.lower()]

        self.config.save("history")



    def isInHistory(self, article):

        """

        HISTORY: Check if an article is in history

        """

        return article.getURL() in self.history



    def clearHistory(self):

        """

        HISTORY: Clears history

        """

        self.history = {}

        self.config.save("history")

        

    def addToEpHistory(self, name, episodes):

        """

        HISTORY: Add to episodes history

        """

        oldepsave = self.config.episodes.Read(name.lower()) or "[]"

        oldepsave = eval(oldepsave)

        episodes = set(episodes)

        episodes.update(oldepsave)

        episodes = list(episodes)

        episodes.sort()

        self.config.episodes.Write(name, episodes)

        self.config.episodes.Flush()

        

    def removeFromEpHistory(self, name, episode):

        """

        HISTORY: Remove from episodes history

        """

        oldepsave = self.config.episodes.Read(name.lower()) or "[]"

        oldepsave = list(eval(oldepsave))

        if episode in oldepsave:

            oldepsave.remove(episode)

        self.config.episodes.Write(name, oldepsave)

        self.config.episodes.Flush()



    def clearEpHistory(self):

        self.config.episodes.DeleteGroup()

        self.config.episodes.Flush()



    #######################

    # Timer

    #######################    

    def startTimer(self):

        """

        TIMER: Start the auto-grab torrents timer

        """

        self.timer.Start(1000)

        if self.timerCount == -1:

            self.timerCount = utility.config.Read('rsstimer', "int")

            self.timerCount = max(self.timerCount, 300)

            self.timerActivated = time()



    def stopTimer(self):

        """

        TIMER: Stop the auto-grab torrents timer

        """

        self.timerCount = -1

        self.timer.Stop()



    def onTimer(self, event):

        """

        TIMER: timer tick

        """

        current = time() - self.timerActivated

        if current > self.timerCount:

            self.onUpdateAll()



    #######################

    # Html Torrent Scanner

    #######################

    def HtmlTorrentScannerFailed(self, article, message):

        """

        Called from HtmlTorrentScanner when faild

        """

        if self.htmlscanners.has_key(article.key):

            del self.htmlscanners[article.key]

        

        self.invokeLater(self._HtmlTorrentScannerFailed, [article.url, message])        



    def _HtmlTorrentScannerFailed(self, url, message):

        utility.frame.SetStatusText(message + url)

        wx.LogError(message + ' ' + url)



    def HtmlTorrentScannerSucceeded(self, article, rule, data):

        """

        Called from HtmlTorrentScanner when succeeded

        """

        if self.htmlscanners.has_key(article.key):

            del self.htmlscanners[article.key]

            

        self.addToHistory(article)

        

        if rule:

            rule.last_match = long(time())

            if rule.Type == "normal" and int(rule.Settings[0]):

                rule.active = False



        self.invokeLater(self._HtmlTorrentScannerSucceeded, [article, data, rule])



    def _HtmlTorrentScannerSucceeded(self, article, data, rule):

        utility.frame.SetStatusText(_("Downloaded:") + ' ' + article.title)

        wx.LogMessage(_("Downloaded:") + ' ' + article.title)

        self.addTorrentFromData(data, rule)



    #######################

    # Checks

    #######################

    def GetEpisodes(self, name):

        class Item(object):

            __slots__ = ('entrytitle', 'url')

        item = Item()

        item.entrytitle = name

        item.url = ""

        return self.CheckEpisodes(item)

        

    def CheckEpisodes(self, rssarticle, name = None, settings = None):

        """

        Episode checker

        @Arguments  :   str rssarticle  = RSS entry to match

                        str name        = Name of the show

                        str settings    = RSS rule settings

        @Returns    :   ([bool]Found a match, [list]episodes, [string]name)

        """                

        # Try to find a match in entrytitle

        string = rssarticle.entrytitle.lower()

        match, current = False, 0

        while not match and current < len(matchesmodes):

            match = matchesmodes[current].match(string)

            current += 1

            

        # Try to find a match in URL

        if not match:

            string = rssarticle.url.lower()

            if string.startswith("http://"):

                string = string.split("/")[-1]

                if "name=" in string:

                    string = string.split("name=")[-1]

            match, current = False, 0

            while not match and current < len(matchesmodes):

                match = matchesmodes[current].match(string)

                current += 1



        # No match found

        if not match:

            return False, None, None



        # Found a match, list its episodes

        episodes = []

        if len(match.groups()) == 2:

            # Single season

            seasonStart = int(match.group(1))

            episodesRange = (seasonStart, 0, seasonStart, 9999)

            episodes.append((seasonStart, -1))

        elif len(match.groups()) == 3:

            # Single episode

            seasonStart = int(match.group(2))

            episodeStart = int(match.group(3))

            episodesRange = (seasonStart, episodeStart, seasonStart, episodeStart)

            episodes.append((seasonStart, episodeStart))

        elif len(match.groups()) == 4:

            # Multiple episodes, same season

            seasonStart = int(match.group(2))

            episodeStart = int(match.group(3))

            episodeEnd = int(match.group(4))

            episodesRange = (seasonStart, episodeStart, seasonStart, episodeEnd)

            i = episodeStart

            while i <= episodeEnd:  

                episodes.append((seasonStart, i))

                i+=1

        elif len(match.groups()) == 5:

            # Multiple episodes, different seasons

            seasonStart = int(match.group(2))

            episodeStart = int(match.group(3))

            seasonEnd = int(match.group(4))

            episodeEnd = int(match.group(5))

            episodesRange = (seasonStart, episodeStart, seasonEnd, episodeEnd)

            s = seasonStart

            while s <= seasonEnd:

                if s == seasonStart:# partial season

                    for x in xrange(episodeStart, 25):

                        episodes.append((s, x))

                elif s < seasonEnd: # -1 = Full season

                    episodes.append((s, -1))

                else:               # specific episodes

                    for x in xrange(1, episodeEnd+1):

                        episodes.append((s, x))

                s+=1

        else:

            # No match found

            return False, None, None



        # Get the show name

        if not name:

            name = match.group(1)

            n = re.match(r'\[\w+\](.*)', name)

            if n:

                name = n.group(1)

            name = name.replace(".", " ").replace("[", " ").replace("]", " ").replace("!", " ").replace("{", " ").strip()



        # Check if needed

        existingEpisodes = self.config.episodes.Read(name.lower())

        if existingEpisodes:

            need = False

            existingEpisodes = eval(existingEpisodes)

            for episode in episodes:

                if not episode in existingEpisodes and \

                   not (episode[0], -1) in existingEpisodes: 

                    need = True

                    break

            # I don't need it...

            if not need:

                return False, None, None

        

        # No settings, just get it

        if not settings:

            return True, episodes, name



        # Match rule settings

        match, current = False, 0

        while not match and current < len(matchesmodes):

            match = matchesmodes[current].match(settings.lower())

            current += 1

            

        # No settings, just get it

        if not match:

            return True, episodes, name



        settings = match.groups()

        if len(match.groups()) == 5:

            setEpisodesRange = (int(settings[1]), int(settings[2]), int(settings[3]), int(settings[4]))

        elif len(match.groups()) == 4:

            setEpisodesRange = (int(settings[1]), int(settings[2]), int(settings[1]), int(settings[3]))

        elif len(match.groups()) == 3:

            setEpisodesRange = (int(settings[1]), int(settings[2]), int(settings[1]), int(settings[2]))

        elif len(match.groups()) == 2:

            setEpisodesRange = (int(settings[1]), 0, int(settings[1]), 9999)            

        else:

            return True, episodes, name



        if self.inRange(episodesRange, setEpisodesRange):

            return True, episodes, name

        return False, None, None

    

    def inRange(self, episodesRange, setEpisodesRange):

        if episodesRange[0] < setEpisodesRange[0]:

            return False

        if episodesRange[0] == setEpisodesRange[0] and episodesRange[1] < setEpisodesRange[1]:

            return False

        if episodesRange[2] > setEpisodesRange[2]:

            return False

        if episodesRange[2] == setEpisodesRange[2] and episodesRange[3] > setEpisodesRange[3]:

            return False

        return True

        

    #######################

    # Functionality

    #######################

    def updateAll(self, event = None):

        """

        Get new results from all feeds

        """

        feeds = [feed for feed in self.feeds if feed.options.active]

        if not feeds:

            self.onUpdateAllDone()



        GetFeeds(self, feeds).start()

        

        return True



    def updateList(self, feed = None):

        """

        updates the list safely

        """

        pass

    

    def updateSingle(self, feed = None, url = None):

        """

        Get results from single feed

        """        

        if url:

            feed = self.findFeeder(url = url)

        

        if feed:

            GetFeeds(self, feed).start()



    def onUpdateAllDone(self):

        pass

    

    def updateFeederSelector(self):

        pass                

    

    def addTorrentFromURL(self, url, cookies, rule = None):

        """

        Downloads a torrent from URL

        """

        label = None

        if rule:

            label = rule.label



        if rule is None:

            t = Thread(target=utility.queue.addtorrents.AddTorrentURL, args=[url,], kwargs={"cookies": cookies, "label":label})

            t.start()

            return True

        return utility.queue.addtorrents.AddTorrentURL(url, cookies = cookies, label = label)



    def addTorrentFromData(self, data, rule = None):

        """

        Adds a torrent from rawdata

        """

        label = None

        if rule:

            label = rule.label

        return utility.queue.addtorrents.AddTorrentFromBencodeData(data.read(), label = label)



    def Download(self, rssarticle):

        """

        Downloads a torrent from rss article

        """

        if rssarticle.url.endswith(".torrent"):

            self.addTorrentFromURL(rssarticle.url, cookies = rssarticle.feeder.getCookie())

            self.addToHistory(rssarticle)

        else:

            try:                    

                self.htmlscanners[rssarticle.key] = HtmlTorrentScanner(self, rssarticle)

                self.htmlscanners[rssarticle.key].start()

            except IOError:

                pass

            

#################################################################################    

class Rule(object):

    """

    Info for a single filtering rule

    """

    TYPES = ["normal", "tv", "date"]

    def __init__(self, name = "", feeds = [], filters = [], negfilters = [],

                 regex = False, min_interval = 0, last_match = 0, 

                 active = True, rule_type = 0, settings = "0|", label = 0):

        

        self.name           = name

        self.feeds          = feeds

        self.filters        = filters

        self.negfilters     = negfilters

        self.regex          = regex

        self.min_interval   = min_interval

        self.last_match     = last_match

        self.active         = active

        self.rule_type      = rule_type

        self.settings       = settings

        self.label          = label

     

    def GetType(self):

        return self.TYPES[self.rule_type]

    def SetType(self, nType):

        self.rule_type = nType

    Type = property(GetType, SetType)



    def GetSettings(self):

        return self.settings.split('|')

    def SetSettings(self, settings):

        self.settings = settings

    Settings = property(GetSettings, SetSettings)



    def toTuple(self):

        return (self.name, self.feeds, self.filters, self.negfilters, self.regex, self.min_interval,

                self.last_match, self.active, self.rule_type, self.settings, self.label)

    

    def fromTuple(self, Tuple):

        (self.name, self.feeds, self.filters, self.negfilters, self.regex, self.min_interval,

         self.last_match, self.active, self.rule_type, self.settings, self.label) = Tuple

        self.active = bool(self.active)

        self.regex = bool(self.regex)



#################################################################################

class RSSFeederOptions(object):

    """

    Info for a single RSSFeeder Options

    """

    def __init__(self, url, name = '', cookie = '', active = True):

        # Set feeder url

        if not ":/" in url and not ":\\" in url:

            url = "http://" + url

        self.url = url



        # Set feeder name

        if not name:

            self.name = self.url

        else:

            self.name = name

                    

        # Set feeder cookie

        self.cookie = cookie



        # Set Activity

        self.active = active



    def toTuple(self):

        return (self.url, self.name, self.cookie, self.active)



    def fromTuple(self, Tuple):

        (self.url, self.name, self.cookie, self.active) = Tuple

        if not ":/" in self.url and not ":\\" in self.url:

            self.url = "http://" + self.url

        if not self.name:

            self.name = self.url

        self.active = bool(self.active)

    

    def __nonzero__(self):

        return self.url and self.url != "http://"

    

#################################################################################        

class RSSFeeder(object):

    """

    RSSFeeder: manages a signle RSS feeder

    """            

    def __init__(self, parent, options):

        self.parent = parent

        self.rssarticles = {}

        self.etag = None

        self.modified = None

        self.filtersChanged = True

        self.options = options

        

    def __nonzero__(self):

        return bool(self.options)



    def getCookie(self):

        """

        Returns the feeder's cookie

        """

        return self.options.cookie



    def toTuple(self):

        return self.options.toTuple()

    

    def fromTuple(self, Tuple):

        """

        Sets feeder options from string

        """

        options = RSSFeederOptions("")

        options.fromTuple(Tuple)

        if options:

            self.options = options

            

    def Getdata(self, result):

        """

        Called automatically after getting all feed results

        """

        self.FeedResults(result)

        

    def FeedResults(self, result):

        """

        Parses the feed results

        """

        # Page not available

        if not hasattr(result, "status"):

            result.status = 0



        # No updates

        if result.status == 304 and self.filtersChanged:

            for article in self.articles:

                self.Match(article)

            self.filtersChanged = False

            return



        # No articles

        if not result.entries:

            return

        

        # Feeder details

        feedbase = result.feed.title_detail.base.lower()

        feedtitle = result.feed.title



        if self.options.name == self.options.url and feedtitle.strip():

            self.parent.changeName(self, feedtitle)



        # Articles details

        for entry in result.entries:

            rssarticle = RSSArticle(self, entry, feedbase, feedtitle)

            if not rssarticle.key in self.rssarticles:

                self.rssarticles[rssarticle.key] = rssarticle

                self.Match(rssarticle)

            elif self.filtersChanged:

                self.Match(rssarticle)

                

        self.filtersChanged = False

        

    def Match(self, rssarticle):

        """

        Gets an article and checks if user is interested with its content

        """        

        def RemoveMarks(item):

            """

            Adjust marks if not searching as regular expression

            """

            p = re.compile('(\*|\^|\$|\+|\?|\}|\{|\[|\]|\||\\\\|\(|\))')

            item = p.sub('.', item)

            p = re.compile('\#')

            item = p.sub('\d', item)

            return item



        # Only continue if torrent isn't in history list

        if self.parent.isInHistory(rssarticle):

            return



        for rule in self.parent.rules:

            if not rule.active:

                continue



            # Check if the feed is in the rule's feeds list

            if not rssarticle.feedbase in rule.feeds \

               and not rssarticle.feedbase2 in rule.feeds:

                continue



            # Minimum interval

            now = time()

            if rule.last_match and now > rule.last_match \

               and now - rule.last_match < rule.min_interval:

                continue

            

            # No filters 

            if not rule.negfilters and not rule.filters:

                continue



            # Check filters

            download = True

            if rule.filters:

                for filter in rule.filters:

                    if not rule.regex:

                        filter = RemoveMarks(filter)

                    find = re.search(filter, rssarticle.entrytitle, re.IGNORECASE)

                    if not find:

                        find = re.search(filter, rssarticle.url, re.IGNORECASE)

                    if not find:

                        download = False

                        break

                    

            if rule.negfilters:

                for filter in rule.negfilters:

                    if not rule.regex:

                        filter = RemoveMarks(filter)

                    find = re.search(filter, rssarticle.entrytitle, re.IGNORECASE)

                    if not find:

                        find = re.search(filter, rssarticle.url, re.IGNORECASE)

                    if find:

                        download = False

                        break



            # Download 

            if download and self.CheckIfDownload(rssarticle, rule):

                break



    def CheckIfDownload(self, rssarticle, rule):

        """

        Check if needs to download from the article

        """

        # Only download if torrent isn't in history list

        if self.parent.isInHistory(rssarticle): 

            download = False



        # TV rule

        elif rule.Type == "tv":                 

            download, episodes, name = self.parent.CheckEpisodes(rssarticle, rule.Settings[0], rule.Settings[1])

            if download:

                self.parent.addToEpHistory(name, episodes)



        # Date rule

        elif rule.Type == "date":               

            raise NotImplementedError

        

        # Normal rule

        else:                                   

            download = True



        if download and self.Download(rssarticle, rule):

            self.parent.addToHistory(rssarticle)

            rule.last_match = long(time())

            if rule.Type == "normal" and rule.Settings[0]:

                rule.active = False

            return True

        return False



    def Download(self, rssarticle, rule):

        """

        Downloads the torrent

        """

        if rssarticle.url.endswith(".torrent"):

            return self.parent.addTorrentFromURL(rssarticle.url, cookies = self.getCookie(), rule = rule)

        else:

            try:                    

                self.parent.htmlscanners[rssarticle.key] = HtmlTorrentScanner(self.parent, rssarticle, rule)

                self.parent.htmlscanners[rssarticle.key].start()

            except IOError:

                pass

            

        return False        



#################################################################################

class RSSArticle(object):

    """

    A single RSS Article (single row in result)

    """

    def __init__(self, feeder, feedentry, feedbase, feedtitle):

        self.feeder = feeder

        self.feedbase = feedbase        

        self.feedbase2 = feedbase.replace('www.', '', 1)

        self.feedtitle = feedtitle

        try:

            self.url = feedentry.enclosures[0].url

        except:

            try:

                self.url = feedentry.links[0].href

            except:

                self.url = "..."

        self.titlelink = feedentry.get('link', "")

        self.entrytitle = feedentry.get('title', "")

        self.entrysummary = feedentry.get('description', "")

        self.entrydatetime = feedentry.get('date', "")

        if not self.entrydatetime:

            self.entrydatetime = "unknown"



        self.enrtydate = feedentry.get('updated_parsed', "")

        if self.enrtydate:

            self.enrtydate = mktime(self.enrtydate)

            

        self.title = self.entrytitle

        self.key = hash((self.title.lower(), self.url.lower(), self.entrydatetime.lower()))



    def getIdentity(self):

        return (self.title, self.url, self.entrydatetime)



    def getTitle(self):

        return self.title.lower()



    def getURL(self):

        return self.url.lower()



    def getTime(self):

        return self.entrydatetime.lower()

    

#################################################################################    

class ConfigManager(object):

    """

    A class for handling the RSS configuration

    """

    def __init__(self, parent):

        self.parent = parent

        self.__version = "1"

        

        # Load Config

        rulesPath   = os.path.join(utility.getConfigPath(), "rules."   + self.__version + ".conf")

        feedersPath = os.path.join(utility.getConfigPath(), "feeders." + self.__version + ".conf")

        historyPath = os.path.join(utility.getConfigPath(), "history." + self.__version + ".conf")

        episodePath = os.path.join(utility.getConfigPath(), "episode." + self.__version + ".conf")

        

        self.rules      = ConfigReader(rulesPath  , "LH/RSS", {})

        self.feeders    = ConfigReader(feedersPath, "LH/RSS", {})

        self.history    = ConfigReader(historyPath, "LH/RSS", {})

        self.episodes   = ConfigReader(episodePath, "LH/RSS", {})



        self.loadAll()

                

    def loadAll(self):

        """

        Loads configuration

        """

        self.load("feeds")

        self.load("rules")

        self.load("history")



    def load(self, obj):

        """

        Loads object from file

        """

        if obj == "feeds":

            data = self.feeders.Read("feeds", "bencode-list")

            for feed_data in data:

                feed = RSSFeeder(self.parent, None)

                feed.fromTuple(feed_data)

                if feed:

                    self.parent.feeds.append(feed)

                    

        elif obj == "rules":

            data = self.rules.Read("rules", "bencode-list")

            for rule_data in data:

                rule = Rule()

                rule.fromTuple(rule_data)

                self.parent.rules.append(rule)



        elif obj == "history":

            self.parent.history = self.history.Read("history", "bencode-dict")



        else:

            raise Exception("Unknown object")

            

    def saveAll(self):

        """

        Saves configuration

        """

        self.save("feeds")

        self.save("rules")

        self.save("history")

        

    def save(self, obj):

        """

        Saves object to file

        """

        if obj == "feeds":

            data = [feed.toTuple() for feed in self.parent.feeds]

            self.feeders.Write("feeds", data, "bencode-list")

            self.feeders.Flush()

        elif obj == "rules":

            data = [rule.toTuple() for rule in self.parent.rules]

            self.rules.Write("rules", data, "bencode-list")

            self.rules.Flush()

        elif obj == "history":

            self.history.Write("history", self.parent.history, "bencode-dict")

            self.history.Flush()

        else:

            raise Exception("Unknown object")

        

#################################################################################

class GetFeeds(Thread):

    """

    Helper class for parsing multiple feeds using ThreadPool

    """

    def __init__(self, parent, feeds):

        Thread.__init__(self)

        self.setDaemon(True)



        self.parent = parent

        

        self.single = False

        if type(feeds) is not list:

            feeds = [feeds]

            self.single = True

        self.feeds = feeds



    def handle_exception(self, request, exc_info):

        """

        This will be called when an exception occurs within a thread

        """

        pass

##        print "Exception occured in request #%s: %s" % \

##          (request.requestID, exc_info[1])

        

    def run(self):

        """

        Create the ThreadPool and add tasks

        """

        try:

            self._run()

        except:

            pass

        

    def _run(self):

        tasks = []

        for feed in self.feeds:

            tasks.extend(makeRequests(self.getFeed, [feed], exc_callback = self.handle_exception))

    

        tp = ThreadPool(4)



        for task in tasks:

            tp.putRequest(task)



        tp.wait()



        self.done()

        

    def getFeed(self, feed):

        """

        The task to perform: parse the feed

        """

        data = None



        try:

            # Get Data

            data = parse(feed.options.url, agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)', 

                         cookies=feed.getCookie(), etag=feed.etag, modified = feed.modified)



            # Save etags

            try:

                feed.etag = data.etag

            except:

                pass

            

            # Save last-modified

            if data.has_key('modified'):

                feed.modified = data.modified

        except:

            return False



        try:

            feed.Getdata(data)

        except wx.PyDeadObjectError:

            pass

        

        return True

        

    def done(self):

        """

        Called when all threads are done

        """

        try:

            if self.single:

                if self.feeds[0].rssarticles:

                    self.parent.updateList(self.feeds[0])

            else:

                self.parent.onUpdateAllDone()

        except wx.PyDeadObjectError:

            pass