/lib/galaxy/jobs/deferred/__init__.py

  1"""
  2Queue for running deferred code via plugins.
  3"""
  4import os, sys, logging, threading
  5from Queue import Queue, Empty
  6
  7from galaxy import model
  8from galaxy.util.bunch import Bunch
  9
 10log = logging.getLogger( __name__ )
 11
 12class DeferredJobQueue( object ):
 13    job_states = Bunch( READY = 'ready',
 14                        WAIT = 'wait',
 15                        INVALID = 'invalid' )
 16    def __init__( self, app ):
 17        self.app = app
 18        self.sa_session = app.model.context.current
 19        self.queue = Queue()
 20        self.plugins = {}
 21        self._load_plugins()
 22        self.sleeper = Sleeper()
 23        self.running = True
 24        self.waiting_jobs = []
 25        self.__check_jobs_at_startup()
 26        self.monitor_thread = threading.Thread( target=self.__monitor )
 27        self.monitor_thread.start()
 28        log.info( 'Deferred job queue started' )
 29    def _load_plugins( self ):
 30        for fname in os.listdir( os.path.dirname( __file__ ) ):
 31            if not fname.startswith( '_' ) and fname.endswith( '.py' ):
 32                name = fname[:-3]
 33                module_name = 'galaxy.jobs.deferred.' + name
 34                try:
 35                    module = __import__( module_name )
 36                except:
 37                    log.exception( 'Deferred job plugin appears to exist but is not loadable: %s' % module_name )
 38                    continue
 39                for comp in module_name.split( "." )[1:]:
 40                    module = getattr( module, comp )
 41                if '__all__' not in dir( module ):
 42                    log.error( 'Plugin "%s" does not contain a list of exported classes in __all__' % module_name )
 43                    continue
 44                for obj in module.__all__:
 45                    display_name = ':'.join( ( module_name, obj ) )
 46                    plugin = getattr( module, obj )
 47                    for name in ( 'check_job', 'run_job' ):
 48                        if name not in dir( plugin ):
 49                            log.error( 'Plugin "%s" does not contain required method "%s()"' % ( display_name, name ) )
 50                            break
 51                    else:
 52                        self.plugins[obj] = plugin( self.app )
 53                        self.plugins[obj].job_states = self.job_states
 54                        log.debug( 'Loaded deferred job plugin: %s' % display_name )
 55    def __check_jobs_at_startup( self ):
 56        waiting_jobs = self.sa_session.query( model.DeferredJob ) \
 57                                      .filter( model.DeferredJob.state == model.DeferredJob.states.WAITING ).all()
 58        for job in waiting_jobs:
 59            if not self.__check_job_plugin( job ):
 60                continue
 61            if 'check_interval' in dir( self.plugins[job.plugin] ):
 62                job.check_interval = self.plugins[job.plugin].check_interval
 63            log.info( 'Recovered deferred job (id: %s) at startup' % job.id )
 64            # Pass the job ID as opposed to the job, since the monitor thread
 65            # needs to load it in its own threadlocal scoped session.
 66            self.waiting_jobs.append( job.id )
 67    def __monitor( self ):
 68        while self.running:
 69            try:
 70                self.__monitor_step()
 71            except:
 72                log.exception( 'Exception in monitor_step' )
 73            self.sleeper.sleep( 1 )
 74        log.info( 'job queue stopped' )
 75    def __monitor_step( self ):
 76        # TODO: Querying the database with this frequency is bad, we need message passing
 77        new_jobs = self.sa_session.query( model.DeferredJob ) \
 78                                  .filter( model.DeferredJob.state == model.DeferredJob.states.NEW ).all()
 79        for job in new_jobs:
 80            if not self.__check_job_plugin( job ):
 81                continue
 82            job.state = model.DeferredJob.states.WAITING
 83            self.sa_session.add( job )
 84            self.sa_session.flush()
 85            if 'check_interval' in dir( self.plugins[job.plugin] ):
 86                job.check_interval = self.plugins[job.plugin].check_interval
 87            self.waiting_jobs.append( job )
 88        new_waiting = []
 89        for job in self.waiting_jobs:
 90            try:
 91                # Recovered jobs are passed in by ID
 92                assert type( job ) is int
 93                job = self.sa_session.query( model.DeferredJob ).get( job )
 94            except:
 95                pass
 96            if job.is_check_time:
 97                try:
 98                    job_state = self.plugins[job.plugin].check_job( job )
 99                except Exception, e:
100                    self.__fail_job( job )
101                    log.exception( 'Set deferred job %s to error because of an exception in check_job(): %s' % ( job.id, str( e ) ) )
102                    continue
103                if job_state == self.job_states.READY:
104                    try:
105                        self.plugins[job.plugin].run_job( job )
106                    except Exception, e:
107                        self.__fail_job( job )
108                        log.exception( 'Set deferred job %s to error because of an exception in run_job(): %s' % ( job.id, str( e ) ) )
109                        continue
110                elif job_state == self.job_states.INVALID:
111                    self.__fail_job( job )
112                    log.error( 'Unable to run deferred job (id: %s): Plugin "%s" marked it as invalid' % ( job.id, job.plugin ) )
113                    continue
114                else:
115                    new_waiting.append( job )
116                job.last_check = 'now'
117            else:
118                new_waiting.append( job )
119        self.waiting_jobs = new_waiting
120    def __check_job_plugin( self, job ):
121        if job.plugin not in self.plugins:
122            log.error( 'Invalid deferred job plugin: %s' ) % job.plugin
123            job.state = model.DeferredJob.states.ERROR
124            self.sa_session.add( job )
125            self.sa_session.flush()
126            return False
127        return True
128    def __check_if_ready_to_run( self, job ):
129        return self.plugins[job.plugin].check_job( job )
130    def __fail_job( self, job ):
131        job.state = model.DeferredJob.states.ERROR
132        self.sa_session.add( job )
133        self.sa_session.flush()
134    def shutdown( self ):
135        self.running = False
136        self.sleeper.wake()
137
138class Sleeper( object ):
139    """
140    Provides a 'sleep' method that sleeps for a number of seconds *unless*
141    the notify method is called (from a different thread).
142    """
143    def __init__( self ):
144        self.condition = threading.Condition()
145    def sleep( self, seconds ):
146        self.condition.acquire()
147        self.condition.wait( seconds )
148        self.condition.release()
149    def wake( self ):
150        self.condition.acquire()
151        self.condition.notify()
152        self.condition.release()
153
154class FakeTrans( object ):
155    """A fake trans for calling the external set metadata tool"""
156    def __init__( self, app, history=None, user=None):
157        class Dummy( object ):
158            def __init__( self ):
159                self.id = None
160        self.app = app
161        self.sa_session = app.model.context.current
162        self.dummy = Dummy()
163        if not history:
164            self.history = Dummy()
165        else:
166            self.history = history
167        if not user:
168            self.user = Dummy()
169        else:
170            self.user = user
171        self.model = app.model
172    def get_galaxy_session( self ):
173        return self.dummy
174    def log_event( self, message, tool_id=None ):
175        pass
176    def get_current_user_roles( self ):
177        if self.user:
178            return self.user.all_roles()
179        else:
180            return []
181    def db_dataset_for( self, dbkey ):
182        if self.history is None:
183            return None
184        datasets = self.sa_session.query( self.app.model.HistoryDatasetAssociation ) \
185                                  .filter_by( deleted=False, history_id=self.history.id, extension="len" )
186        for ds in datasets:
187            if dbkey == ds.dbkey:
188                return ds
189        return None