PageRenderTime 91ms CodeModel.GetById 23ms app.highlight 54ms RepoModel.GetById 2ms app.codeStats 0ms

/lib/galaxy/jobs/__init__.py

https://bitbucket.org/cistrome/cistrome-harvard/
Python | 1663 lines | 1452 code | 55 blank | 156 comment | 116 complexity | 67fd55bd5e5369ca562536fa8927b8c2 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1"""
   2Support for running a tool in Galaxy via an internal job management system
   3"""
   4from abc import ABCMeta
   5from abc import abstractmethod
   6
   7import time
   8import copy
   9import datetime
  10import galaxy
  11import logging
  12import os
  13import pwd
  14import random
  15import re
  16import shutil
  17import subprocess
  18import sys
  19import traceback
  20from galaxy import model, util
  21from galaxy.datatypes import metadata
  22from galaxy.exceptions import ObjectInvalid, ObjectNotFound
  23from galaxy.jobs.actions.post import ActionBox
  24from galaxy.jobs.mapper import JobRunnerMapper
  25from galaxy.jobs.runners import BaseJobRunner
  26from galaxy.util.bunch import Bunch
  27from galaxy.util.expressions import ExpressionContext
  28from galaxy.util.json import from_json_string
  29from galaxy.util import unicodify
  30
  31from .output_checker import check_output
  32from .datasets import TaskPathRewriter
  33from .datasets import OutputsToWorkingDirectoryPathRewriter
  34from .datasets import NullDatasetPathRewriter
  35from .datasets import DatasetPath
  36
  37log = logging.getLogger( __name__ )
  38
  39DATABASE_MAX_STRING_SIZE = util.DATABASE_MAX_STRING_SIZE
  40DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY
  41
  42# This file, if created in the job's working directory, will be used for
  43# setting advanced metadata properties on the job and its associated outputs.
  44# This interface is currently experimental, is only used by the upload tool,
  45# and should eventually become API'd
  46TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
  47
  48
  49class JobDestination( Bunch ):
  50    """
  51    Provides details about where a job runs
  52    """
  53    def __init__(self, **kwds):
  54        self['id'] = None
  55        self['url'] = None
  56        self['tags'] = None
  57        self['runner'] = None
  58        self['legacy'] = False
  59        self['converted'] = False
  60        # dict is appropriate (rather than a bunch) since keys may not be valid as attributes
  61        self['params'] = dict()
  62        super(JobDestination, self).__init__(**kwds)
  63
  64        # Store tags as a list
  65        if self.tags is not None:
  66            self['tags'] = [ x.strip() for x in self.tags.split(',') ]
  67
  68
  69class JobToolConfiguration( Bunch ):
  70    """
  71    Provides details on what handler and destination a tool should use
  72
  73    A JobToolConfiguration will have the required attribute 'id' and optional
  74    attributes 'handler', 'destination', and 'params'
  75    """
  76    def __init__(self, **kwds):
  77        self['handler'] = None
  78        self['destination'] = None
  79        self['params'] = dict()
  80        super(JobToolConfiguration, self).__init__(**kwds)
  81
  82
  83class JobConfiguration( object ):
  84    """A parser and interface to advanced job management features.
  85
  86    These features are configured in the job configuration, by default, ``job_conf.xml``
  87    """
  88    DEFAULT_NWORKERS = 4
  89
  90    def __init__(self, app):
  91        """Parse the job configuration XML.
  92        """
  93        self.app = app
  94        self.runner_plugins = []
  95        self.handlers = {}
  96        self.handler_runner_plugins = {}
  97        self.default_handler_id = None
  98        self.destinations = {}
  99        self.destination_tags = {}
 100        self.default_destination_id = None
 101        self.tools = {}
 102        self.limits = Bunch()
 103
 104        # Initialize the config
 105        try:
 106            tree = util.parse_xml(self.app.config.job_config_file)
 107            self.__parse_job_conf_xml(tree)
 108        except IOError:
 109            log.warning( 'Job configuration "%s" does not exist, using legacy job configuration from Galaxy config file "%s" instead' % ( self.app.config.job_config_file, self.app.config.config_file ) )
 110            self.__parse_job_conf_legacy()
 111
 112    def __parse_job_conf_xml(self, tree):
 113        """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml).
 114
 115        :param tree: Object representing the root ``<job_conf>`` object in the job config file.
 116        :type tree: ``xml.etree.ElementTree.Element``
 117        """
 118        root = tree.getroot()
 119        log.debug('Loading job configuration from %s' % self.app.config.job_config_file)
 120
 121        # Parse job plugins
 122        plugins = root.find('plugins')
 123        if plugins is not None:
 124            for plugin in self.__findall_with_required(plugins, 'plugin', ('id', 'type', 'load')):
 125                if plugin.get('type') == 'runner':
 126                    workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS))
 127                    runner_kwds = self.__get_params(plugin)
 128                    runner_info = dict(id=plugin.get('id'),
 129                                       load=plugin.get('load'),
 130                                       workers=int(workers),
 131                                       kwds=runner_kwds)
 132                    self.runner_plugins.append(runner_info)
 133                else:
 134                    log.error('Unknown plugin type: %s' % plugin.get('type'))
 135        # Load tasks if configured
 136        if self.app.config.use_tasked_jobs:
 137            self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
 138
 139        # Parse handlers
 140        handlers = root.find('handlers')
 141        if handlers is not None:
 142            for handler in self.__findall_with_required(handlers, 'handler'):
 143                id = handler.get('id')
 144                if id in self.handlers:
 145                    log.error("Handler '%s' overlaps handler with the same name, ignoring" % id)
 146                else:
 147                    log.debug("Read definition for handler '%s'" % id)
 148                    self.handlers[id] = (id,)
 149                    for plugin in handler.findall('plugin'):
 150                        if id not in self.handler_runner_plugins:
 151                            self.handler_runner_plugins[id] = []
 152                        self.handler_runner_plugins[id].append( plugin.get('id') )
 153                    if handler.get('tags', None) is not None:
 154                        for tag in [ x.strip() for x in handler.get('tags').split(',') ]:
 155                            if tag in self.handlers:
 156                                self.handlers[tag].append(id)
 157                            else:
 158                                self.handlers[tag] = [id]
 159
 160        # Determine the default handler(s)
 161        self.default_handler_id = self.__get_default(handlers, self.handlers.keys())
 162
 163        # Parse destinations
 164        destinations = root.find('destinations')
 165        for destination in self.__findall_with_required(destinations, 'destination', ('id', 'runner')):
 166            id = destination.get('id')
 167            job_destination = JobDestination(**dict(destination.items()))
 168            job_destination['params'] = self.__get_params(destination)
 169            self.destinations[id] = (job_destination,)
 170            if job_destination.tags is not None:
 171                for tag in job_destination.tags:
 172                    if tag not in self.destinations:
 173                        self.destinations[tag] = []
 174                    self.destinations[tag].append(job_destination)
 175
 176        # Determine the default destination
 177        self.default_destination_id = self.__get_default(destinations, self.destinations.keys())
 178
 179        # Parse tool mappings
 180        tools = root.find('tools')
 181        if tools is not None:
 182            for tool in self.__findall_with_required(tools, 'tool'):
 183                # There can be multiple definitions with identical ids, but different params
 184                id = tool.get('id').lower().rstrip('/')
 185                if id not in self.tools:
 186                    self.tools[id] = list()
 187                self.tools[id].append(JobToolConfiguration(**dict(tool.items())))
 188                self.tools[id][-1]['params'] = self.__get_params(tool)
 189
 190        types = dict(registered_user_concurrent_jobs=int,
 191                     anonymous_user_concurrent_jobs=int,
 192                     walltime=str,
 193                     output_size=int)
 194
 195        self.limits = Bunch(registered_user_concurrent_jobs=None,
 196                            anonymous_user_concurrent_jobs=None,
 197                            walltime=None,
 198                            walltime_delta=None,
 199                            output_size=None,
 200                            concurrent_jobs={})
 201
 202        # Parse job limits
 203        limits = root.find('limits')
 204        if limits is not None:
 205            for limit in self.__findall_with_required(limits, 'limit', ('type',)):
 206                type = limit.get('type')
 207                if type == 'concurrent_jobs':
 208                    id = limit.get('tag', None) or limit.get('id')
 209                    self.limits.concurrent_jobs[id] = int(limit.text)
 210                elif limit.text:
 211                    self.limits.__dict__[type] = types.get(type, str)(limit.text)
 212
 213        if self.limits.walltime is not None:
 214            h, m, s = [ int( v ) for v in self.limits.walltime.split( ':' ) ]
 215            self.limits.walltime_delta = datetime.timedelta( 0, s, 0, 0, m, h )
 216
 217        log.debug('Done loading job configuration')
 218
 219    def __parse_job_conf_legacy(self):
 220        """Loads the old-style job configuration from options in the galaxy config file (by default, universe_wsgi.ini).
 221        """
 222        log.debug('Loading job configuration from %s' % self.app.config.config_file)
 223
 224        # Always load local and lwr
 225        self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers), dict(id='lwr', load='lwr', workers=self.app.config.cluster_job_queue_workers)]
 226        # Load tasks if configured
 227        if self.app.config.use_tasked_jobs:
 228            self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
 229        for runner in self.app.config.start_job_runners:
 230            self.runner_plugins.append(dict(id=runner, load=runner, workers=self.app.config.cluster_job_queue_workers))
 231
 232        # Set the handlers
 233        for id in self.app.config.job_handlers:
 234            self.handlers[id] = (id,)
 235
 236        self.handlers['default_job_handlers'] = self.app.config.default_job_handlers
 237        self.default_handler_id = 'default_job_handlers'
 238
 239        # Set tool handler configs
 240        for id, tool_handlers in self.app.config.tool_handlers.items():
 241            self.tools[id] = list()
 242            for handler_config in tool_handlers:
 243                # rename the 'name' key to 'handler'
 244                handler_config['handler'] = handler_config.pop('name')
 245                self.tools[id].append(JobToolConfiguration(**handler_config))
 246
 247        # Set tool runner configs
 248        for id, tool_runners in self.app.config.tool_runners.items():
 249            # Might have been created in the handler parsing above
 250            if id not in self.tools:
 251                self.tools[id] = list()
 252            for runner_config in tool_runners:
 253                url = runner_config['url']
 254                if url not in self.destinations:
 255                    # Create a new "legacy" JobDestination - it will have its URL converted to a destination params once the appropriate plugin has loaded
 256                    self.destinations[url] = (JobDestination(id=url, runner=url.split(':', 1)[0], url=url, legacy=True, converted=False),)
 257                for tool_conf in self.tools[id]:
 258                    if tool_conf.params == runner_config.get('params', {}):
 259                        tool_conf['destination'] = url
 260                        break
 261                else:
 262                    # There was not an existing config (from the handlers section) with the same params
 263                    # rename the 'url' key to 'destination'
 264                    runner_config['destination'] = runner_config.pop('url')
 265                    self.tools[id].append(JobToolConfiguration(**runner_config))
 266
 267        self.destinations[self.app.config.default_cluster_job_runner] = (JobDestination(id=self.app.config.default_cluster_job_runner, runner=self.app.config.default_cluster_job_runner.split(':', 1)[0], url=self.app.config.default_cluster_job_runner, legacy=True, converted=False),)
 268        self.default_destination_id = self.app.config.default_cluster_job_runner
 269
 270        # Set the job limits
 271        self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit,
 272                            anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit,
 273                            walltime=self.app.config.job_walltime,
 274                            walltime_delta=self.app.config.job_walltime_delta,
 275                            output_size=self.app.config.output_size_limit,
 276                            concurrent_jobs={})
 277
 278        log.debug('Done loading job configuration')
 279
 280    def __get_default(self, parent, names):
 281        """Returns the default attribute set in a parent tag like <handlers> or <destinations>, or return the ID of the child, if there is no explicit default and only one child.
 282
 283        :param parent: Object representing a tag that may or may not have a 'default' attribute.
 284        :type parent: ``xml.etree.ElementTree.Element``
 285        :param names: The list of destination or handler IDs or tags that were loaded.
 286        :type names: list of str
 287
 288        :returns: str -- id or tag representing the default.
 289        """
 290        rval = parent.get('default')
 291        if rval is not None:
 292            # If the parent element has a 'default' attribute, use the id or tag in that attribute
 293            if rval not in names:
 294                raise Exception("<%s> default attribute '%s' does not match a defined id or tag in a child element" % (parent.tag, rval))
 295            log.debug("<%s> default set to child with id or tag '%s'" % (parent.tag, rval))
 296        elif len(names) == 1:
 297            log.info("Setting <%s> default to child with id '%s'" % (parent.tag, names[0]))
 298            rval = names[0]
 299        else:
 300            raise Exception("No <%s> default specified, please specify a valid id or tag with the 'default' attribute" % parent.tag)
 301        return rval
 302
 303    def __findall_with_required(self, parent, match, attribs=None):
 304        """Like ``xml.etree.ElementTree.Element.findall()``, except only returns children that have the specified attribs.
 305
 306        :param parent: Parent element in which to find.
 307        :type parent: ``xml.etree.ElementTree.Element``
 308        :param match: Name of child elements to find.
 309        :type match: str
 310        :param attribs: List of required attributes in children elements.
 311        :type attribs: list of str
 312
 313        :returns: list of ``xml.etree.ElementTree.Element``
 314        """
 315        rval = []
 316        if attribs is None:
 317            attribs = ('id',)
 318        for elem in parent.findall(match):
 319            for attrib in attribs:
 320                if attrib not in elem.attrib:
 321                    log.warning("required '%s' attribute is missing from <%s> element" % (attrib, match))
 322                    break
 323            else:
 324                rval.append(elem)
 325        return rval
 326
 327    def __get_params(self, parent):
 328        """Parses any child <param> tags in to a dictionary suitable for persistence.
 329
 330        :param parent: Parent element in which to find child <param> tags.
 331        :type parent: ``xml.etree.ElementTree.Element``
 332
 333        :returns: dict
 334        """
 335        rval = {}
 336        for param in parent.findall('param'):
 337            rval[param.get('id')] = param.text
 338        return rval
 339
 340    @property
 341    def default_job_tool_configuration(self):
 342        """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration.  It consists of a reference to the default handler and default destination.
 343
 344        :returns: JobToolConfiguration -- a representation of a <tool> element that uses the default handler and destination
 345        """
 346        return JobToolConfiguration(id='default', handler=self.default_handler_id, destination=self.default_destination_id)
 347
 348    # Called upon instantiation of a Tool object
 349    def get_job_tool_configurations(self, ids):
 350        """Get all configured JobToolConfigurations for a tool ID, or, if given a list of IDs, the JobToolConfigurations for the first id in ``ids`` matching a tool definition.
 351
 352        .. note::
 353
 354            You should not mix tool shed tool IDs, versionless tool shed IDs, and tool config tool IDs that refer to the same tool.
 355
 356        :param ids: Tool ID or IDs to fetch the JobToolConfiguration of.
 357        :type ids: list or str.
 358        :returns: list -- JobToolConfiguration Bunches representing <tool> elements matching the specified ID(s).
 359
 360        Example tool ID strings include:
 361
 362        * Full tool shed id: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool/1.0.0``
 363        * Tool shed id less version: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool``
 364        * Tool config tool id: ``filter_tool``
 365        """
 366        rval = []
 367        # listify if ids is a single (string) id
 368        ids = util.listify(ids)
 369        for id in ids:
 370            if id in self.tools:
 371                # If a tool has definitions that include job params but not a
 372                # definition for jobs without params, include the default
 373                # config
 374                for job_tool_configuration in self.tools[id]:
 375                    if not job_tool_configuration.params:
 376                        break
 377                else:
 378                    rval.append(self.default_job_tool_configuration)
 379                rval.extend(self.tools[id])
 380                break
 381        else:
 382            rval.append(self.default_job_tool_configuration)
 383        return rval
 384
 385    def __get_single_item(self, collection):
 386        """Given a collection of handlers or destinations, return one item from the collection at random.
 387        """
 388        # Done like this to avoid random under the assumption it's faster to avoid it
 389        if len(collection) == 1:
 390            return collection[0]
 391        else:
 392            return random.choice(collection)
 393
 394    # This is called by Tool.get_job_handler()
 395    def get_handler(self, id_or_tag):
 396        """Given a handler ID or tag, return the provided ID or an ID matching the provided tag
 397
 398        :param id_or_tag: A handler ID or tag.
 399        :type id_or_tag: str
 400
 401        :returns: str -- A valid job handler ID.
 402        """
 403        if id_or_tag is None:
 404            id_or_tag = self.default_handler_id
 405        return self.__get_single_item(self.handlers[id_or_tag])
 406
 407    def get_destination(self, id_or_tag):
 408        """Given a destination ID or tag, return the JobDestination matching the provided ID or tag
 409
 410        :param id_or_tag: A destination ID or tag.
 411        :type id_or_tag: str
 412
 413        :returns: JobDestination -- A valid destination
 414
 415        Destinations are deepcopied as they are expected to be passed in to job
 416        runners, which will modify them for persisting params set at runtime.
 417        """
 418        if id_or_tag is None:
 419            id_or_tag = self.default_destination_id
 420        return copy.deepcopy(self.__get_single_item(self.destinations[id_or_tag]))
 421
 422    def get_destinations(self, id_or_tag):
 423        """Given a destination ID or tag, return all JobDestinations matching the provided ID or tag
 424
 425        :param id_or_tag: A destination ID or tag.
 426        :type id_or_tag: str
 427
 428        :returns: list or tuple of JobDestinations
 429
 430        Destinations are not deepcopied, so they should not be passed to
 431        anything which might modify them.
 432        """
 433        return self.destinations.get(id_or_tag, None)
 434
 435    def get_job_runner_plugins(self, handler_id):
 436        """Load all configured job runner plugins
 437
 438        :returns: list of job runner plugins
 439        """
 440        rval = {}
 441        if handler_id in self.handler_runner_plugins:
 442            plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ]
 443            log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) )
 444        else:
 445            plugins_to_load = self.runner_plugins
 446            log.info( "Handler '%s' will load all configured runner plugins", handler_id )
 447        for runner in plugins_to_load:
 448            class_names = []
 449            module = None
 450            id = runner['id']
 451            load = runner['load']
 452            if ':' in load:
 453                # Name to load was specified as '<module>:<class>'
 454                module_name, class_name = load.rsplit(':', 1)
 455                class_names = [ class_name ]
 456                module = __import__( module_name )
 457            else:
 458                # Name to load was specified as '<module>'
 459                if '.' not in load:
 460                    # For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name
 461                    module_name = 'galaxy.jobs.runners.' + load
 462                    try:
 463                        module = __import__( module_name )
 464                    except ImportError:
 465                        # No such module, we'll retry without prepending galaxy.jobs.runners.
 466                        # All other exceptions (e.g. something wrong with the module code) will raise
 467                        pass
 468                if module is None:
 469                    # If the name included a '.' or loading from the static runners path failed, try the original name
 470                    module = __import__( load )
 471                    module_name = load
 472            if module is None:
 473                # Module couldn't be loaded, error should have already been displayed
 474                continue
 475            for comp in module_name.split( "." )[1:]:
 476                module = getattr( module, comp )
 477            if not class_names:
 478                # If there's not a ':', we check <module>.__all__ for class names
 479                try:
 480                    assert module.__all__
 481                    class_names = module.__all__
 482                except AssertionError:
 483                    log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % load )
 484                    continue
 485            for class_name in class_names:
 486                runner_class = getattr( module, class_name )
 487                try:
 488                    assert issubclass(runner_class, BaseJobRunner)
 489                except TypeError:
 490                    log.warning("A non-class name was found in __all__, ignoring: %s" % id)
 491                    continue
 492                except AssertionError:
 493                    log.warning("Job runner classes must be subclassed from BaseJobRunner, %s has bases: %s" % (id, runner_class.__bases__))
 494                    continue
 495                try:
 496                    rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) )
 497                except TypeError:
 498                    log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) )
 499                    rval[id] = runner_class( self.app )
 500                log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) )
 501        return rval
 502
 503    def is_id(self, collection):
 504        """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID
 505
 506        :param collection: A representation of a destination or handler
 507        :type collection: tuple or list
 508
 509        :returns: bool
 510        """
 511        return type(collection) == tuple
 512
 513    def is_tag(self, collection):
 514        """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID
 515
 516        :param collection: A representation of a destination or handler
 517        :type collection: tuple or list
 518
 519        :returns: bool
 520        """
 521        return type(collection) == list
 522
 523    def is_handler(self, server_name):
 524        """Given a server name, indicate whether the server is a job handler
 525
 526        :param server_name: The name to check
 527        :type server_name: str
 528
 529        :return: bool
 530        """
 531        for collection in self.handlers.values():
 532            if server_name in collection:
 533                return True
 534        return False
 535
 536    def convert_legacy_destinations(self, job_runners):
 537        """Converts legacy (from a URL) destinations to contain the appropriate runner params defined in the URL.
 538
 539        :param job_runners: All loaded job runner plugins.
 540        :type job_runners: list of job runner plugins
 541        """
 542        for id, destination in [ ( id, destinations[0] ) for id, destinations in self.destinations.items() if self.is_id(destinations) ]:
 543            # Only need to deal with real destinations, not members of tags
 544            if destination.legacy and not destination.converted:
 545                if destination.runner in job_runners:
 546                    destination.params = job_runners[destination.runner].url_to_destination(destination.url).params
 547                    destination.converted = True
 548                    if destination.params:
 549                        log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url))
 550                        for k, v in destination.params.items():
 551                            log.debug("    %s: %s" % (k, v))
 552                    else:
 553                        log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url))
 554                else:
 555                    log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner))
 556
 557
 558class JobWrapper( object ):
 559    """
 560    Wraps a 'model.Job' with convenience methods for running processes and
 561    state management.
 562    """
 563    def __init__( self, job, queue ):
 564        self.job_id = job.id
 565        self.session_id = job.session_id
 566        self.user_id = job.user_id
 567        self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None )
 568        self.queue = queue
 569        self.app = queue.app
 570        self.sa_session = self.app.model.context
 571        self.extra_filenames = []
 572        self.command_line = None
 573        # Tool versioning variables
 574        self.write_version_cmd = None
 575        self.version_string = ""
 576        self.galaxy_lib_dir = None
 577        # With job outputs in the working directory, we need the working
 578        # directory to be set before prepare is run, or else premature deletion
 579        # and job recovery fail.
 580        # Create the working dir if necessary
 581        try:
 582            self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
 583            self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
 584            log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory))
 585        except ObjectInvalid:
 586            raise Exception('Unable to create job working directory, job failure')
 587        self.dataset_path_rewriter = self._job_dataset_path_rewriter( self.working_directory )
 588        self.output_paths = None
 589        self.output_hdas_and_paths = None
 590        self.tool_provided_job_metadata = None
 591        # Wrapper holding the info required to restore and clean up from files used for setting metadata externally
 592        self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job )
 593        self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination, self.app.job_config )
 594        self.params = None
 595        if job.params:
 596            self.params = from_json_string( job.params )
 597
 598        self.__user_system_pwent = None
 599        self.__galaxy_system_pwent = None
 600
 601    def _job_dataset_path_rewriter( self, working_directory ):
 602        if self.app.config.outputs_to_working_directory:
 603            dataset_path_rewriter = OutputsToWorkingDirectoryPathRewriter( working_directory )
 604        else:
 605            dataset_path_rewriter = NullDatasetPathRewriter( )
 606        return dataset_path_rewriter
 607
 608    def can_split( self ):
 609        # Should the job handler split this job up?
 610        return self.app.config.use_tasked_jobs and self.tool.parallelism
 611
 612    def get_job_runner_url( self ):
 613        log.warning('(%s) Job runner URLs are deprecated, use destinations instead.' % self.job_id)
 614        return self.job_destination.url
 615
 616    def get_parallelism(self):
 617        return self.tool.parallelism
 618
 619    # legacy naming
 620    get_job_runner = get_job_runner_url
 621
 622    @property
 623    def job_destination(self):
 624        """Return the JobDestination that this job will use to run.  This will
 625        either be a configured destination, a randomly selected destination if
 626        the configured destination was a tag, or a dynamically generated
 627        destination from the dynamic runner.
 628
 629        Calling this method for the first time causes the dynamic runner to do
 630        its calculation, if any.
 631
 632        :returns: ``JobDestination``
 633        """
 634        return self.job_runner_mapper.get_job_destination(self.params)
 635
 636    def get_job( self ):
 637        return self.sa_session.query( model.Job ).get( self.job_id )
 638
 639    def get_id_tag(self):
 640        # For compatability with drmaa, which uses job_id right now, and TaskWrapper
 641        return self.get_job().get_id_tag()
 642
 643    def get_param_dict( self ):
 644        """
 645        Restore the dictionary of parameters from the database.
 646        """
 647        job = self.get_job()
 648        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
 649        param_dict = self.tool.params_from_strings( param_dict, self.app )
 650        return param_dict
 651
 652    def get_version_string_path( self ):
 653        return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id))
 654
 655    def prepare( self, compute_environment=None ):
 656        """
 657        Prepare the job to run by creating the working directory and the
 658        config files.
 659        """
 660        self.sa_session.expunge_all()  # this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
 661
 662        if not os.path.exists( self.working_directory ):
 663            os.mkdir( self.working_directory )
 664
 665        job = self._load_job()
 666
 667        def get_special( ):
 668            special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
 669            if not special:
 670                special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
 671            return special
 672
 673        tool_evaluator = self._get_tool_evaluator( job )
 674        compute_environment = compute_environment or self.default_compute_environment( job )
 675        tool_evaluator.set_compute_environment( compute_environment, get_special=get_special )
 676
 677        self.sa_session.flush()
 678
 679        self.command_line, self.extra_filenames = tool_evaluator.build()
 680        # FIXME: for now, tools get Galaxy's lib dir in their path
 681        if self.command_line and self.command_line.startswith( 'python' ):
 682            self.galaxy_lib_dir = os.path.abspath( "lib" )  # cwd = galaxy root
 683        # Shell fragment to inject dependencies
 684        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
 685        # We need command_line persisted to the db in order for Galaxy to re-queue the job
 686        # if the server was stopped and restarted before the job finished
 687        job.command_line = self.command_line
 688        self.sa_session.add( job )
 689        self.sa_session.flush()
 690        # Return list of all extra files
 691        self.param_dict = tool_evaluator.param_dict
 692        version_string_cmd = self.tool.version_string_cmd
 693        if version_string_cmd:
 694            self.write_version_cmd = "%s > %s 2>&1" % ( version_string_cmd, compute_environment.version_path() )
 695        else:
 696            self.write_version_cmd = None
 697        return self.extra_filenames
 698
 699    def default_compute_environment( self, job=None ):
 700        if not job:
 701            job = self.get_job()
 702        return SharedComputeEnvironment( self, job )
 703
 704    def _load_job( self ):
 705        # Load job from database and verify it has user or session.
 706        # Restore parameters from the database
 707        job = self.get_job()
 708        if job.user is None and job.galaxy_session is None:
 709            raise Exception( 'Job %s has no user and no session.' % job.id )
 710        return job
 711
 712    def _get_tool_evaluator( self, job ):
 713        # Hacky way to avoid cirular import for now.
 714        # Placing ToolEvaluator in either jobs or tools
 715        # result in ciruclar dependency.
 716        from galaxy.tools.evaluation import ToolEvaluator
 717
 718        tool_evaluator = ToolEvaluator(
 719            app=self.app,
 720            job=job,
 721            tool=self.tool,
 722            local_working_directory=self.working_directory,
 723        )
 724        return tool_evaluator
 725
 726    def fail( self, message, exception=False, stdout="", stderr="", exit_code=None ):
 727        """
 728        Indicate job failure by setting state and message on all output
 729        datasets.
 730        """
 731        job = self.get_job()
 732        self.sa_session.refresh( job )
 733        # if the job was deleted, don't fail it
 734        if not job.state == job.states.DELETED:
 735            # Check if the failure is due to an exception
 736            if exception:
 737                # Save the traceback immediately in case we generate another
 738                # below
 739                job.traceback = traceback.format_exc()
 740                # Get the exception and let the tool attempt to generate
 741                # a better message
 742                etype, evalue, tb = sys.exc_info()
 743                m = self.tool.handle_job_failure_exception( evalue )
 744                if m:
 745                    message = m
 746            if self.app.config.outputs_to_working_directory:
 747                for dataset_path in self.get_output_fnames():
 748                    try:
 749                        shutil.move( dataset_path.false_path, dataset_path.real_path )
 750                        log.debug( "fail(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
 751                    except ( IOError, OSError ), e:
 752                        log.error( "fail(): Missing output file in working directory: %s" % e )
 753            for dataset_assoc in job.output_datasets + job.output_library_datasets:
 754                dataset = dataset_assoc.dataset
 755                self.sa_session.refresh( dataset )
 756                dataset.state = dataset.states.ERROR
 757                dataset.blurb = 'tool error'
 758                dataset.info = message
 759                dataset.set_size()
 760                dataset.dataset.set_total_size()
 761                dataset.mark_unhidden()
 762                if dataset.ext == 'auto':
 763                    dataset.extension = 'data'
 764                # Update (non-library) job output datasets through the object store
 765                if dataset not in job.output_library_datasets:
 766                    self.app.object_store.update_from_file(dataset.dataset, create=True)
 767                # Pause any dependent jobs (and those jobs' outputs)
 768                for dep_job_assoc in dataset.dependent_jobs:
 769                    self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." )
 770                self.sa_session.add( dataset )
 771                self.sa_session.flush()
 772            job.state = job.states.ERROR
 773            job.command_line = self.command_line
 774            job.info = message
 775            # TODO: Put setting the stdout, stderr, and exit code in one place
 776            # (not duplicated with the finish method).
 777            if ( len( stdout ) > DATABASE_MAX_STRING_SIZE ):
 778                stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
 779                log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
 780            job.stdout = stdout
 781            if ( len( stderr ) > DATABASE_MAX_STRING_SIZE ):
 782                stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
 783                log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
 784            job.stderr = stderr
 785            # Let the exit code be Null if one is not provided:
 786            if ( exit_code != None ):
 787                job.exit_code = exit_code
 788
 789            self.sa_session.add( job )
 790            self.sa_session.flush()
 791        #Perform email action even on failure.
 792        for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]:
 793            ActionBox.execute(self.app, self.sa_session, pja, job)
 794        # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up
 795        if self.tool:
 796            self.tool.job_failed( self, message, exception )
 797        delete_files = self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED)
 798        self.cleanup( delete_files=delete_files )
 799
 800    def pause( self, job=None, message=None ):
 801        if job is None:
 802            job = self.get_job()
 803        if message is None:
 804            message = "Execution of this dataset's job is paused"
 805        if job.state == job.states.NEW:
 806            for dataset_assoc in job.output_datasets + job.output_library_datasets:
 807                dataset_assoc.dataset.dataset.state = dataset_assoc.dataset.dataset.states.PAUSED
 808                dataset_assoc.dataset.info = message
 809                self.sa_session.add( dataset_assoc.dataset )
 810            job.state = job.states.PAUSED
 811            self.sa_session.add( job )
 812
 813    def change_state( self, state, info=False ):
 814        job = self.get_job()
 815        self.sa_session.refresh( job )
 816        for dataset_assoc in job.output_datasets + job.output_library_datasets:
 817            dataset = dataset_assoc.dataset
 818            self.sa_session.refresh( dataset )
 819            dataset.state = state
 820            if info:
 821                dataset.info = info
 822            self.sa_session.add( dataset )
 823            self.sa_session.flush()
 824        if info:
 825            job.info = info
 826        job.state = state
 827        self.sa_session.add( job )
 828        self.sa_session.flush()
 829
 830    def get_state( self ):
 831        job = self.get_job()
 832        self.sa_session.refresh( job )
 833        return job.state
 834
 835    def set_runner( self, runner_url, external_id ):
 836        log.warning('set_runner() is deprecated, use set_job_destination()')
 837        self.set_job_destination(self.job_destination, external_id)
 838
 839    def set_job_destination( self, job_destination, external_id=None ):
 840        """
 841        Persist job destination params in the database for recovery.
 842
 843        self.job_destination is not used because a runner may choose to rewrite
 844        parts of the destination (e.g. the params).
 845        """
 846        job = self.get_job()
 847        self.sa_session.refresh(job)
 848        log.debug('(%s) Persisting job destination (destination id: %s)' % (job.id, job_destination.id))
 849        job.destination_id = job_destination.id
 850        job.destination_params = job_destination.params
 851        job.job_runner_name = job_destination.runner
 852        job.job_runner_external_id = external_id
 853        self.sa_session.add(job)
 854        self.sa_session.flush()
 855
 856    def finish( self, stdout, stderr, tool_exit_code=None ):
 857        """
 858        Called to indicate that the associated command has been run. Updates
 859        the output datasets based on stderr and stdout from the command, and
 860        the contents of the output files.
 861        """
 862        stdout = unicodify( stdout )
 863        stderr = unicodify( stderr )
 864
 865        # default post job setup
 866        self.sa_session.expunge_all()
 867        job = self.get_job()
 868
 869        # TODO: After failing here, consider returning from the function.
 870        try:
 871            self.reclaim_ownership()
 872        except:
 873            log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) )
 874            return self.fail( job.info, stdout=stdout, stderr=stderr, exit_code=tool_exit_code )
 875
 876        # if the job was deleted, don't finish it
 877        if job.state == job.states.DELETED or job.state == job.states.ERROR:
 878            # SM: Note that, at this point, the exit code must be saved in case
 879            # there was an error. Errors caught here could mean that the job
 880            # was deleted by an administrator (based on old comments), but it
 881            # could also mean that a job was broken up into tasks and one of
 882            # the tasks failed. So include the stderr, stdout, and exit code:
 883            return self.fail( job.info, stderr=stderr, stdout=stdout, exit_code=tool_exit_code )
 884
 885        # Check the tool's stdout, stderr, and exit code for errors, but only
 886        # if the job has not already been marked as having an error.
 887        # The job's stdout and stderr will be set accordingly.
 888
 889        # We set final_job_state to use for dataset management, but *don't* set
 890        # job.state until after dataset collection to prevent history issues
 891        if ( self.check_tool_output( stdout, stderr, tool_exit_code, job ) ):
 892            final_job_state = job.states.OK
 893        else:
 894            final_job_state = job.states.ERROR
 895
 896        if self.write_version_cmd:
 897            version_filename = self.get_version_string_path()
 898            if os.path.exists(version_filename):
 899                self.version_string = open(version_filename).read()
 900                os.unlink(version_filename)
 901
 902        if self.app.config.outputs_to_working_directory and not self.__link_file_check():
 903            for dataset_path in self.get_output_fnames():
 904                try:
 905                    shutil.move( dataset_path.false_path, dataset_path.real_path )
 906                    log.debug( "finish(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
 907                except ( IOError, OSError ):
 908                    # this can happen if Galaxy is restarted during the job's
 909                    # finish method - the false_path file has already moved,
 910                    # and when the job is recovered, it won't be found.
 911                    if os.path.exists( dataset_path.real_path ) and os.stat( dataset_path.real_path ).st_size > 0:
 912                        log.warning( "finish(): %s not found, but %s is not empty, so it will be used instead" % ( dataset_path.false_path, dataset_path.real_path ) )
 913                    else:
 914                        # Prior to fail we need to set job.state
 915                        job.state = final_job_state
 916                        return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
 917
 918        job_context = ExpressionContext( dict( stdout=job.stdout, stderr=job.stderr ) )
 919        for dataset_assoc in job.output_datasets + job.output_library_datasets:
 920            context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
 921            #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
 922            for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations:  # need to update all associated output hdas, i.e. history was shared with job running
 923                trynum = 0
 924                while trynum < self.app.config.retry_job_output_collection:
 925                    try:
 926                        # Attempt to short circuit NFS attribute caching
 927                        os.stat( dataset.dataset.file_name )
 928                        os.chown( dataset.dataset.file_name, os.getuid(), -1 )
 929                        trynum = self.app.config.retry_job_output_collection
 930                    except ( OSError, ObjectNotFound ), e:
 931                        trynum += 1
 932                        log.warning( 'Error accessing %s, will retry: %s', dataset.dataset.file_name, e )
 933                        time.sleep( 2 )
 934                dataset.blurb = 'done'
 935                dataset.peek = 'no peek'
 936                dataset.info = (dataset.info or '')
 937                if context['stdout'].strip():
 938                    #Ensure white space between entries
 939                    dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip()
 940                if context['stderr'].strip():
 941                    #Ensure white space between entries
 942                    dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip()
 943                dataset.tool_version = self.version_string
 944                dataset.set_size()
 945                if 'uuid' in context:
 946                    dataset.dataset.uuid = context['uuid']
 947                # Update (non-library) job output datasets through the object store
 948                if dataset not in job.output_library_datasets:
 949                    self.app.object_store.update_from_file(dataset.dataset, create=True)
 950                if job.states.ERROR == final_job_state:
 951                    dataset.blurb = "error"
 952                    dataset.mark_unhidden()
 953                elif dataset.has_data():
 954                    # If the tool was expected to set the extension, attempt to retrieve it
 955                    if dataset.ext == 'auto':
 956                        dataset.extension = context.get( 'ext', 'data' )
 957                        dataset.init_meta( copy_from=dataset )
 958                    #if a dataset was copied, it won't appear in our dictionary:
 959                    #either use the metadata from originating output dataset, or call set_meta on the copies
 960                    #it would be quicker to just copy the metadata from the originating output dataset,
 961                    #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
 962                    if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ):
 963                        dataset.datatype.set_meta( dataset, overwrite=False )  # call datatype.set_meta directly for the initial set_meta call during dataset creation
 964                    elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state:
 965                        dataset._state = model.Dataset.states.FAILED_METADATA
 966                    else:
 967                        #load metadata from file
 968                        #we need to no longer allow metadata to be edited while the job is still running,
 969                        #since if it is edited, the metadata changed on the running output will no longer match
 970                        #the metadata that was stored to disk for use via the external process,
 971                        #and the changes made by the user will be lost, without warning or notice
 972                        dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset, self.sa_session ).filename_out )
 973                    try:
 974                        assert context.get( 'line_count', None ) is not None
 975                        if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
 976                            dataset.set_peek( line_count=context['line_count'], is_multi_byte=True )
 977                        else:
 978                            dataset.set_peek( line_count=context['line_count'] )
 979                    except:
 980                        if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
 981                            dataset.set_peek( is_multi_byte=True )
 982                        else:
 983                            dataset.set_peek()
 984                    try:
 985                        # set the name if provided by the tool
 986                        dataset.name = context['name']
 987                    except:
 988                        pass
 989                else:
 990                    dataset.blurb = "empty"
 991                    if dataset.ext == 'auto':
 992                        dataset.extension = 'txt'
 993                self.sa_session.add( dataset )
 994            if job.states.ERROR == final_job_state:
 995                log.debug( "setting dataset state to ERROR" )
 996                # TODO: This is where the state is being set to error. Change it!
 997                dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR
 998                # Pause any dependent jobs (and those jobs' outputs)
 999                for dep_job_assoc in dataset_assoc.dataset.dependent_jobs:
1000                    self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." )
1001            else:
1002                dataset_assoc.dataset.dataset.state = model.Dataset.states.OK
1003            # If any of the rest of the finish method below raises an
1004            # exception, the fail method will run and set the datasets to
1005            # ERROR.  The user will never see that the datasets are in error if
1006            # they were flushed as OK here, since upon doing so, the history
1007            # panel stops checking for updates.  So allow the
1008            # self.sa_session.flush() at the bottom of this method set
1009            # the state instead.
1010
1011        for pja in job.post_job_actions:
1012            ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job)
1013        # Flush all the dataset and job changes above.  Dataset state changes
1014        # will now be seen by the user.
1015        self.sa_session.flush()
1016        # Save stdout and stderr
1017        if len( job.stdout ) > DATABASE_MAX_STRING_SIZE:
1018            log.info( "stdout for job …

Large files files are truncated, but you can click here to view the full file