PageRenderTime 86ms CodeModel.GetById 13ms app.highlight 59ms RepoModel.GetById 0ms app.codeStats 1ms

/lib/galaxy/jobs/__init__.py

https://bitbucket.org/cistrome/cistrome-harvard/
Python | 1663 lines | 1452 code | 55 blank | 156 comment | 116 complexity | 67fd55bd5e5369ca562536fa8927b8c2 MD5 | raw file
   1"""
   2Support for running a tool in Galaxy via an internal job management system
   3"""
   4from abc import ABCMeta
   5from abc import abstractmethod
   6
   7import time
   8import copy
   9import datetime
  10import galaxy
  11import logging
  12import os
  13import pwd
  14import random
  15import re
  16import shutil
  17import subprocess
  18import sys
  19import traceback
  20from galaxy import model, util
  21from galaxy.datatypes import metadata
  22from galaxy.exceptions import ObjectInvalid, ObjectNotFound
  23from galaxy.jobs.actions.post import ActionBox
  24from galaxy.jobs.mapper import JobRunnerMapper
  25from galaxy.jobs.runners import BaseJobRunner
  26from galaxy.util.bunch import Bunch
  27from galaxy.util.expressions import ExpressionContext
  28from galaxy.util.json import from_json_string
  29from galaxy.util import unicodify
  30
  31from .output_checker import check_output
  32from .datasets import TaskPathRewriter
  33from .datasets import OutputsToWorkingDirectoryPathRewriter
  34from .datasets import NullDatasetPathRewriter
  35from .datasets import DatasetPath
  36
  37log = logging.getLogger( __name__ )
  38
  39DATABASE_MAX_STRING_SIZE = util.DATABASE_MAX_STRING_SIZE
  40DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY
  41
  42# This file, if created in the job's working directory, will be used for
  43# setting advanced metadata properties on the job and its associated outputs.
  44# This interface is currently experimental, is only used by the upload tool,
  45# and should eventually become API'd
  46TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json'
  47
  48
  49class JobDestination( Bunch ):
  50    """
  51    Provides details about where a job runs
  52    """
  53    def __init__(self, **kwds):
  54        self['id'] = None
  55        self['url'] = None
  56        self['tags'] = None
  57        self['runner'] = None
  58        self['legacy'] = False
  59        self['converted'] = False
  60        # dict is appropriate (rather than a bunch) since keys may not be valid as attributes
  61        self['params'] = dict()
  62        super(JobDestination, self).__init__(**kwds)
  63
  64        # Store tags as a list
  65        if self.tags is not None:
  66            self['tags'] = [ x.strip() for x in self.tags.split(',') ]
  67
  68
  69class JobToolConfiguration( Bunch ):
  70    """
  71    Provides details on what handler and destination a tool should use
  72
  73    A JobToolConfiguration will have the required attribute 'id' and optional
  74    attributes 'handler', 'destination', and 'params'
  75    """
  76    def __init__(self, **kwds):
  77        self['handler'] = None
  78        self['destination'] = None
  79        self['params'] = dict()
  80        super(JobToolConfiguration, self).__init__(**kwds)
  81
  82
  83class JobConfiguration( object ):
  84    """A parser and interface to advanced job management features.
  85
  86    These features are configured in the job configuration, by default, ``job_conf.xml``
  87    """
  88    DEFAULT_NWORKERS = 4
  89
  90    def __init__(self, app):
  91        """Parse the job configuration XML.
  92        """
  93        self.app = app
  94        self.runner_plugins = []
  95        self.handlers = {}
  96        self.handler_runner_plugins = {}
  97        self.default_handler_id = None
  98        self.destinations = {}
  99        self.destination_tags = {}
 100        self.default_destination_id = None
 101        self.tools = {}
 102        self.limits = Bunch()
 103
 104        # Initialize the config
 105        try:
 106            tree = util.parse_xml(self.app.config.job_config_file)
 107            self.__parse_job_conf_xml(tree)
 108        except IOError:
 109            log.warning( 'Job configuration "%s" does not exist, using legacy job configuration from Galaxy config file "%s" instead' % ( self.app.config.job_config_file, self.app.config.config_file ) )
 110            self.__parse_job_conf_legacy()
 111
 112    def __parse_job_conf_xml(self, tree):
 113        """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml).
 114
 115        :param tree: Object representing the root ``<job_conf>`` object in the job config file.
 116        :type tree: ``xml.etree.ElementTree.Element``
 117        """
 118        root = tree.getroot()
 119        log.debug('Loading job configuration from %s' % self.app.config.job_config_file)
 120
 121        # Parse job plugins
 122        plugins = root.find('plugins')
 123        if plugins is not None:
 124            for plugin in self.__findall_with_required(plugins, 'plugin', ('id', 'type', 'load')):
 125                if plugin.get('type') == 'runner':
 126                    workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS))
 127                    runner_kwds = self.__get_params(plugin)
 128                    runner_info = dict(id=plugin.get('id'),
 129                                       load=plugin.get('load'),
 130                                       workers=int(workers),
 131                                       kwds=runner_kwds)
 132                    self.runner_plugins.append(runner_info)
 133                else:
 134                    log.error('Unknown plugin type: %s' % plugin.get('type'))
 135        # Load tasks if configured
 136        if self.app.config.use_tasked_jobs:
 137            self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
 138
 139        # Parse handlers
 140        handlers = root.find('handlers')
 141        if handlers is not None:
 142            for handler in self.__findall_with_required(handlers, 'handler'):
 143                id = handler.get('id')
 144                if id in self.handlers:
 145                    log.error("Handler '%s' overlaps handler with the same name, ignoring" % id)
 146                else:
 147                    log.debug("Read definition for handler '%s'" % id)
 148                    self.handlers[id] = (id,)
 149                    for plugin in handler.findall('plugin'):
 150                        if id not in self.handler_runner_plugins:
 151                            self.handler_runner_plugins[id] = []
 152                        self.handler_runner_plugins[id].append( plugin.get('id') )
 153                    if handler.get('tags', None) is not None:
 154                        for tag in [ x.strip() for x in handler.get('tags').split(',') ]:
 155                            if tag in self.handlers:
 156                                self.handlers[tag].append(id)
 157                            else:
 158                                self.handlers[tag] = [id]
 159
 160        # Determine the default handler(s)
 161        self.default_handler_id = self.__get_default(handlers, self.handlers.keys())
 162
 163        # Parse destinations
 164        destinations = root.find('destinations')
 165        for destination in self.__findall_with_required(destinations, 'destination', ('id', 'runner')):
 166            id = destination.get('id')
 167            job_destination = JobDestination(**dict(destination.items()))
 168            job_destination['params'] = self.__get_params(destination)
 169            self.destinations[id] = (job_destination,)
 170            if job_destination.tags is not None:
 171                for tag in job_destination.tags:
 172                    if tag not in self.destinations:
 173                        self.destinations[tag] = []
 174                    self.destinations[tag].append(job_destination)
 175
 176        # Determine the default destination
 177        self.default_destination_id = self.__get_default(destinations, self.destinations.keys())
 178
 179        # Parse tool mappings
 180        tools = root.find('tools')
 181        if tools is not None:
 182            for tool in self.__findall_with_required(tools, 'tool'):
 183                # There can be multiple definitions with identical ids, but different params
 184                id = tool.get('id').lower().rstrip('/')
 185                if id not in self.tools:
 186                    self.tools[id] = list()
 187                self.tools[id].append(JobToolConfiguration(**dict(tool.items())))
 188                self.tools[id][-1]['params'] = self.__get_params(tool)
 189
 190        types = dict(registered_user_concurrent_jobs=int,
 191                     anonymous_user_concurrent_jobs=int,
 192                     walltime=str,
 193                     output_size=int)
 194
 195        self.limits = Bunch(registered_user_concurrent_jobs=None,
 196                            anonymous_user_concurrent_jobs=None,
 197                            walltime=None,
 198                            walltime_delta=None,
 199                            output_size=None,
 200                            concurrent_jobs={})
 201
 202        # Parse job limits
 203        limits = root.find('limits')
 204        if limits is not None:
 205            for limit in self.__findall_with_required(limits, 'limit', ('type',)):
 206                type = limit.get('type')
 207                if type == 'concurrent_jobs':
 208                    id = limit.get('tag', None) or limit.get('id')
 209                    self.limits.concurrent_jobs[id] = int(limit.text)
 210                elif limit.text:
 211                    self.limits.__dict__[type] = types.get(type, str)(limit.text)
 212
 213        if self.limits.walltime is not None:
 214            h, m, s = [ int( v ) for v in self.limits.walltime.split( ':' ) ]
 215            self.limits.walltime_delta = datetime.timedelta( 0, s, 0, 0, m, h )
 216
 217        log.debug('Done loading job configuration')
 218
 219    def __parse_job_conf_legacy(self):
 220        """Loads the old-style job configuration from options in the galaxy config file (by default, universe_wsgi.ini).
 221        """
 222        log.debug('Loading job configuration from %s' % self.app.config.config_file)
 223
 224        # Always load local and lwr
 225        self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers), dict(id='lwr', load='lwr', workers=self.app.config.cluster_job_queue_workers)]
 226        # Load tasks if configured
 227        if self.app.config.use_tasked_jobs:
 228            self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers))
 229        for runner in self.app.config.start_job_runners:
 230            self.runner_plugins.append(dict(id=runner, load=runner, workers=self.app.config.cluster_job_queue_workers))
 231
 232        # Set the handlers
 233        for id in self.app.config.job_handlers:
 234            self.handlers[id] = (id,)
 235
 236        self.handlers['default_job_handlers'] = self.app.config.default_job_handlers
 237        self.default_handler_id = 'default_job_handlers'
 238
 239        # Set tool handler configs
 240        for id, tool_handlers in self.app.config.tool_handlers.items():
 241            self.tools[id] = list()
 242            for handler_config in tool_handlers:
 243                # rename the 'name' key to 'handler'
 244                handler_config['handler'] = handler_config.pop('name')
 245                self.tools[id].append(JobToolConfiguration(**handler_config))
 246
 247        # Set tool runner configs
 248        for id, tool_runners in self.app.config.tool_runners.items():
 249            # Might have been created in the handler parsing above
 250            if id not in self.tools:
 251                self.tools[id] = list()
 252            for runner_config in tool_runners:
 253                url = runner_config['url']
 254                if url not in self.destinations:
 255                    # Create a new "legacy" JobDestination - it will have its URL converted to a destination params once the appropriate plugin has loaded
 256                    self.destinations[url] = (JobDestination(id=url, runner=url.split(':', 1)[0], url=url, legacy=True, converted=False),)
 257                for tool_conf in self.tools[id]:
 258                    if tool_conf.params == runner_config.get('params', {}):
 259                        tool_conf['destination'] = url
 260                        break
 261                else:
 262                    # There was not an existing config (from the handlers section) with the same params
 263                    # rename the 'url' key to 'destination'
 264                    runner_config['destination'] = runner_config.pop('url')
 265                    self.tools[id].append(JobToolConfiguration(**runner_config))
 266
 267        self.destinations[self.app.config.default_cluster_job_runner] = (JobDestination(id=self.app.config.default_cluster_job_runner, runner=self.app.config.default_cluster_job_runner.split(':', 1)[0], url=self.app.config.default_cluster_job_runner, legacy=True, converted=False),)
 268        self.default_destination_id = self.app.config.default_cluster_job_runner
 269
 270        # Set the job limits
 271        self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit,
 272                            anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit,
 273                            walltime=self.app.config.job_walltime,
 274                            walltime_delta=self.app.config.job_walltime_delta,
 275                            output_size=self.app.config.output_size_limit,
 276                            concurrent_jobs={})
 277
 278        log.debug('Done loading job configuration')
 279
 280    def __get_default(self, parent, names):
 281        """Returns the default attribute set in a parent tag like <handlers> or <destinations>, or return the ID of the child, if there is no explicit default and only one child.
 282
 283        :param parent: Object representing a tag that may or may not have a 'default' attribute.
 284        :type parent: ``xml.etree.ElementTree.Element``
 285        :param names: The list of destination or handler IDs or tags that were loaded.
 286        :type names: list of str
 287
 288        :returns: str -- id or tag representing the default.
 289        """
 290        rval = parent.get('default')
 291        if rval is not None:
 292            # If the parent element has a 'default' attribute, use the id or tag in that attribute
 293            if rval not in names:
 294                raise Exception("<%s> default attribute '%s' does not match a defined id or tag in a child element" % (parent.tag, rval))
 295            log.debug("<%s> default set to child with id or tag '%s'" % (parent.tag, rval))
 296        elif len(names) == 1:
 297            log.info("Setting <%s> default to child with id '%s'" % (parent.tag, names[0]))
 298            rval = names[0]
 299        else:
 300            raise Exception("No <%s> default specified, please specify a valid id or tag with the 'default' attribute" % parent.tag)
 301        return rval
 302
 303    def __findall_with_required(self, parent, match, attribs=None):
 304        """Like ``xml.etree.ElementTree.Element.findall()``, except only returns children that have the specified attribs.
 305
 306        :param parent: Parent element in which to find.
 307        :type parent: ``xml.etree.ElementTree.Element``
 308        :param match: Name of child elements to find.
 309        :type match: str
 310        :param attribs: List of required attributes in children elements.
 311        :type attribs: list of str
 312
 313        :returns: list of ``xml.etree.ElementTree.Element``
 314        """
 315        rval = []
 316        if attribs is None:
 317            attribs = ('id',)
 318        for elem in parent.findall(match):
 319            for attrib in attribs:
 320                if attrib not in elem.attrib:
 321                    log.warning("required '%s' attribute is missing from <%s> element" % (attrib, match))
 322                    break
 323            else:
 324                rval.append(elem)
 325        return rval
 326
 327    def __get_params(self, parent):
 328        """Parses any child <param> tags in to a dictionary suitable for persistence.
 329
 330        :param parent: Parent element in which to find child <param> tags.
 331        :type parent: ``xml.etree.ElementTree.Element``
 332
 333        :returns: dict
 334        """
 335        rval = {}
 336        for param in parent.findall('param'):
 337            rval[param.get('id')] = param.text
 338        return rval
 339
 340    @property
 341    def default_job_tool_configuration(self):
 342        """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration.  It consists of a reference to the default handler and default destination.
 343
 344        :returns: JobToolConfiguration -- a representation of a <tool> element that uses the default handler and destination
 345        """
 346        return JobToolConfiguration(id='default', handler=self.default_handler_id, destination=self.default_destination_id)
 347
 348    # Called upon instantiation of a Tool object
 349    def get_job_tool_configurations(self, ids):
 350        """Get all configured JobToolConfigurations for a tool ID, or, if given a list of IDs, the JobToolConfigurations for the first id in ``ids`` matching a tool definition.
 351
 352        .. note::
 353
 354            You should not mix tool shed tool IDs, versionless tool shed IDs, and tool config tool IDs that refer to the same tool.
 355
 356        :param ids: Tool ID or IDs to fetch the JobToolConfiguration of.
 357        :type ids: list or str.
 358        :returns: list -- JobToolConfiguration Bunches representing <tool> elements matching the specified ID(s).
 359
 360        Example tool ID strings include:
 361
 362        * Full tool shed id: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool/1.0.0``
 363        * Tool shed id less version: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool``
 364        * Tool config tool id: ``filter_tool``
 365        """
 366        rval = []
 367        # listify if ids is a single (string) id
 368        ids = util.listify(ids)
 369        for id in ids:
 370            if id in self.tools:
 371                # If a tool has definitions that include job params but not a
 372                # definition for jobs without params, include the default
 373                # config
 374                for job_tool_configuration in self.tools[id]:
 375                    if not job_tool_configuration.params:
 376                        break
 377                else:
 378                    rval.append(self.default_job_tool_configuration)
 379                rval.extend(self.tools[id])
 380                break
 381        else:
 382            rval.append(self.default_job_tool_configuration)
 383        return rval
 384
 385    def __get_single_item(self, collection):
 386        """Given a collection of handlers or destinations, return one item from the collection at random.
 387        """
 388        # Done like this to avoid random under the assumption it's faster to avoid it
 389        if len(collection) == 1:
 390            return collection[0]
 391        else:
 392            return random.choice(collection)
 393
 394    # This is called by Tool.get_job_handler()
 395    def get_handler(self, id_or_tag):
 396        """Given a handler ID or tag, return the provided ID or an ID matching the provided tag
 397
 398        :param id_or_tag: A handler ID or tag.
 399        :type id_or_tag: str
 400
 401        :returns: str -- A valid job handler ID.
 402        """
 403        if id_or_tag is None:
 404            id_or_tag = self.default_handler_id
 405        return self.__get_single_item(self.handlers[id_or_tag])
 406
 407    def get_destination(self, id_or_tag):
 408        """Given a destination ID or tag, return the JobDestination matching the provided ID or tag
 409
 410        :param id_or_tag: A destination ID or tag.
 411        :type id_or_tag: str
 412
 413        :returns: JobDestination -- A valid destination
 414
 415        Destinations are deepcopied as they are expected to be passed in to job
 416        runners, which will modify them for persisting params set at runtime.
 417        """
 418        if id_or_tag is None:
 419            id_or_tag = self.default_destination_id
 420        return copy.deepcopy(self.__get_single_item(self.destinations[id_or_tag]))
 421
 422    def get_destinations(self, id_or_tag):
 423        """Given a destination ID or tag, return all JobDestinations matching the provided ID or tag
 424
 425        :param id_or_tag: A destination ID or tag.
 426        :type id_or_tag: str
 427
 428        :returns: list or tuple of JobDestinations
 429
 430        Destinations are not deepcopied, so they should not be passed to
 431        anything which might modify them.
 432        """
 433        return self.destinations.get(id_or_tag, None)
 434
 435    def get_job_runner_plugins(self, handler_id):
 436        """Load all configured job runner plugins
 437
 438        :returns: list of job runner plugins
 439        """
 440        rval = {}
 441        if handler_id in self.handler_runner_plugins:
 442            plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ]
 443            log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) )
 444        else:
 445            plugins_to_load = self.runner_plugins
 446            log.info( "Handler '%s' will load all configured runner plugins", handler_id )
 447        for runner in plugins_to_load:
 448            class_names = []
 449            module = None
 450            id = runner['id']
 451            load = runner['load']
 452            if ':' in load:
 453                # Name to load was specified as '<module>:<class>'
 454                module_name, class_name = load.rsplit(':', 1)
 455                class_names = [ class_name ]
 456                module = __import__( module_name )
 457            else:
 458                # Name to load was specified as '<module>'
 459                if '.' not in load:
 460                    # For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name
 461                    module_name = 'galaxy.jobs.runners.' + load
 462                    try:
 463                        module = __import__( module_name )
 464                    except ImportError:
 465                        # No such module, we'll retry without prepending galaxy.jobs.runners.
 466                        # All other exceptions (e.g. something wrong with the module code) will raise
 467                        pass
 468                if module is None:
 469                    # If the name included a '.' or loading from the static runners path failed, try the original name
 470                    module = __import__( load )
 471                    module_name = load
 472            if module is None:
 473                # Module couldn't be loaded, error should have already been displayed
 474                continue
 475            for comp in module_name.split( "." )[1:]:
 476                module = getattr( module, comp )
 477            if not class_names:
 478                # If there's not a ':', we check <module>.__all__ for class names
 479                try:
 480                    assert module.__all__
 481                    class_names = module.__all__
 482                except AssertionError:
 483                    log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % load )
 484                    continue
 485            for class_name in class_names:
 486                runner_class = getattr( module, class_name )
 487                try:
 488                    assert issubclass(runner_class, BaseJobRunner)
 489                except TypeError:
 490                    log.warning("A non-class name was found in __all__, ignoring: %s" % id)
 491                    continue
 492                except AssertionError:
 493                    log.warning("Job runner classes must be subclassed from BaseJobRunner, %s has bases: %s" % (id, runner_class.__bases__))
 494                    continue
 495                try:
 496                    rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) )
 497                except TypeError:
 498                    log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) )
 499                    rval[id] = runner_class( self.app )
 500                log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) )
 501        return rval
 502
 503    def is_id(self, collection):
 504        """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID
 505
 506        :param collection: A representation of a destination or handler
 507        :type collection: tuple or list
 508
 509        :returns: bool
 510        """
 511        return type(collection) == tuple
 512
 513    def is_tag(self, collection):
 514        """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID
 515
 516        :param collection: A representation of a destination or handler
 517        :type collection: tuple or list
 518
 519        :returns: bool
 520        """
 521        return type(collection) == list
 522
 523    def is_handler(self, server_name):
 524        """Given a server name, indicate whether the server is a job handler
 525
 526        :param server_name: The name to check
 527        :type server_name: str
 528
 529        :return: bool
 530        """
 531        for collection in self.handlers.values():
 532            if server_name in collection:
 533                return True
 534        return False
 535
 536    def convert_legacy_destinations(self, job_runners):
 537        """Converts legacy (from a URL) destinations to contain the appropriate runner params defined in the URL.
 538
 539        :param job_runners: All loaded job runner plugins.
 540        :type job_runners: list of job runner plugins
 541        """
 542        for id, destination in [ ( id, destinations[0] ) for id, destinations in self.destinations.items() if self.is_id(destinations) ]:
 543            # Only need to deal with real destinations, not members of tags
 544            if destination.legacy and not destination.converted:
 545                if destination.runner in job_runners:
 546                    destination.params = job_runners[destination.runner].url_to_destination(destination.url).params
 547                    destination.converted = True
 548                    if destination.params:
 549                        log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url))
 550                        for k, v in destination.params.items():
 551                            log.debug("    %s: %s" % (k, v))
 552                    else:
 553                        log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url))
 554                else:
 555                    log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner))
 556
 557
 558class JobWrapper( object ):
 559    """
 560    Wraps a 'model.Job' with convenience methods for running processes and
 561    state management.
 562    """
 563    def __init__( self, job, queue ):
 564        self.job_id = job.id
 565        self.session_id = job.session_id
 566        self.user_id = job.user_id
 567        self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None )
 568        self.queue = queue
 569        self.app = queue.app
 570        self.sa_session = self.app.model.context
 571        self.extra_filenames = []
 572        self.command_line = None
 573        # Tool versioning variables
 574        self.write_version_cmd = None
 575        self.version_string = ""
 576        self.galaxy_lib_dir = None
 577        # With job outputs in the working directory, we need the working
 578        # directory to be set before prepare is run, or else premature deletion
 579        # and job recovery fail.
 580        # Create the working dir if necessary
 581        try:
 582            self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
 583            self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id))
 584            log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory))
 585        except ObjectInvalid:
 586            raise Exception('Unable to create job working directory, job failure')
 587        self.dataset_path_rewriter = self._job_dataset_path_rewriter( self.working_directory )
 588        self.output_paths = None
 589        self.output_hdas_and_paths = None
 590        self.tool_provided_job_metadata = None
 591        # Wrapper holding the info required to restore and clean up from files used for setting metadata externally
 592        self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job )
 593        self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination, self.app.job_config )
 594        self.params = None
 595        if job.params:
 596            self.params = from_json_string( job.params )
 597
 598        self.__user_system_pwent = None
 599        self.__galaxy_system_pwent = None
 600
 601    def _job_dataset_path_rewriter( self, working_directory ):
 602        if self.app.config.outputs_to_working_directory:
 603            dataset_path_rewriter = OutputsToWorkingDirectoryPathRewriter( working_directory )
 604        else:
 605            dataset_path_rewriter = NullDatasetPathRewriter( )
 606        return dataset_path_rewriter
 607
 608    def can_split( self ):
 609        # Should the job handler split this job up?
 610        return self.app.config.use_tasked_jobs and self.tool.parallelism
 611
 612    def get_job_runner_url( self ):
 613        log.warning('(%s) Job runner URLs are deprecated, use destinations instead.' % self.job_id)
 614        return self.job_destination.url
 615
 616    def get_parallelism(self):
 617        return self.tool.parallelism
 618
 619    # legacy naming
 620    get_job_runner = get_job_runner_url
 621
 622    @property
 623    def job_destination(self):
 624        """Return the JobDestination that this job will use to run.  This will
 625        either be a configured destination, a randomly selected destination if
 626        the configured destination was a tag, or a dynamically generated
 627        destination from the dynamic runner.
 628
 629        Calling this method for the first time causes the dynamic runner to do
 630        its calculation, if any.
 631
 632        :returns: ``JobDestination``
 633        """
 634        return self.job_runner_mapper.get_job_destination(self.params)
 635
 636    def get_job( self ):
 637        return self.sa_session.query( model.Job ).get( self.job_id )
 638
 639    def get_id_tag(self):
 640        # For compatability with drmaa, which uses job_id right now, and TaskWrapper
 641        return self.get_job().get_id_tag()
 642
 643    def get_param_dict( self ):
 644        """
 645        Restore the dictionary of parameters from the database.
 646        """
 647        job = self.get_job()
 648        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
 649        param_dict = self.tool.params_from_strings( param_dict, self.app )
 650        return param_dict
 651
 652    def get_version_string_path( self ):
 653        return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id))
 654
 655    def prepare( self, compute_environment=None ):
 656        """
 657        Prepare the job to run by creating the working directory and the
 658        config files.
 659        """
 660        self.sa_session.expunge_all()  # this prevents the metadata reverting that has been seen in conjunction with the PBS job runner
 661
 662        if not os.path.exists( self.working_directory ):
 663            os.mkdir( self.working_directory )
 664
 665        job = self._load_job()
 666
 667        def get_special( ):
 668            special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
 669            if not special:
 670                special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
 671            return special
 672
 673        tool_evaluator = self._get_tool_evaluator( job )
 674        compute_environment = compute_environment or self.default_compute_environment( job )
 675        tool_evaluator.set_compute_environment( compute_environment, get_special=get_special )
 676
 677        self.sa_session.flush()
 678
 679        self.command_line, self.extra_filenames = tool_evaluator.build()
 680        # FIXME: for now, tools get Galaxy's lib dir in their path
 681        if self.command_line and self.command_line.startswith( 'python' ):
 682            self.galaxy_lib_dir = os.path.abspath( "lib" )  # cwd = galaxy root
 683        # Shell fragment to inject dependencies
 684        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
 685        # We need command_line persisted to the db in order for Galaxy to re-queue the job
 686        # if the server was stopped and restarted before the job finished
 687        job.command_line = self.command_line
 688        self.sa_session.add( job )
 689        self.sa_session.flush()
 690        # Return list of all extra files
 691        self.param_dict = tool_evaluator.param_dict
 692        version_string_cmd = self.tool.version_string_cmd
 693        if version_string_cmd:
 694            self.write_version_cmd = "%s > %s 2>&1" % ( version_string_cmd, compute_environment.version_path() )
 695        else:
 696            self.write_version_cmd = None
 697        return self.extra_filenames
 698
 699    def default_compute_environment( self, job=None ):
 700        if not job:
 701            job = self.get_job()
 702        return SharedComputeEnvironment( self, job )
 703
 704    def _load_job( self ):
 705        # Load job from database and verify it has user or session.
 706        # Restore parameters from the database
 707        job = self.get_job()
 708        if job.user is None and job.galaxy_session is None:
 709            raise Exception( 'Job %s has no user and no session.' % job.id )
 710        return job
 711
 712    def _get_tool_evaluator( self, job ):
 713        # Hacky way to avoid cirular import for now.
 714        # Placing ToolEvaluator in either jobs or tools
 715        # result in ciruclar dependency.
 716        from galaxy.tools.evaluation import ToolEvaluator
 717
 718        tool_evaluator = ToolEvaluator(
 719            app=self.app,
 720            job=job,
 721            tool=self.tool,
 722            local_working_directory=self.working_directory,
 723        )
 724        return tool_evaluator
 725
 726    def fail( self, message, exception=False, stdout="", stderr="", exit_code=None ):
 727        """
 728        Indicate job failure by setting state and message on all output
 729        datasets.
 730        """
 731        job = self.get_job()
 732        self.sa_session.refresh( job )
 733        # if the job was deleted, don't fail it
 734        if not job.state == job.states.DELETED:
 735            # Check if the failure is due to an exception
 736            if exception:
 737                # Save the traceback immediately in case we generate another
 738                # below
 739                job.traceback = traceback.format_exc()
 740                # Get the exception and let the tool attempt to generate
 741                # a better message
 742                etype, evalue, tb = sys.exc_info()
 743                m = self.tool.handle_job_failure_exception( evalue )
 744                if m:
 745                    message = m
 746            if self.app.config.outputs_to_working_directory:
 747                for dataset_path in self.get_output_fnames():
 748                    try:
 749                        shutil.move( dataset_path.false_path, dataset_path.real_path )
 750                        log.debug( "fail(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
 751                    except ( IOError, OSError ), e:
 752                        log.error( "fail(): Missing output file in working directory: %s" % e )
 753            for dataset_assoc in job.output_datasets + job.output_library_datasets:
 754                dataset = dataset_assoc.dataset
 755                self.sa_session.refresh( dataset )
 756                dataset.state = dataset.states.ERROR
 757                dataset.blurb = 'tool error'
 758                dataset.info = message
 759                dataset.set_size()
 760                dataset.dataset.set_total_size()
 761                dataset.mark_unhidden()
 762                if dataset.ext == 'auto':
 763                    dataset.extension = 'data'
 764                # Update (non-library) job output datasets through the object store
 765                if dataset not in job.output_library_datasets:
 766                    self.app.object_store.update_from_file(dataset.dataset, create=True)
 767                # Pause any dependent jobs (and those jobs' outputs)
 768                for dep_job_assoc in dataset.dependent_jobs:
 769                    self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." )
 770                self.sa_session.add( dataset )
 771                self.sa_session.flush()
 772            job.state = job.states.ERROR
 773            job.command_line = self.command_line
 774            job.info = message
 775            # TODO: Put setting the stdout, stderr, and exit code in one place
 776            # (not duplicated with the finish method).
 777            if ( len( stdout ) > DATABASE_MAX_STRING_SIZE ):
 778                stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
 779                log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
 780            job.stdout = stdout
 781            if ( len( stderr ) > DATABASE_MAX_STRING_SIZE ):
 782                stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
 783                log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
 784            job.stderr = stderr
 785            # Let the exit code be Null if one is not provided:
 786            if ( exit_code != None ):
 787                job.exit_code = exit_code
 788
 789            self.sa_session.add( job )
 790            self.sa_session.flush()
 791        #Perform email action even on failure.
 792        for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]:
 793            ActionBox.execute(self.app, self.sa_session, pja, job)
 794        # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up
 795        if self.tool:
 796            self.tool.job_failed( self, message, exception )
 797        delete_files = self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED)
 798        self.cleanup( delete_files=delete_files )
 799
 800    def pause( self, job=None, message=None ):
 801        if job is None:
 802            job = self.get_job()
 803        if message is None:
 804            message = "Execution of this dataset's job is paused"
 805        if job.state == job.states.NEW:
 806            for dataset_assoc in job.output_datasets + job.output_library_datasets:
 807                dataset_assoc.dataset.dataset.state = dataset_assoc.dataset.dataset.states.PAUSED
 808                dataset_assoc.dataset.info = message
 809                self.sa_session.add( dataset_assoc.dataset )
 810            job.state = job.states.PAUSED
 811            self.sa_session.add( job )
 812
 813    def change_state( self, state, info=False ):
 814        job = self.get_job()
 815        self.sa_session.refresh( job )
 816        for dataset_assoc in job.output_datasets + job.output_library_datasets:
 817            dataset = dataset_assoc.dataset
 818            self.sa_session.refresh( dataset )
 819            dataset.state = state
 820            if info:
 821                dataset.info = info
 822            self.sa_session.add( dataset )
 823            self.sa_session.flush()
 824        if info:
 825            job.info = info
 826        job.state = state
 827        self.sa_session.add( job )
 828        self.sa_session.flush()
 829
 830    def get_state( self ):
 831        job = self.get_job()
 832        self.sa_session.refresh( job )
 833        return job.state
 834
 835    def set_runner( self, runner_url, external_id ):
 836        log.warning('set_runner() is deprecated, use set_job_destination()')
 837        self.set_job_destination(self.job_destination, external_id)
 838
 839    def set_job_destination( self, job_destination, external_id=None ):
 840        """
 841        Persist job destination params in the database for recovery.
 842
 843        self.job_destination is not used because a runner may choose to rewrite
 844        parts of the destination (e.g. the params).
 845        """
 846        job = self.get_job()
 847        self.sa_session.refresh(job)
 848        log.debug('(%s) Persisting job destination (destination id: %s)' % (job.id, job_destination.id))
 849        job.destination_id = job_destination.id
 850        job.destination_params = job_destination.params
 851        job.job_runner_name = job_destination.runner
 852        job.job_runner_external_id = external_id
 853        self.sa_session.add(job)
 854        self.sa_session.flush()
 855
 856    def finish( self, stdout, stderr, tool_exit_code=None ):
 857        """
 858        Called to indicate that the associated command has been run. Updates
 859        the output datasets based on stderr and stdout from the command, and
 860        the contents of the output files.
 861        """
 862        stdout = unicodify( stdout )
 863        stderr = unicodify( stderr )
 864
 865        # default post job setup
 866        self.sa_session.expunge_all()
 867        job = self.get_job()
 868
 869        # TODO: After failing here, consider returning from the function.
 870        try:
 871            self.reclaim_ownership()
 872        except:
 873            log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) )
 874            return self.fail( job.info, stdout=stdout, stderr=stderr, exit_code=tool_exit_code )
 875
 876        # if the job was deleted, don't finish it
 877        if job.state == job.states.DELETED or job.state == job.states.ERROR:
 878            # SM: Note that, at this point, the exit code must be saved in case
 879            # there was an error. Errors caught here could mean that the job
 880            # was deleted by an administrator (based on old comments), but it
 881            # could also mean that a job was broken up into tasks and one of
 882            # the tasks failed. So include the stderr, stdout, and exit code:
 883            return self.fail( job.info, stderr=stderr, stdout=stdout, exit_code=tool_exit_code )
 884
 885        # Check the tool's stdout, stderr, and exit code for errors, but only
 886        # if the job has not already been marked as having an error.
 887        # The job's stdout and stderr will be set accordingly.
 888
 889        # We set final_job_state to use for dataset management, but *don't* set
 890        # job.state until after dataset collection to prevent history issues
 891        if ( self.check_tool_output( stdout, stderr, tool_exit_code, job ) ):
 892            final_job_state = job.states.OK
 893        else:
 894            final_job_state = job.states.ERROR
 895
 896        if self.write_version_cmd:
 897            version_filename = self.get_version_string_path()
 898            if os.path.exists(version_filename):
 899                self.version_string = open(version_filename).read()
 900                os.unlink(version_filename)
 901
 902        if self.app.config.outputs_to_working_directory and not self.__link_file_check():
 903            for dataset_path in self.get_output_fnames():
 904                try:
 905                    shutil.move( dataset_path.false_path, dataset_path.real_path )
 906                    log.debug( "finish(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) )
 907                except ( IOError, OSError ):
 908                    # this can happen if Galaxy is restarted during the job's
 909                    # finish method - the false_path file has already moved,
 910                    # and when the job is recovered, it won't be found.
 911                    if os.path.exists( dataset_path.real_path ) and os.stat( dataset_path.real_path ).st_size > 0:
 912                        log.warning( "finish(): %s not found, but %s is not empty, so it will be used instead" % ( dataset_path.false_path, dataset_path.real_path ) )
 913                    else:
 914                        # Prior to fail we need to set job.state
 915                        job.state = final_job_state
 916                        return self.fail( "Job %s's output dataset(s) could not be read" % job.id )
 917
 918        job_context = ExpressionContext( dict( stdout=job.stdout, stderr=job.stderr ) )
 919        for dataset_assoc in job.output_datasets + job.output_library_datasets:
 920            context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset )
 921            #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur
 922            for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations:  # need to update all associated output hdas, i.e. history was shared with job running
 923                trynum = 0
 924                while trynum < self.app.config.retry_job_output_collection:
 925                    try:
 926                        # Attempt to short circuit NFS attribute caching
 927                        os.stat( dataset.dataset.file_name )
 928                        os.chown( dataset.dataset.file_name, os.getuid(), -1 )
 929                        trynum = self.app.config.retry_job_output_collection
 930                    except ( OSError, ObjectNotFound ), e:
 931                        trynum += 1
 932                        log.warning( 'Error accessing %s, will retry: %s', dataset.dataset.file_name, e )
 933                        time.sleep( 2 )
 934                dataset.blurb = 'done'
 935                dataset.peek = 'no peek'
 936                dataset.info = (dataset.info or '')
 937                if context['stdout'].strip():
 938                    #Ensure white space between entries
 939                    dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip()
 940                if context['stderr'].strip():
 941                    #Ensure white space between entries
 942                    dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip()
 943                dataset.tool_version = self.version_string
 944                dataset.set_size()
 945                if 'uuid' in context:
 946                    dataset.dataset.uuid = context['uuid']
 947                # Update (non-library) job output datasets through the object store
 948                if dataset not in job.output_library_datasets:
 949                    self.app.object_store.update_from_file(dataset.dataset, create=True)
 950                if job.states.ERROR == final_job_state:
 951                    dataset.blurb = "error"
 952                    dataset.mark_unhidden()
 953                elif dataset.has_data():
 954                    # If the tool was expected to set the extension, attempt to retrieve it
 955                    if dataset.ext == 'auto':
 956                        dataset.extension = context.get( 'ext', 'data' )
 957                        dataset.init_meta( copy_from=dataset )
 958                    #if a dataset was copied, it won't appear in our dictionary:
 959                    #either use the metadata from originating output dataset, or call set_meta on the copies
 960                    #it would be quicker to just copy the metadata from the originating output dataset,
 961                    #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta()
 962                    if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ):
 963                        dataset.datatype.set_meta( dataset, overwrite=False )  # call datatype.set_meta directly for the initial set_meta call during dataset creation
 964                    elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state:
 965                        dataset._state = model.Dataset.states.FAILED_METADATA
 966                    else:
 967                        #load metadata from file
 968                        #we need to no longer allow metadata to be edited while the job is still running,
 969                        #since if it is edited, the metadata changed on the running output will no longer match
 970                        #the metadata that was stored to disk for use via the external process,
 971                        #and the changes made by the user will be lost, without warning or notice
 972                        dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset, self.sa_session ).filename_out )
 973                    try:
 974                        assert context.get( 'line_count', None ) is not None
 975                        if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
 976                            dataset.set_peek( line_count=context['line_count'], is_multi_byte=True )
 977                        else:
 978                            dataset.set_peek( line_count=context['line_count'] )
 979                    except:
 980                        if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte:
 981                            dataset.set_peek( is_multi_byte=True )
 982                        else:
 983                            dataset.set_peek()
 984                    try:
 985                        # set the name if provided by the tool
 986                        dataset.name = context['name']
 987                    except:
 988                        pass
 989                else:
 990                    dataset.blurb = "empty"
 991                    if dataset.ext == 'auto':
 992                        dataset.extension = 'txt'
 993                self.sa_session.add( dataset )
 994            if job.states.ERROR == final_job_state:
 995                log.debug( "setting dataset state to ERROR" )
 996                # TODO: This is where the state is being set to error. Change it!
 997                dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR
 998                # Pause any dependent jobs (and those jobs' outputs)
 999                for dep_job_assoc in dataset_assoc.dataset.dependent_jobs:
1000                    self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." )
1001            else:
1002                dataset_assoc.dataset.dataset.state = model.Dataset.states.OK
1003            # If any of the rest of the finish method below raises an
1004            # exception, the fail method will run and set the datasets to
1005            # ERROR.  The user will never see that the datasets are in error if
1006            # they were flushed as OK here, since upon doing so, the history
1007            # panel stops checking for updates.  So allow the
1008            # self.sa_session.flush() at the bottom of this method set
1009            # the state instead.
1010
1011        for pja in job.post_job_actions:
1012            ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job)
1013        # Flush all the dataset and job changes above.  Dataset state changes
1014        # will now be seen by the user.
1015        self.sa_session.flush()
1016        # Save stdout and stderr
1017        if len( job.stdout ) > DATABASE_MAX_STRING_SIZE:
1018            log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
1019        job.stdout = util.shrink_string_by_size( job.stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
1020        if len( job.stderr ) > DATABASE_MAX_STRING_SIZE:
1021            log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
1022        job.stderr = util.shrink_string_by_size( job.stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
1023        # The exit code will be null if there is no exit code to be set.
1024        # This is so that we don't assign an exit code, such as 0, that
1025        # is either incorrect or has the wrong semantics.
1026        if None != tool_exit_code:
1027            job.exit_code = tool_exit_code
1028        # custom post process setup
1029        inp_data = dict( [ ( da.name, da.dataset ) for da in job.input_datasets ] )
1030        out_data = dict( [ ( da.name, da.dataset ) for da in job.output_datasets ] )
1031        inp_data.update( [ ( da.name, da.dataset ) for da in job.input_library_datasets ] )
1032        out_data.update( [ ( da.name, da.dataset ) for da in job.output_library_datasets ] )
1033        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )  # why not re-use self.param_dict here? ##dunno...probably should, this causes tools.parameters.basic.UnvalidatedValue to be used in following methods instead of validated and transformed values during i.e. running workflows
1034        param_dict = self.tool.params_from_strings( param_dict, self.app )
1035        # Check for and move associated_files
1036        self.tool.collect_associated_files(out_data, self.working_directory)
1037        gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
1038        if gitd:
1039            self.tool.collect_associated_files({'': gitd}, self.working_directory)
1040        # Create generated output children and primary datasets and add to param_dict
1041        collected_datasets = {
1042            'children': self.tool.collect_child_datasets(out_data, self.working_directory),
1043            'primary': self.tool.collect_primary_datasets(out_data, self.working_directory)
1044        }
1045        param_dict.update({'__collected_datasets__': collected_datasets})
1046        # Certain tools require tasks to be completed after job execution
1047        # ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
1048        self.tool.exec_after_process( self.queue.app, inp_data, out_data, param_dict, job=job )
1049        # Call 'exec_after_process' hook
1050        self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data,
1051                             out_data=out_data, param_dict=param_dict,
1052                             tool=self.tool, stdout=job.stdout, stderr=job.stderr )
1053        job.command_line = self.command_line
1054
1055        bytes = 0
1056        # Once datasets are collected, set the total dataset size (includes extra files)
1057        for dataset_assoc in job.output_datasets:
1058            dataset_assoc.dataset.dataset.set_total_size()
1059            bytes += dataset_assoc.dataset.dataset.get_total_size()
1060
1061        if job.user:
1062            job.user.total_disk_usage += bytes
1063
1064        # fix permissions
1065        for path in [ dp.real_path for dp in self.get_mutable_output_fnames() ]:
1066            util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid )
1067
1068        # Finally set the job state.  This should only happen *after* all
1069        # dataset creation, and will allow us to eliminate force_history_refresh.
1070        job.state = final_job_state
1071        self.sa_session.flush()
1072
1073        log.debug( 'job %d ended' % self.job_id )
1074        delete_files = self.app.config.cleanup_job == 'always' or ( job.state == job.states.OK and self.app.config.cleanup_job == 'onsuccess' )
1075        self.cleanup( delete_files=delete_files )
1076
1077    def check_tool_output( self, stdout, stderr, tool_exit_code, job ):
1078        return check_output( self.tool, stdout, stderr, tool_exit_code, job )
1079
1080    def cleanup( self, delete_files=True ):
1081        # At least one of these tool cleanup actions (job import), is needed
1082        # for thetool to work properly, that is why one might want to run
1083        # cleanup but not delete files.
1084        try:
1085            if delete_files:
1086                for fname in self.extra_filenames:
1087                    os.remove( fname )
1088                self.external_output_metadata.cleanup_external_metadata( self.sa_session )
1089            galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session )
1090            galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job()
1091            galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app )
1092            if delete_files:
1093                self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id))
1094        except:
1095            log.exception( "Unable to cleanup job %d" % self.job_id )
1096
1097    def get_output_sizes( self ):
1098        sizes = []
1099        output_paths = self.get_output_fnames()
1100        for outfile in [ str( o ) for o in output_paths ]:
1101            if os.path.exists( outfile ):
1102                sizes.append( ( outfile, os.stat( outfile ).st_size ) )
1103            else:
1104                sizes.append( ( outfile, 0 ) )
1105        return sizes
1106
1107    def check_limits(self, runtime=None):
1108        if self.app.job_config.limits.output_size > 0:
1109            for outfile, size in self.get_output_sizes():
1110                if size > self.app.config.output_size_limit:
1111                    log.warning( '(%s) Job output %s is over the output size limit' % ( self.get_id_tag(), os.path.basename( outfile ) ) )
1112                    return 'Job output file grew too large (greater than %s), please try different inputs or parameters' % util.nice_size( self.app.job_config.limits.output_size )
1113        if self.app.job_config.limits.walltime_delta is not None and runtime is not None:
1114            if runtime > self.app.job_config.limits.walltime_delta:
1115                log.warning( '(%s) Job has reached walltime, it will be terminated' % ( self.get_id_tag() ) )
1116                return 'Job ran longer than the maximum allowed execution time (%s), please try different inputs or parameters' % self.app.job_config.limits.walltime
1117        return None
1118
1119    def get_command_line( self ):
1120        return self.command_line
1121
1122    def get_session_id( self ):
1123        return self.session_id
1124
1125    def get_env_setup_clause( self ):
1126        if self.app.config.environment_setup_file is None:
1127            return ''
1128        return '[ -f "%s" ] && . %s' % ( self.app.config.environment_setup_file, self.app.config.environment_setup_file )
1129
1130    def get_input_dataset_fnames( self,  ds ):
1131        filenames = []
1132        filenames = [ ds.file_name ]
1133        #we will need to stage in metadata file names also
1134        #TODO: would be better to only stage in metadata files that are actually needed (found in command line, referenced in config files, etc.)
1135        for key, value in ds.metadata.items():
1136            if isinstance( value, model.MetadataFile ):
1137                filenames.append( value.file_name )
1138        return filenames
1139
1140    def get_input_fnames( self ):
1141        job = self.get_job()
1142        filenames = []
1143        for da in job.input_datasets + job.input_library_datasets:  # da is JobToInputDatasetAssociation object
1144            if da.dataset:
1145                filenames.extend(self.get_input_dataset_fnames(da.dataset))
1146        return filenames
1147
1148    def get_input_paths( self, job=None ):
1149        if job is None:
1150            job = self.get_job()
1151        paths = []
1152        for da in job.input_datasets + job.input_library_datasets:  # da is JobToInputDatasetAssociation object
1153            if da.dataset:
1154                filenames = self.get_input_dataset_fnames(da.dataset)
1155                for real_path in filenames:
1156                    false_path = self.dataset_path_rewriter.rewrite_dataset_path( da.dataset, 'input' )
1157                    paths.append( DatasetPath( da.id, real_path=real_path, false_path=false_path, mutable=False ) )
1158        return paths
1159
1160    def get_output_fnames( self ):
1161        if self.output_paths is None:
1162            self.compute_outputs()
1163        return self.output_paths
1164
1165    def get_mutable_output_fnames( self ):
1166        if self.output_paths is None:
1167            self.compute_outputs()
1168        return filter( lambda dsp: dsp.mutable, self.output_paths )
1169
1170    def get_output_hdas_and_fnames( self ):
1171        if self.output_hdas_and_paths is None:
1172            self.compute_outputs()
1173        return self.output_hdas_and_paths
1174
1175    def compute_outputs( self ) :
1176        dataset_path_rewriter = self.dataset_path_rewriter
1177
1178        job = self.get_job()
1179        # Job output datasets are combination of history, library, jeha and gitd datasets.
1180        special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first()
1181        if not special:
1182            special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first()
1183        false_path = None
1184
1185        results = []
1186        for da in job.output_datasets + job.output_library_datasets:
1187            da_false_path = dataset_path_rewriter.rewrite_dataset_path( da.dataset, 'output' )
1188            mutable = da.dataset.dataset.external_filename is None
1189            dataset_path = DatasetPath( da.dataset.dataset.id, da.dataset.file_name, false_path=da_false_path, mutable=mutable )
1190            results.append( ( da.name, da.dataset, dataset_path ) )
1191
1192        self.output_paths = [t[2] for t in results]
1193        self.output_hdas_and_paths = dict([(t[0],  t[1:]) for t in results])
1194        if special:
1195            false_path = dataset_path_rewriter.rewrite_dataset_path( special.dataset, 'output' )
1196            dsp = DatasetPath( special.dataset.id, special.dataset.file_name, false_path )
1197            self.output_paths.append( dsp )
1198        return self.output_paths
1199
1200    def get_output_file_id( self, file ):
1201        if self.output_paths is None:
1202            self.get_output_fnames()
1203        for dp in self.output_paths:
1204            if self.app.config.outputs_to_working_directory and os.path.basename( dp.false_path ) == file:
1205                return dp.dataset_id
1206            elif os.path.basename( dp.real_path ) == file:
1207                return dp.dataset_id
1208        return None
1209
1210    def get_tool_provided_job_metadata( self ):
1211        if self.tool_provided_job_metadata is not None:
1212            return self.tool_provided_job_metadata
1213
1214        # Look for JSONified job metadata
1215        self.tool_provided_job_metadata = []
1216        meta_file = os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE )
1217        if os.path.exists( meta_file ):
1218            for line in open( meta_file, 'r' ):
1219                try:
1220                    line = from_json_string( line )
1221                    assert 'type' in line
1222                except:
1223                    log.exception( '(%s) Got JSON data from tool, but data is improperly formatted or no "type" key in data' % self.job_id )
1224                    log.debug( 'Offending data was: %s' % line )
1225                    continue
1226                # Set the dataset id if it's a dataset entry and isn't set.
1227                # This isn't insecure.  We loop the job's output datasets in
1228                # the finish method, so if a tool writes out metadata for a
1229                # dataset id that it doesn't own, it'll just be ignored.
1230                if line['type'] == 'dataset' and 'dataset_id' not in line:
1231                    try:
1232                        line['dataset_id'] = self.get_output_file_id( line['dataset'] )
1233                    except KeyError:
1234                        log.warning( '(%s) Tool provided job dataset-specific metadata without specifying a dataset' % self.job_id )
1235                        continue
1236                self.tool_provided_job_metadata.append( line )
1237        return self.tool_provided_job_metadata
1238
1239    def get_dataset_finish_context( self, job_context, dataset ):
1240        for meta in self.get_tool_provided_job_metadata():
1241            if meta['type'] == 'dataset' and meta['dataset_id'] == dataset.id:
1242                return ExpressionContext( meta, job_context )
1243        return job_context
1244
1245    def setup_external_metadata( self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds ):
1246        # extension could still be 'auto' if this is the upload tool.
1247        job = self.get_job()
1248        if set_extension:
1249            for output_dataset_assoc in job.output_datasets:
1250                if output_dataset_assoc.dataset.ext == 'auto':
1251                    context = self.get_dataset_finish_context( dict(), output_dataset_assoc.dataset.dataset )
1252                    output_dataset_assoc.dataset.extension = context.get( 'ext', 'data' )
1253            self.sa_session.flush()
1254        if tmp_dir is None:
1255            #this dir should should relative to the exec_dir
1256            tmp_dir = self.app.config.new_file_path
1257        if dataset_files_path is None:
1258            dataset_files_path = self.app.model.Dataset.file_path
1259        if config_root is None:
1260            config_root = self.app.config.root
1261        if config_file is None:
1262            config_file = self.app.config.config_file
1263        if datatypes_config is None:
1264            datatypes_config = self.app.datatypes_registry.integrated_datatypes_configs
1265        return self.external_output_metadata.setup_external_metadata( [ output_dataset_assoc.dataset for output_dataset_assoc in job.output_datasets ],
1266                                                                      self.sa_session,
1267                                                                      exec_dir=exec_dir,
1268                                                                      tmp_dir=tmp_dir,
1269                                                                      dataset_files_path=dataset_files_path,
1270                                                                      config_root=config_root,
1271                                                                      config_file=config_file,
1272                                                                      datatypes_config=datatypes_config,
1273                                                                      job_metadata=os.path.join( self.working_directory, TOOL_PROVIDED_JOB_METADATA_FILE ),
1274                                                                      **kwds )
1275
1276    @property
1277    def user( self ):
1278        job = self.get_job()
1279        if job.user is not None:
1280            return job.user.email
1281        elif job.galaxy_session is not None and job.galaxy_session.user is not None:
1282            return job.galaxy_session.user.email
1283        elif job.history is not None and job.history.user is not None:
1284            return job.history.user.email
1285        elif job.galaxy_session is not None:
1286            return 'anonymous@' + job.galaxy_session.remote_addr.split()[-1]
1287        else:
1288            return 'anonymous@unknown'
1289
1290    def __link_file_check( self ):
1291        """ outputs_to_working_directory breaks library uploads where data is
1292        linked.  This method is a hack that solves that problem, but is
1293        specific to the upload tool and relies on an injected job param.  This
1294        method should be removed ASAP and replaced with some properly generic
1295        and stateful way of determining link-only datasets. -nate
1296        """
1297        job = self.get_job()
1298        param_dict = job.get_param_values( self.app )
1299        return self.tool.id == 'upload1' and param_dict.get( 'link_data_only', None ) == 'link_to_files'
1300
1301    def _change_ownership( self, username, gid ):
1302        job = self.get_job()
1303        # FIXME: hardcoded path
1304        cmd = [ '/usr/bin/sudo', '-E', self.app.config.external_chown_script, self.working_directory, username, str( gid ) ]
1305        log.debug( '(%s) Changing ownership of working directory with: %s' % ( job.id, ' '.join( cmd ) ) )
1306        p = subprocess.Popen( cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
1307        # TODO: log stdout/stderr
1308        stdout, stderr = p.communicate()
1309        assert p.returncode == 0
1310
1311    def change_ownership_for_run( self ):
1312        job = self.get_job()
1313        if self.app.config.external_chown_script and job.user is not None:
1314            try:
1315                self._change_ownership( self.user_system_pwent[0], str( self.user_system_pwent[3] ) )
1316            except:
1317                log.exception( '(%s) Failed to change ownership of %s, making world-writable instead' % ( job.id, self.working_directory ) )
1318                os.chmod( self.working_directory, 0777 )
1319
1320    def reclaim_ownership( self ):
1321        job = self.get_job()
1322        if self.app.config.external_chown_script and job.user is not None:
1323            self._change_ownership( self.galaxy_system_pwent[0], str( self.galaxy_system_pwent[3] ) )
1324
1325    @property
1326    def user_system_pwent( self ):
1327        if self.__user_system_pwent is None:
1328            job = self.get_job()
1329            try:
1330                self.__user_system_pwent = pwd.getpwnam( job.user.email.split('@')[0] )
1331            except:
1332                pass
1333        return self.__user_system_pwent
1334
1335    @property
1336    def galaxy_system_pwent( self ):
1337        if self.__galaxy_system_pwent is None:
1338            self.__galaxy_system_pwent = pwd.getpwuid(os.getuid())
1339        return self.__galaxy_system_pwent
1340
1341    def get_output_destination( self, output_path ):
1342        """
1343        Destination for outputs marked as from_work_dir. This is the normal case,
1344        just copy these files directly to the ulimate destination.
1345        """
1346        return output_path
1347
1348    @property
1349    def requires_setting_metadata( self ):
1350        if self.tool:
1351            return self.tool.requires_setting_metadata
1352        return False
1353
1354
1355class TaskWrapper(JobWrapper):
1356    """
1357    Extension of JobWrapper intended for running tasks.
1358    Should be refactored into a generalized executable unit wrapper parent, then jobs and tasks.
1359    """
1360    # Abstract this to be more useful for running tasks that *don't* necessarily compose a job.
1361
1362    def __init__(self, task, queue):
1363        super(TaskWrapper, self).__init__(task.job, queue)
1364        self.task_id = task.id
1365        working_directory = task.working_directory
1366        self.working_directory = working_directory
1367        job_dataset_path_rewriter = self._job_dataset_path_rewriter( self.working_directory )
1368        self.dataset_path_rewriter = TaskPathRewriter( working_directory, job_dataset_path_rewriter )
1369        if task.prepare_input_files_cmd is not None:
1370            self.prepare_input_files_cmds = [ task.prepare_input_files_cmd ]
1371        else:
1372            self.prepare_input_files_cmds = None
1373        self.status = task.states.NEW
1374
1375    def can_split( self ):
1376        # Should the job handler split this job up? TaskWrapper should
1377        # always return False as the job has already been split.
1378        return False
1379
1380    def get_job( self ):
1381        if self.job_id:
1382            return self.sa_session.query( model.Job ).get( self.job_id )
1383        else:
1384            return None
1385
1386    def get_task( self ):
1387        return self.sa_session.query(model.Task).get(self.task_id)
1388
1389    def get_id_tag(self):
1390        # For compatibility with drmaa job runner and TaskWrapper, instead of using job_id directly
1391        return self.get_task().get_id_tag()
1392
1393    def get_param_dict( self ):
1394        """
1395        Restore the dictionary of parameters from the database.
1396        """
1397        job = self.sa_session.query( model.Job ).get( self.job_id )
1398        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
1399        param_dict = self.tool.params_from_strings( param_dict, self.app )
1400        return param_dict
1401
1402    def prepare( self, compute_environment=None ):
1403        """
1404        Prepare the job to run by creating the working directory and the
1405        config files.
1406        """
1407        # Restore parameters from the database
1408        job = self._load_job()
1409        task = self.get_task()
1410
1411        # DBTODO New method for generating command line for a task?
1412
1413        tool_evaluator = self._get_tool_evaluator( job )
1414        compute_environment = compute_environment or self.default_compute_environment( job )
1415        tool_evaluator.set_compute_environment( compute_environment )
1416
1417        self.sa_session.flush()
1418
1419        self.command_line, self.extra_filenames = tool_evaluator.build()
1420
1421        # FIXME: for now, tools get Galaxy's lib dir in their path
1422        if self.command_line and self.command_line.startswith( 'python' ):
1423            self.galaxy_lib_dir = os.path.abspath( "lib" )  # cwd = galaxy root
1424        # Shell fragment to inject dependencies
1425        self.dependency_shell_commands = self.tool.build_dependency_shell_commands()
1426        # We need command_line persisted to the db in order for Galaxy to re-queue the job
1427        # if the server was stopped and restarted before the job finished
1428        task.command_line = self.command_line
1429        self.sa_session.add( task )
1430        self.sa_session.flush()
1431
1432        self.param_dict = tool_evaluator.param_dict
1433        self.status = 'prepared'
1434        return self.extra_filenames
1435
1436    def fail( self, message, exception=False ):
1437        log.error("TaskWrapper Failure %s" % message)
1438        self.status = 'error'
1439        # How do we want to handle task failure?  Fail the job and let it clean up?
1440
1441    def change_state( self, state, info=False ):
1442        task = self.get_task()
1443        self.sa_session.refresh( task )
1444        if info:
1445            task.info = info
1446        task.state = state
1447        self.sa_session.add( task )
1448        self.sa_session.flush()
1449
1450    def get_state( self ):
1451        task = self.get_task()
1452        self.sa_session.refresh( task )
1453        return task.state
1454
1455    def get_exit_code( self ):
1456        task = self.get_task()
1457        self.sa_session.refresh( task )
1458        return task.exit_code
1459
1460    def set_runner( self, runner_url, external_id ):
1461        task = self.get_task()
1462        self.sa_session.refresh( task )
1463        task.task_runner_name = runner_url
1464        task.task_runner_external_id = external_id
1465        # DBTODO Check task job_runner_stuff
1466        self.sa_session.add( task )
1467        self.sa_session.flush()
1468
1469    def finish( self, stdout, stderr, tool_exit_code=None ):
1470        # DBTODO integrate previous finish logic.
1471        # Simple finish for tasks.  Just set the flag OK.
1472        """
1473        Called to indicate that the associated command has been run. Updates
1474        the output datasets based on stderr and stdout from the command, and
1475        the contents of the output files.
1476        """
1477        stdout = unicodify( stdout )
1478        stderr = unicodify( stderr )
1479
1480        # This may have ended too soon
1481        log.debug( 'task %s for job %d ended; exit code: %d'
1482                 % (self.task_id, self.job_id,
1483                    tool_exit_code if tool_exit_code != None else -256 ) )
1484        # default post job setup_external_metadata
1485        self.sa_session.expunge_all()
1486        task = self.get_task()
1487        # if the job was deleted, don't finish it
1488        if task.state == task.states.DELETED:
1489            # Job was deleted by an administrator
1490            delete_files = self.app.config.cleanup_job in ( 'always', 'onsuccess' )
1491            self.cleanup( delete_files=delete_files )
1492            return
1493        elif task.state == task.states.ERROR:
1494            self.fail( task.info )
1495            return
1496
1497        # Check what the tool returned. If the stdout or stderr matched
1498        # regular expressions that indicate errors, then set an error.
1499        # The same goes if the tool's exit code was in a given range.
1500        if ( self.check_tool_output( stdout, stderr, tool_exit_code, task ) ):
1501            task.state = task.states.OK
1502        else:
1503            task.state = task.states.ERROR
1504
1505        # Save stdout and stderr
1506        if len( stdout ) > DATABASE_MAX_STRING_SIZE:
1507            log.error( "stdout for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
1508        task.stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
1509        if len( stderr ) > DATABASE_MAX_STRING_SIZE:
1510            log.error( "stderr for task %d is greater than %s, only a portion will be logged to database" % ( task.id, DATABASE_MAX_STRING_SIZE_PRETTY ) )
1511        task.stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True )
1512        task.exit_code = tool_exit_code
1513        task.command_line = self.command_line
1514        self.sa_session.flush()
1515
1516    def cleanup( self ):
1517        # There is no task cleanup.  The job cleans up for all tasks.
1518        pass
1519
1520    def get_command_line( self ):
1521        return self.command_line
1522
1523    def get_session_id( self ):
1524        return self.session_id
1525
1526    def get_output_file_id( self, file ):
1527        # There is no permanent output file for tasks.
1528        return None
1529
1530    def get_tool_provided_job_metadata( self ):
1531        # DBTODO Handle this as applicable for tasks.
1532        return None
1533
1534    def get_dataset_finish_context( self, job_context, dataset ):
1535        # Handled at the parent job level.  Do nothing here.
1536        pass
1537
1538    def setup_external_metadata( self, exec_dir=None, tmp_dir=None, dataset_files_path=None, config_root=None, config_file=None, datatypes_config=None, set_extension=True, **kwds ):
1539        # There is no metadata setting for tasks.  This is handled after the merge, at the job level.
1540        return ""
1541
1542    def get_output_destination( self, output_path ):
1543        """
1544        Destination for outputs marked as from_work_dir. These must be copied with
1545        the same basenme as the path for the ultimate output destination. This is
1546        required in the task case so they can be merged.
1547        """
1548        return os.path.join( self.working_directory, os.path.basename( output_path ) )
1549
1550
1551class ComputeEnvironment( object ):
1552    """ Definition of the job as it will be run on the (potentially) remote
1553    compute server.
1554    """
1555    __metaclass__ = ABCMeta
1556
1557    @abstractmethod
1558    def output_paths( self ):
1559        """ Output DatasetPaths defined by job. """
1560
1561    @abstractmethod
1562    def input_paths( self ):
1563        """ Input DatasetPaths defined by job. """
1564
1565    @abstractmethod
1566    def working_directory( self ):
1567        """ Job working directory (potentially remote) """
1568
1569    @abstractmethod
1570    def config_directory( self ):
1571        """ Directory containing config files (potentially remote) """
1572
1573    @abstractmethod
1574    def sep( self ):
1575        """ os.path.sep for the platform this job will execute in.
1576        """
1577
1578    @abstractmethod
1579    def new_file_path( self ):
1580        """ Location to dump new files for this job on remote server. """
1581
1582    @abstractmethod
1583    def version_path( self ):
1584        """ Location of the version file for the underlying tool. """
1585
1586    @abstractmethod
1587    def unstructured_path_rewriter( self ):
1588        """ Return a function that takes in a value, determines if it is path
1589        to be rewritten (will be passed non-path values as well - onus is on
1590        this function to determine both if its input is a path and if it should
1591        be rewritten.)
1592        """
1593
1594
1595class SimpleComputeEnvironment( object ):
1596
1597    def config_directory( self ):
1598        return self.working_directory( )
1599
1600    def sep( self ):
1601        return os.path.sep
1602
1603    def unstructured_path_rewriter( self ):
1604        return lambda v: v
1605
1606
1607class SharedComputeEnvironment( SimpleComputeEnvironment ):
1608    """ Default ComputeEnviornment for job and task wrapper to pass
1609    to ToolEvaluator - valid when Galaxy and compute share all the relevant
1610    file systems.
1611    """
1612
1613    def __init__( self, job_wrapper, job ):
1614        self.app = job_wrapper.app
1615        self.job_wrapper = job_wrapper
1616        self.job = job
1617
1618    def output_paths( self ):
1619        return self.job_wrapper.get_output_fnames()
1620
1621    def input_paths( self ):
1622        return self.job_wrapper.get_input_paths( self.job )
1623
1624    def working_directory( self ):
1625        return self.job_wrapper.working_directory
1626
1627    def new_file_path( self ):
1628        return os.path.abspath( self.app.config.new_file_path )
1629
1630    def version_path( self ):
1631        return self.job_wrapper.get_version_string_path()
1632
1633
1634class NoopQueue( object ):
1635    """
1636    Implements the JobQueue / JobStopQueue interface but does nothing
1637    """
1638    def put( self, *args, **kwargs ):
1639        return
1640
1641    def put_stop( self, *args ):
1642        return
1643
1644    def shutdown( self ):
1645        return
1646
1647
1648class ParallelismInfo(object):
1649    """
1650    Stores the information (if any) for running multiple instances of the tool in parallel
1651    on the same set of inputs.
1652    """
1653    def __init__(self, tag):
1654        self.method = tag.get('method')
1655        if isinstance(tag, dict):
1656            items = tag.iteritems()
1657        else:
1658            items = tag.attrib.items()
1659        self.attributes = dict( [ item for item in items if item[ 0 ] != 'method' ])
1660        if len(self.attributes) == 0:
1661            # legacy basic mode - provide compatible defaults
1662            self.attributes['split_size'] = 20
1663            self.attributes['split_mode'] = 'number_of_parts'