/lib/galaxy/jobs/__init__.py
Python | 1663 lines | 1452 code | 55 blank | 156 comment | 116 complexity | 67fd55bd5e5369ca562536fa8927b8c2 MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1""" 2Support for running a tool in Galaxy via an internal job management system 3""" 4from abc import ABCMeta 5from abc import abstractmethod 6 7import time 8import copy 9import datetime 10import galaxy 11import logging 12import os 13import pwd 14import random 15import re 16import shutil 17import subprocess 18import sys 19import traceback 20from galaxy import model, util 21from galaxy.datatypes import metadata 22from galaxy.exceptions import ObjectInvalid, ObjectNotFound 23from galaxy.jobs.actions.post import ActionBox 24from galaxy.jobs.mapper import JobRunnerMapper 25from galaxy.jobs.runners import BaseJobRunner 26from galaxy.util.bunch import Bunch 27from galaxy.util.expressions import ExpressionContext 28from galaxy.util.json import from_json_string 29from galaxy.util import unicodify 30 31from .output_checker import check_output 32from .datasets import TaskPathRewriter 33from .datasets import OutputsToWorkingDirectoryPathRewriter 34from .datasets import NullDatasetPathRewriter 35from .datasets import DatasetPath 36 37log = logging.getLogger( __name__ ) 38 39DATABASE_MAX_STRING_SIZE = util.DATABASE_MAX_STRING_SIZE 40DATABASE_MAX_STRING_SIZE_PRETTY = util.DATABASE_MAX_STRING_SIZE_PRETTY 41 42# This file, if created in the job's working directory, will be used for 43# setting advanced metadata properties on the job and its associated outputs. 44# This interface is currently experimental, is only used by the upload tool, 45# and should eventually become API'd 46TOOL_PROVIDED_JOB_METADATA_FILE = 'galaxy.json' 47 48 49class JobDestination( Bunch ): 50 """ 51 Provides details about where a job runs 52 """ 53 def __init__(self, **kwds): 54 self['id'] = None 55 self['url'] = None 56 self['tags'] = None 57 self['runner'] = None 58 self['legacy'] = False 59 self['converted'] = False 60 # dict is appropriate (rather than a bunch) since keys may not be valid as attributes 61 self['params'] = dict() 62 super(JobDestination, self).__init__(**kwds) 63 64 # Store tags as a list 65 if self.tags is not None: 66 self['tags'] = [ x.strip() for x in self.tags.split(',') ] 67 68 69class JobToolConfiguration( Bunch ): 70 """ 71 Provides details on what handler and destination a tool should use 72 73 A JobToolConfiguration will have the required attribute 'id' and optional 74 attributes 'handler', 'destination', and 'params' 75 """ 76 def __init__(self, **kwds): 77 self['handler'] = None 78 self['destination'] = None 79 self['params'] = dict() 80 super(JobToolConfiguration, self).__init__(**kwds) 81 82 83class JobConfiguration( object ): 84 """A parser and interface to advanced job management features. 85 86 These features are configured in the job configuration, by default, ``job_conf.xml`` 87 """ 88 DEFAULT_NWORKERS = 4 89 90 def __init__(self, app): 91 """Parse the job configuration XML. 92 """ 93 self.app = app 94 self.runner_plugins = [] 95 self.handlers = {} 96 self.handler_runner_plugins = {} 97 self.default_handler_id = None 98 self.destinations = {} 99 self.destination_tags = {} 100 self.default_destination_id = None 101 self.tools = {} 102 self.limits = Bunch() 103 104 # Initialize the config 105 try: 106 tree = util.parse_xml(self.app.config.job_config_file) 107 self.__parse_job_conf_xml(tree) 108 except IOError: 109 log.warning( 'Job configuration "%s" does not exist, using legacy job configuration from Galaxy config file "%s" instead' % ( self.app.config.job_config_file, self.app.config.config_file ) ) 110 self.__parse_job_conf_legacy() 111 112 def __parse_job_conf_xml(self, tree): 113 """Loads the new-style job configuration from options in the job config file (by default, job_conf.xml). 114 115 :param tree: Object representing the root ``<job_conf>`` object in the job config file. 116 :type tree: ``xml.etree.ElementTree.Element`` 117 """ 118 root = tree.getroot() 119 log.debug('Loading job configuration from %s' % self.app.config.job_config_file) 120 121 # Parse job plugins 122 plugins = root.find('plugins') 123 if plugins is not None: 124 for plugin in self.__findall_with_required(plugins, 'plugin', ('id', 'type', 'load')): 125 if plugin.get('type') == 'runner': 126 workers = plugin.get('workers', plugins.get('workers', JobConfiguration.DEFAULT_NWORKERS)) 127 runner_kwds = self.__get_params(plugin) 128 runner_info = dict(id=plugin.get('id'), 129 load=plugin.get('load'), 130 workers=int(workers), 131 kwds=runner_kwds) 132 self.runner_plugins.append(runner_info) 133 else: 134 log.error('Unknown plugin type: %s' % plugin.get('type')) 135 # Load tasks if configured 136 if self.app.config.use_tasked_jobs: 137 self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers)) 138 139 # Parse handlers 140 handlers = root.find('handlers') 141 if handlers is not None: 142 for handler in self.__findall_with_required(handlers, 'handler'): 143 id = handler.get('id') 144 if id in self.handlers: 145 log.error("Handler '%s' overlaps handler with the same name, ignoring" % id) 146 else: 147 log.debug("Read definition for handler '%s'" % id) 148 self.handlers[id] = (id,) 149 for plugin in handler.findall('plugin'): 150 if id not in self.handler_runner_plugins: 151 self.handler_runner_plugins[id] = [] 152 self.handler_runner_plugins[id].append( plugin.get('id') ) 153 if handler.get('tags', None) is not None: 154 for tag in [ x.strip() for x in handler.get('tags').split(',') ]: 155 if tag in self.handlers: 156 self.handlers[tag].append(id) 157 else: 158 self.handlers[tag] = [id] 159 160 # Determine the default handler(s) 161 self.default_handler_id = self.__get_default(handlers, self.handlers.keys()) 162 163 # Parse destinations 164 destinations = root.find('destinations') 165 for destination in self.__findall_with_required(destinations, 'destination', ('id', 'runner')): 166 id = destination.get('id') 167 job_destination = JobDestination(**dict(destination.items())) 168 job_destination['params'] = self.__get_params(destination) 169 self.destinations[id] = (job_destination,) 170 if job_destination.tags is not None: 171 for tag in job_destination.tags: 172 if tag not in self.destinations: 173 self.destinations[tag] = [] 174 self.destinations[tag].append(job_destination) 175 176 # Determine the default destination 177 self.default_destination_id = self.__get_default(destinations, self.destinations.keys()) 178 179 # Parse tool mappings 180 tools = root.find('tools') 181 if tools is not None: 182 for tool in self.__findall_with_required(tools, 'tool'): 183 # There can be multiple definitions with identical ids, but different params 184 id = tool.get('id').lower().rstrip('/') 185 if id not in self.tools: 186 self.tools[id] = list() 187 self.tools[id].append(JobToolConfiguration(**dict(tool.items()))) 188 self.tools[id][-1]['params'] = self.__get_params(tool) 189 190 types = dict(registered_user_concurrent_jobs=int, 191 anonymous_user_concurrent_jobs=int, 192 walltime=str, 193 output_size=int) 194 195 self.limits = Bunch(registered_user_concurrent_jobs=None, 196 anonymous_user_concurrent_jobs=None, 197 walltime=None, 198 walltime_delta=None, 199 output_size=None, 200 concurrent_jobs={}) 201 202 # Parse job limits 203 limits = root.find('limits') 204 if limits is not None: 205 for limit in self.__findall_with_required(limits, 'limit', ('type',)): 206 type = limit.get('type') 207 if type == 'concurrent_jobs': 208 id = limit.get('tag', None) or limit.get('id') 209 self.limits.concurrent_jobs[id] = int(limit.text) 210 elif limit.text: 211 self.limits.__dict__[type] = types.get(type, str)(limit.text) 212 213 if self.limits.walltime is not None: 214 h, m, s = [ int( v ) for v in self.limits.walltime.split( ':' ) ] 215 self.limits.walltime_delta = datetime.timedelta( 0, s, 0, 0, m, h ) 216 217 log.debug('Done loading job configuration') 218 219 def __parse_job_conf_legacy(self): 220 """Loads the old-style job configuration from options in the galaxy config file (by default, universe_wsgi.ini). 221 """ 222 log.debug('Loading job configuration from %s' % self.app.config.config_file) 223 224 # Always load local and lwr 225 self.runner_plugins = [dict(id='local', load='local', workers=self.app.config.local_job_queue_workers), dict(id='lwr', load='lwr', workers=self.app.config.cluster_job_queue_workers)] 226 # Load tasks if configured 227 if self.app.config.use_tasked_jobs: 228 self.runner_plugins.append(dict(id='tasks', load='tasks', workers=self.app.config.local_task_queue_workers)) 229 for runner in self.app.config.start_job_runners: 230 self.runner_plugins.append(dict(id=runner, load=runner, workers=self.app.config.cluster_job_queue_workers)) 231 232 # Set the handlers 233 for id in self.app.config.job_handlers: 234 self.handlers[id] = (id,) 235 236 self.handlers['default_job_handlers'] = self.app.config.default_job_handlers 237 self.default_handler_id = 'default_job_handlers' 238 239 # Set tool handler configs 240 for id, tool_handlers in self.app.config.tool_handlers.items(): 241 self.tools[id] = list() 242 for handler_config in tool_handlers: 243 # rename the 'name' key to 'handler' 244 handler_config['handler'] = handler_config.pop('name') 245 self.tools[id].append(JobToolConfiguration(**handler_config)) 246 247 # Set tool runner configs 248 for id, tool_runners in self.app.config.tool_runners.items(): 249 # Might have been created in the handler parsing above 250 if id not in self.tools: 251 self.tools[id] = list() 252 for runner_config in tool_runners: 253 url = runner_config['url'] 254 if url not in self.destinations: 255 # Create a new "legacy" JobDestination - it will have its URL converted to a destination params once the appropriate plugin has loaded 256 self.destinations[url] = (JobDestination(id=url, runner=url.split(':', 1)[0], url=url, legacy=True, converted=False),) 257 for tool_conf in self.tools[id]: 258 if tool_conf.params == runner_config.get('params', {}): 259 tool_conf['destination'] = url 260 break 261 else: 262 # There was not an existing config (from the handlers section) with the same params 263 # rename the 'url' key to 'destination' 264 runner_config['destination'] = runner_config.pop('url') 265 self.tools[id].append(JobToolConfiguration(**runner_config)) 266 267 self.destinations[self.app.config.default_cluster_job_runner] = (JobDestination(id=self.app.config.default_cluster_job_runner, runner=self.app.config.default_cluster_job_runner.split(':', 1)[0], url=self.app.config.default_cluster_job_runner, legacy=True, converted=False),) 268 self.default_destination_id = self.app.config.default_cluster_job_runner 269 270 # Set the job limits 271 self.limits = Bunch(registered_user_concurrent_jobs=self.app.config.registered_user_job_limit, 272 anonymous_user_concurrent_jobs=self.app.config.anonymous_user_job_limit, 273 walltime=self.app.config.job_walltime, 274 walltime_delta=self.app.config.job_walltime_delta, 275 output_size=self.app.config.output_size_limit, 276 concurrent_jobs={}) 277 278 log.debug('Done loading job configuration') 279 280 def __get_default(self, parent, names): 281 """Returns the default attribute set in a parent tag like <handlers> or <destinations>, or return the ID of the child, if there is no explicit default and only one child. 282 283 :param parent: Object representing a tag that may or may not have a 'default' attribute. 284 :type parent: ``xml.etree.ElementTree.Element`` 285 :param names: The list of destination or handler IDs or tags that were loaded. 286 :type names: list of str 287 288 :returns: str -- id or tag representing the default. 289 """ 290 rval = parent.get('default') 291 if rval is not None: 292 # If the parent element has a 'default' attribute, use the id or tag in that attribute 293 if rval not in names: 294 raise Exception("<%s> default attribute '%s' does not match a defined id or tag in a child element" % (parent.tag, rval)) 295 log.debug("<%s> default set to child with id or tag '%s'" % (parent.tag, rval)) 296 elif len(names) == 1: 297 log.info("Setting <%s> default to child with id '%s'" % (parent.tag, names[0])) 298 rval = names[0] 299 else: 300 raise Exception("No <%s> default specified, please specify a valid id or tag with the 'default' attribute" % parent.tag) 301 return rval 302 303 def __findall_with_required(self, parent, match, attribs=None): 304 """Like ``xml.etree.ElementTree.Element.findall()``, except only returns children that have the specified attribs. 305 306 :param parent: Parent element in which to find. 307 :type parent: ``xml.etree.ElementTree.Element`` 308 :param match: Name of child elements to find. 309 :type match: str 310 :param attribs: List of required attributes in children elements. 311 :type attribs: list of str 312 313 :returns: list of ``xml.etree.ElementTree.Element`` 314 """ 315 rval = [] 316 if attribs is None: 317 attribs = ('id',) 318 for elem in parent.findall(match): 319 for attrib in attribs: 320 if attrib not in elem.attrib: 321 log.warning("required '%s' attribute is missing from <%s> element" % (attrib, match)) 322 break 323 else: 324 rval.append(elem) 325 return rval 326 327 def __get_params(self, parent): 328 """Parses any child <param> tags in to a dictionary suitable for persistence. 329 330 :param parent: Parent element in which to find child <param> tags. 331 :type parent: ``xml.etree.ElementTree.Element`` 332 333 :returns: dict 334 """ 335 rval = {} 336 for param in parent.findall('param'): 337 rval[param.get('id')] = param.text 338 return rval 339 340 @property 341 def default_job_tool_configuration(self): 342 """The default JobToolConfiguration, used if a tool does not have an explicit defintion in the configuration. It consists of a reference to the default handler and default destination. 343 344 :returns: JobToolConfiguration -- a representation of a <tool> element that uses the default handler and destination 345 """ 346 return JobToolConfiguration(id='default', handler=self.default_handler_id, destination=self.default_destination_id) 347 348 # Called upon instantiation of a Tool object 349 def get_job_tool_configurations(self, ids): 350 """Get all configured JobToolConfigurations for a tool ID, or, if given a list of IDs, the JobToolConfigurations for the first id in ``ids`` matching a tool definition. 351 352 .. note:: 353 354 You should not mix tool shed tool IDs, versionless tool shed IDs, and tool config tool IDs that refer to the same tool. 355 356 :param ids: Tool ID or IDs to fetch the JobToolConfiguration of. 357 :type ids: list or str. 358 :returns: list -- JobToolConfiguration Bunches representing <tool> elements matching the specified ID(s). 359 360 Example tool ID strings include: 361 362 * Full tool shed id: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool/1.0.0`` 363 * Tool shed id less version: ``toolshed.example.org/repos/nate/filter_tool_repo/filter_tool`` 364 * Tool config tool id: ``filter_tool`` 365 """ 366 rval = [] 367 # listify if ids is a single (string) id 368 ids = util.listify(ids) 369 for id in ids: 370 if id in self.tools: 371 # If a tool has definitions that include job params but not a 372 # definition for jobs without params, include the default 373 # config 374 for job_tool_configuration in self.tools[id]: 375 if not job_tool_configuration.params: 376 break 377 else: 378 rval.append(self.default_job_tool_configuration) 379 rval.extend(self.tools[id]) 380 break 381 else: 382 rval.append(self.default_job_tool_configuration) 383 return rval 384 385 def __get_single_item(self, collection): 386 """Given a collection of handlers or destinations, return one item from the collection at random. 387 """ 388 # Done like this to avoid random under the assumption it's faster to avoid it 389 if len(collection) == 1: 390 return collection[0] 391 else: 392 return random.choice(collection) 393 394 # This is called by Tool.get_job_handler() 395 def get_handler(self, id_or_tag): 396 """Given a handler ID or tag, return the provided ID or an ID matching the provided tag 397 398 :param id_or_tag: A handler ID or tag. 399 :type id_or_tag: str 400 401 :returns: str -- A valid job handler ID. 402 """ 403 if id_or_tag is None: 404 id_or_tag = self.default_handler_id 405 return self.__get_single_item(self.handlers[id_or_tag]) 406 407 def get_destination(self, id_or_tag): 408 """Given a destination ID or tag, return the JobDestination matching the provided ID or tag 409 410 :param id_or_tag: A destination ID or tag. 411 :type id_or_tag: str 412 413 :returns: JobDestination -- A valid destination 414 415 Destinations are deepcopied as they are expected to be passed in to job 416 runners, which will modify them for persisting params set at runtime. 417 """ 418 if id_or_tag is None: 419 id_or_tag = self.default_destination_id 420 return copy.deepcopy(self.__get_single_item(self.destinations[id_or_tag])) 421 422 def get_destinations(self, id_or_tag): 423 """Given a destination ID or tag, return all JobDestinations matching the provided ID or tag 424 425 :param id_or_tag: A destination ID or tag. 426 :type id_or_tag: str 427 428 :returns: list or tuple of JobDestinations 429 430 Destinations are not deepcopied, so they should not be passed to 431 anything which might modify them. 432 """ 433 return self.destinations.get(id_or_tag, None) 434 435 def get_job_runner_plugins(self, handler_id): 436 """Load all configured job runner plugins 437 438 :returns: list of job runner plugins 439 """ 440 rval = {} 441 if handler_id in self.handler_runner_plugins: 442 plugins_to_load = [ rp for rp in self.runner_plugins if rp['id'] in self.handler_runner_plugins[handler_id] ] 443 log.info( "Handler '%s' will load specified runner plugins: %s", handler_id, ', '.join( [ rp['id'] for rp in plugins_to_load ] ) ) 444 else: 445 plugins_to_load = self.runner_plugins 446 log.info( "Handler '%s' will load all configured runner plugins", handler_id ) 447 for runner in plugins_to_load: 448 class_names = [] 449 module = None 450 id = runner['id'] 451 load = runner['load'] 452 if ':' in load: 453 # Name to load was specified as '<module>:<class>' 454 module_name, class_name = load.rsplit(':', 1) 455 class_names = [ class_name ] 456 module = __import__( module_name ) 457 else: 458 # Name to load was specified as '<module>' 459 if '.' not in load: 460 # For legacy reasons, try from galaxy.jobs.runners first if there's no '.' in the name 461 module_name = 'galaxy.jobs.runners.' + load 462 try: 463 module = __import__( module_name ) 464 except ImportError: 465 # No such module, we'll retry without prepending galaxy.jobs.runners. 466 # All other exceptions (e.g. something wrong with the module code) will raise 467 pass 468 if module is None: 469 # If the name included a '.' or loading from the static runners path failed, try the original name 470 module = __import__( load ) 471 module_name = load 472 if module is None: 473 # Module couldn't be loaded, error should have already been displayed 474 continue 475 for comp in module_name.split( "." )[1:]: 476 module = getattr( module, comp ) 477 if not class_names: 478 # If there's not a ':', we check <module>.__all__ for class names 479 try: 480 assert module.__all__ 481 class_names = module.__all__ 482 except AssertionError: 483 log.error( 'Runner "%s" does not contain a list of exported classes in __all__' % load ) 484 continue 485 for class_name in class_names: 486 runner_class = getattr( module, class_name ) 487 try: 488 assert issubclass(runner_class, BaseJobRunner) 489 except TypeError: 490 log.warning("A non-class name was found in __all__, ignoring: %s" % id) 491 continue 492 except AssertionError: 493 log.warning("Job runner classes must be subclassed from BaseJobRunner, %s has bases: %s" % (id, runner_class.__bases__)) 494 continue 495 try: 496 rval[id] = runner_class( self.app, runner[ 'workers' ], **runner.get( 'kwds', {} ) ) 497 except TypeError: 498 log.exception( "Job runner '%s:%s' has not been converted to a new-style runner or encountered TypeError on load" % ( module_name, class_name ) ) 499 rval[id] = runner_class( self.app ) 500 log.debug( "Loaded job runner '%s:%s' as '%s'" % ( module_name, class_name, id ) ) 501 return rval 502 503 def is_id(self, collection): 504 """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID 505 506 :param collection: A representation of a destination or handler 507 :type collection: tuple or list 508 509 :returns: bool 510 """ 511 return type(collection) == tuple 512 513 def is_tag(self, collection): 514 """Given a collection of handlers or destinations, indicate whether the collection represents a tag or a real ID 515 516 :param collection: A representation of a destination or handler 517 :type collection: tuple or list 518 519 :returns: bool 520 """ 521 return type(collection) == list 522 523 def is_handler(self, server_name): 524 """Given a server name, indicate whether the server is a job handler 525 526 :param server_name: The name to check 527 :type server_name: str 528 529 :return: bool 530 """ 531 for collection in self.handlers.values(): 532 if server_name in collection: 533 return True 534 return False 535 536 def convert_legacy_destinations(self, job_runners): 537 """Converts legacy (from a URL) destinations to contain the appropriate runner params defined in the URL. 538 539 :param job_runners: All loaded job runner plugins. 540 :type job_runners: list of job runner plugins 541 """ 542 for id, destination in [ ( id, destinations[0] ) for id, destinations in self.destinations.items() if self.is_id(destinations) ]: 543 # Only need to deal with real destinations, not members of tags 544 if destination.legacy and not destination.converted: 545 if destination.runner in job_runners: 546 destination.params = job_runners[destination.runner].url_to_destination(destination.url).params 547 destination.converted = True 548 if destination.params: 549 log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url)) 550 for k, v in destination.params.items(): 551 log.debug(" %s: %s" % (k, v)) 552 else: 553 log.debug("Legacy destination with id '%s', url '%s' converted, got params:" % (id, destination.url)) 554 else: 555 log.warning("Legacy destination with id '%s' could not be converted: Unknown runner plugin: %s" % (id, destination.runner)) 556 557 558class JobWrapper( object ): 559 """ 560 Wraps a 'model.Job' with convenience methods for running processes and 561 state management. 562 """ 563 def __init__( self, job, queue ): 564 self.job_id = job.id 565 self.session_id = job.session_id 566 self.user_id = job.user_id 567 self.tool = queue.app.toolbox.tools_by_id.get( job.tool_id, None ) 568 self.queue = queue 569 self.app = queue.app 570 self.sa_session = self.app.model.context 571 self.extra_filenames = [] 572 self.command_line = None 573 # Tool versioning variables 574 self.write_version_cmd = None 575 self.version_string = "" 576 self.galaxy_lib_dir = None 577 # With job outputs in the working directory, we need the working 578 # directory to be set before prepare is run, or else premature deletion 579 # and job recovery fail. 580 # Create the working dir if necessary 581 try: 582 self.app.object_store.create(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) 583 self.working_directory = self.app.object_store.get_filename(job, base_dir='job_work', dir_only=True, extra_dir=str(self.job_id)) 584 log.debug('(%s) Working directory for job is: %s' % (self.job_id, self.working_directory)) 585 except ObjectInvalid: 586 raise Exception('Unable to create job working directory, job failure') 587 self.dataset_path_rewriter = self._job_dataset_path_rewriter( self.working_directory ) 588 self.output_paths = None 589 self.output_hdas_and_paths = None 590 self.tool_provided_job_metadata = None 591 # Wrapper holding the info required to restore and clean up from files used for setting metadata externally 592 self.external_output_metadata = metadata.JobExternalOutputMetadataWrapper( job ) 593 self.job_runner_mapper = JobRunnerMapper( self, queue.dispatcher.url_to_destination, self.app.job_config ) 594 self.params = None 595 if job.params: 596 self.params = from_json_string( job.params ) 597 598 self.__user_system_pwent = None 599 self.__galaxy_system_pwent = None 600 601 def _job_dataset_path_rewriter( self, working_directory ): 602 if self.app.config.outputs_to_working_directory: 603 dataset_path_rewriter = OutputsToWorkingDirectoryPathRewriter( working_directory ) 604 else: 605 dataset_path_rewriter = NullDatasetPathRewriter( ) 606 return dataset_path_rewriter 607 608 def can_split( self ): 609 # Should the job handler split this job up? 610 return self.app.config.use_tasked_jobs and self.tool.parallelism 611 612 def get_job_runner_url( self ): 613 log.warning('(%s) Job runner URLs are deprecated, use destinations instead.' % self.job_id) 614 return self.job_destination.url 615 616 def get_parallelism(self): 617 return self.tool.parallelism 618 619 # legacy naming 620 get_job_runner = get_job_runner_url 621 622 @property 623 def job_destination(self): 624 """Return the JobDestination that this job will use to run. This will 625 either be a configured destination, a randomly selected destination if 626 the configured destination was a tag, or a dynamically generated 627 destination from the dynamic runner. 628 629 Calling this method for the first time causes the dynamic runner to do 630 its calculation, if any. 631 632 :returns: ``JobDestination`` 633 """ 634 return self.job_runner_mapper.get_job_destination(self.params) 635 636 def get_job( self ): 637 return self.sa_session.query( model.Job ).get( self.job_id ) 638 639 def get_id_tag(self): 640 # For compatability with drmaa, which uses job_id right now, and TaskWrapper 641 return self.get_job().get_id_tag() 642 643 def get_param_dict( self ): 644 """ 645 Restore the dictionary of parameters from the database. 646 """ 647 job = self.get_job() 648 param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] ) 649 param_dict = self.tool.params_from_strings( param_dict, self.app ) 650 return param_dict 651 652 def get_version_string_path( self ): 653 return os.path.abspath(os.path.join(self.app.config.new_file_path, "GALAXY_VERSION_STRING_%s" % self.job_id)) 654 655 def prepare( self, compute_environment=None ): 656 """ 657 Prepare the job to run by creating the working directory and the 658 config files. 659 """ 660 self.sa_session.expunge_all() # this prevents the metadata reverting that has been seen in conjunction with the PBS job runner 661 662 if not os.path.exists( self.working_directory ): 663 os.mkdir( self.working_directory ) 664 665 job = self._load_job() 666 667 def get_special( ): 668 special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() 669 if not special: 670 special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() 671 return special 672 673 tool_evaluator = self._get_tool_evaluator( job ) 674 compute_environment = compute_environment or self.default_compute_environment( job ) 675 tool_evaluator.set_compute_environment( compute_environment, get_special=get_special ) 676 677 self.sa_session.flush() 678 679 self.command_line, self.extra_filenames = tool_evaluator.build() 680 # FIXME: for now, tools get Galaxy's lib dir in their path 681 if self.command_line and self.command_line.startswith( 'python' ): 682 self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root 683 # Shell fragment to inject dependencies 684 self.dependency_shell_commands = self.tool.build_dependency_shell_commands() 685 # We need command_line persisted to the db in order for Galaxy to re-queue the job 686 # if the server was stopped and restarted before the job finished 687 job.command_line = self.command_line 688 self.sa_session.add( job ) 689 self.sa_session.flush() 690 # Return list of all extra files 691 self.param_dict = tool_evaluator.param_dict 692 version_string_cmd = self.tool.version_string_cmd 693 if version_string_cmd: 694 self.write_version_cmd = "%s > %s 2>&1" % ( version_string_cmd, compute_environment.version_path() ) 695 else: 696 self.write_version_cmd = None 697 return self.extra_filenames 698 699 def default_compute_environment( self, job=None ): 700 if not job: 701 job = self.get_job() 702 return SharedComputeEnvironment( self, job ) 703 704 def _load_job( self ): 705 # Load job from database and verify it has user or session. 706 # Restore parameters from the database 707 job = self.get_job() 708 if job.user is None and job.galaxy_session is None: 709 raise Exception( 'Job %s has no user and no session.' % job.id ) 710 return job 711 712 def _get_tool_evaluator( self, job ): 713 # Hacky way to avoid cirular import for now. 714 # Placing ToolEvaluator in either jobs or tools 715 # result in ciruclar dependency. 716 from galaxy.tools.evaluation import ToolEvaluator 717 718 tool_evaluator = ToolEvaluator( 719 app=self.app, 720 job=job, 721 tool=self.tool, 722 local_working_directory=self.working_directory, 723 ) 724 return tool_evaluator 725 726 def fail( self, message, exception=False, stdout="", stderr="", exit_code=None ): 727 """ 728 Indicate job failure by setting state and message on all output 729 datasets. 730 """ 731 job = self.get_job() 732 self.sa_session.refresh( job ) 733 # if the job was deleted, don't fail it 734 if not job.state == job.states.DELETED: 735 # Check if the failure is due to an exception 736 if exception: 737 # Save the traceback immediately in case we generate another 738 # below 739 job.traceback = traceback.format_exc() 740 # Get the exception and let the tool attempt to generate 741 # a better message 742 etype, evalue, tb = sys.exc_info() 743 m = self.tool.handle_job_failure_exception( evalue ) 744 if m: 745 message = m 746 if self.app.config.outputs_to_working_directory: 747 for dataset_path in self.get_output_fnames(): 748 try: 749 shutil.move( dataset_path.false_path, dataset_path.real_path ) 750 log.debug( "fail(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) ) 751 except ( IOError, OSError ), e: 752 log.error( "fail(): Missing output file in working directory: %s" % e ) 753 for dataset_assoc in job.output_datasets + job.output_library_datasets: 754 dataset = dataset_assoc.dataset 755 self.sa_session.refresh( dataset ) 756 dataset.state = dataset.states.ERROR 757 dataset.blurb = 'tool error' 758 dataset.info = message 759 dataset.set_size() 760 dataset.dataset.set_total_size() 761 dataset.mark_unhidden() 762 if dataset.ext == 'auto': 763 dataset.extension = 'data' 764 # Update (non-library) job output datasets through the object store 765 if dataset not in job.output_library_datasets: 766 self.app.object_store.update_from_file(dataset.dataset, create=True) 767 # Pause any dependent jobs (and those jobs' outputs) 768 for dep_job_assoc in dataset.dependent_jobs: 769 self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." ) 770 self.sa_session.add( dataset ) 771 self.sa_session.flush() 772 job.state = job.states.ERROR 773 job.command_line = self.command_line 774 job.info = message 775 # TODO: Put setting the stdout, stderr, and exit code in one place 776 # (not duplicated with the finish method). 777 if ( len( stdout ) > DATABASE_MAX_STRING_SIZE ): 778 stdout = util.shrink_string_by_size( stdout, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True ) 779 log.info( "stdout for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) ) 780 job.stdout = stdout 781 if ( len( stderr ) > DATABASE_MAX_STRING_SIZE ): 782 stderr = util.shrink_string_by_size( stderr, DATABASE_MAX_STRING_SIZE, join_by="\n..\n", left_larger=True, beginning_on_size_error=True ) 783 log.info( "stderr for job %d is greater than %s, only a portion will be logged to database" % ( job.id, DATABASE_MAX_STRING_SIZE_PRETTY ) ) 784 job.stderr = stderr 785 # Let the exit code be Null if one is not provided: 786 if ( exit_code != None ): 787 job.exit_code = exit_code 788 789 self.sa_session.add( job ) 790 self.sa_session.flush() 791 #Perform email action even on failure. 792 for pja in [pjaa.post_job_action for pjaa in job.post_job_actions if pjaa.post_job_action.action_type == "EmailAction"]: 793 ActionBox.execute(self.app, self.sa_session, pja, job) 794 # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up 795 if self.tool: 796 self.tool.job_failed( self, message, exception ) 797 delete_files = self.app.config.cleanup_job == 'always' or (self.app.config.cleanup_job == 'onsuccess' and job.state == job.states.DELETED) 798 self.cleanup( delete_files=delete_files ) 799 800 def pause( self, job=None, message=None ): 801 if job is None: 802 job = self.get_job() 803 if message is None: 804 message = "Execution of this dataset's job is paused" 805 if job.state == job.states.NEW: 806 for dataset_assoc in job.output_datasets + job.output_library_datasets: 807 dataset_assoc.dataset.dataset.state = dataset_assoc.dataset.dataset.states.PAUSED 808 dataset_assoc.dataset.info = message 809 self.sa_session.add( dataset_assoc.dataset ) 810 job.state = job.states.PAUSED 811 self.sa_session.add( job ) 812 813 def change_state( self, state, info=False ): 814 job = self.get_job() 815 self.sa_session.refresh( job ) 816 for dataset_assoc in job.output_datasets + job.output_library_datasets: 817 dataset = dataset_assoc.dataset 818 self.sa_session.refresh( dataset ) 819 dataset.state = state 820 if info: 821 dataset.info = info 822 self.sa_session.add( dataset ) 823 self.sa_session.flush() 824 if info: 825 job.info = info 826 job.state = state 827 self.sa_session.add( job ) 828 self.sa_session.flush() 829 830 def get_state( self ): 831 job = self.get_job() 832 self.sa_session.refresh( job ) 833 return job.state 834 835 def set_runner( self, runner_url, external_id ): 836 log.warning('set_runner() is deprecated, use set_job_destination()') 837 self.set_job_destination(self.job_destination, external_id) 838 839 def set_job_destination( self, job_destination, external_id=None ): 840 """ 841 Persist job destination params in the database for recovery. 842 843 self.job_destination is not used because a runner may choose to rewrite 844 parts of the destination (e.g. the params). 845 """ 846 job = self.get_job() 847 self.sa_session.refresh(job) 848 log.debug('(%s) Persisting job destination (destination id: %s)' % (job.id, job_destination.id)) 849 job.destination_id = job_destination.id 850 job.destination_params = job_destination.params 851 job.job_runner_name = job_destination.runner 852 job.job_runner_external_id = external_id 853 self.sa_session.add(job) 854 self.sa_session.flush() 855 856 def finish( self, stdout, stderr, tool_exit_code=None ): 857 """ 858 Called to indicate that the associated command has been run. Updates 859 the output datasets based on stderr and stdout from the command, and 860 the contents of the output files. 861 """ 862 stdout = unicodify( stdout ) 863 stderr = unicodify( stderr ) 864 865 # default post job setup 866 self.sa_session.expunge_all() 867 job = self.get_job() 868 869 # TODO: After failing here, consider returning from the function. 870 try: 871 self.reclaim_ownership() 872 except: 873 log.exception( '(%s) Failed to change ownership of %s, failing' % ( job.id, self.working_directory ) ) 874 return self.fail( job.info, stdout=stdout, stderr=stderr, exit_code=tool_exit_code ) 875 876 # if the job was deleted, don't finish it 877 if job.state == job.states.DELETED or job.state == job.states.ERROR: 878 # SM: Note that, at this point, the exit code must be saved in case 879 # there was an error. Errors caught here could mean that the job 880 # was deleted by an administrator (based on old comments), but it 881 # could also mean that a job was broken up into tasks and one of 882 # the tasks failed. So include the stderr, stdout, and exit code: 883 return self.fail( job.info, stderr=stderr, stdout=stdout, exit_code=tool_exit_code ) 884 885 # Check the tool's stdout, stderr, and exit code for errors, but only 886 # if the job has not already been marked as having an error. 887 # The job's stdout and stderr will be set accordingly. 888 889 # We set final_job_state to use for dataset management, but *don't* set 890 # job.state until after dataset collection to prevent history issues 891 if ( self.check_tool_output( stdout, stderr, tool_exit_code, job ) ): 892 final_job_state = job.states.OK 893 else: 894 final_job_state = job.states.ERROR 895 896 if self.write_version_cmd: 897 version_filename = self.get_version_string_path() 898 if os.path.exists(version_filename): 899 self.version_string = open(version_filename).read() 900 os.unlink(version_filename) 901 902 if self.app.config.outputs_to_working_directory and not self.__link_file_check(): 903 for dataset_path in self.get_output_fnames(): 904 try: 905 shutil.move( dataset_path.false_path, dataset_path.real_path ) 906 log.debug( "finish(): Moved %s to %s" % ( dataset_path.false_path, dataset_path.real_path ) ) 907 except ( IOError, OSError ): 908 # this can happen if Galaxy is restarted during the job's 909 # finish method - the false_path file has already moved, 910 # and when the job is recovered, it won't be found. 911 if os.path.exists( dataset_path.real_path ) and os.stat( dataset_path.real_path ).st_size > 0: 912 log.warning( "finish(): %s not found, but %s is not empty, so it will be used instead" % ( dataset_path.false_path, dataset_path.real_path ) ) 913 else: 914 # Prior to fail we need to set job.state 915 job.state = final_job_state 916 return self.fail( "Job %s's output dataset(s) could not be read" % job.id ) 917 918 job_context = ExpressionContext( dict( stdout=job.stdout, stderr=job.stderr ) ) 919 for dataset_assoc in job.output_datasets + job.output_library_datasets: 920 context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset ) 921 #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur 922 for dataset in dataset_assoc.dataset.dataset.history_associations + dataset_assoc.dataset.dataset.library_associations: # need to update all associated output hdas, i.e. history was shared with job running 923 trynum = 0 924 while trynum < self.app.config.retry_job_output_collection: 925 try: 926 # Attempt to short circuit NFS attribute caching 927 os.stat( dataset.dataset.file_name ) 928 os.chown( dataset.dataset.file_name, os.getuid(), -1 ) 929 trynum = self.app.config.retry_job_output_collection 930 except ( OSError, ObjectNotFound ), e: 931 trynum += 1 932 log.warning( 'Error accessing %s, will retry: %s', dataset.dataset.file_name, e ) 933 time.sleep( 2 ) 934 dataset.blurb = 'done' 935 dataset.peek = 'no peek' 936 dataset.info = (dataset.info or '') 937 if context['stdout'].strip(): 938 #Ensure white space between entries 939 dataset.info = dataset.info.rstrip() + "\n" + context['stdout'].strip() 940 if context['stderr'].strip(): 941 #Ensure white space between entries 942 dataset.info = dataset.info.rstrip() + "\n" + context['stderr'].strip() 943 dataset.tool_version = self.version_string 944 dataset.set_size() 945 if 'uuid' in context: 946 dataset.dataset.uuid = context['uuid'] 947 # Update (non-library) job output datasets through the object store 948 if dataset not in job.output_library_datasets: 949 self.app.object_store.update_from_file(dataset.dataset, create=True) 950 if job.states.ERROR == final_job_state: 951 dataset.blurb = "error" 952 dataset.mark_unhidden() 953 elif dataset.has_data(): 954 # If the tool was expected to set the extension, attempt to retrieve it 955 if dataset.ext == 'auto': 956 dataset.extension = context.get( 'ext', 'data' ) 957 dataset.init_meta( copy_from=dataset ) 958 #if a dataset was copied, it won't appear in our dictionary: 959 #either use the metadata from originating output dataset, or call set_meta on the copies 960 #it would be quicker to just copy the metadata from the originating output dataset, 961 #but somewhat trickier (need to recurse up the copied_from tree), for now we'll call set_meta() 962 if ( not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and self.app.config.retry_metadata_internally ): 963 dataset.datatype.set_meta( dataset, overwrite=False ) # call datatype.set_meta directly for the initial set_meta call during dataset creation 964 elif not self.external_output_metadata.external_metadata_set_successfully( dataset, self.sa_session ) and job.states.ERROR != final_job_state: 965 dataset._state = model.Dataset.states.FAILED_METADATA 966 else: 967 #load metadata from file 968 #we need to no longer allow metadata to be edited while the job is still running, 969 #since if it is edited, the metadata changed on the running output will no longer match 970 #the metadata that was stored to disk for use via the external process, 971 #and the changes made by the user will be lost, without warning or notice 972 dataset.metadata.from_JSON_dict( self.external_output_metadata.get_output_filenames_by_dataset( dataset, self.sa_session ).filename_out ) 973 try: 974 assert context.get( 'line_count', None ) is not None 975 if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte: 976 dataset.set_peek( line_count=context['line_count'], is_multi_byte=True ) 977 else: 978 dataset.set_peek( line_count=context['line_count'] ) 979 except: 980 if ( not dataset.datatype.composite_type and dataset.dataset.is_multi_byte() ) or self.tool.is_multi_byte: 981 dataset.set_peek( is_multi_byte=True ) 982 else: 983 dataset.set_peek() 984 try: 985 # set the name if provided by the tool 986 dataset.name = context['name'] 987 except: 988 pass 989 else: 990 dataset.blurb = "empty" 991 if dataset.ext == 'auto': 992 dataset.extension = 'txt' 993 self.sa_session.add( dataset ) 994 if job.states.ERROR == final_job_state: 995 log.debug( "setting dataset state to ERROR" ) 996 # TODO: This is where the state is being set to error. Change it! 997 dataset_assoc.dataset.dataset.state = model.Dataset.states.ERROR 998 # Pause any dependent jobs (and those jobs' outputs) 999 for dep_job_assoc in dataset_assoc.dataset.dependent_jobs: 1000 self.pause( dep_job_assoc.job, "Execution of this dataset's job is paused because its input datasets are in an error state." ) 1001 else: 1002 dataset_assoc.dataset.dataset.state = model.Dataset.states.OK 1003 # If any of the rest of the finish method below raises an 1004 # exception, the fail method will run and set the datasets to 1005 # ERROR. The user will never see that the datasets are in error if 1006 # they were flushed as OK here, since upon doing so, the history 1007 # panel stops checking for updates. So allow the 1008 # self.sa_session.flush() at the bottom of this method set 1009 # the state instead. 1010 1011 for pja in job.post_job_actions: 1012 ActionBox.execute(self.app, self.sa_session, pja.post_job_action, job) 1013 # Flush all the dataset and job changes above. Dataset state changes 1014 # will now be seen by the user. 1015 self.sa_session.flush() 1016 # Save stdout and stderr 1017 if len( job.stdout ) > DATABASE_MAX_STRING_SIZE: 1018 log.info( "stdout for job …
Large files files are truncated, but you can click here to view the full file