/lib/galaxy/tools/__init__.py
Python | 3260 lines | 3210 code | 12 blank | 38 comment | 89 complexity | 766fca0cbb3c5eeef2edff23518a8b96 MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1""" 2Classes encapsulating galaxy tools and tool configuration. 3""" 4 5import binascii 6import glob 7import json 8import logging 9import os 10import pipes 11import re 12import shutil 13import sys 14import tempfile 15import threading 16import traceback 17import types 18import urllib 19 20from math import isinf 21 22from galaxy import eggs 23eggs.require( "MarkupSafe" ) # MarkupSafe must load before mako 24eggs.require( "Mako" ) 25eggs.require( "elementtree" ) 26eggs.require( "Paste" ) 27eggs.require( "SQLAlchemy >= 0.4" ) 28 29from cgi import FieldStorage 30from elementtree import ElementTree 31from mako.template import Template 32from paste import httpexceptions 33from sqlalchemy import and_ 34 35from galaxy import jobs, model 36from galaxy.jobs.error_level import StdioErrorLevel 37from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper 38from galaxy.jobs import ParallelismInfo 39from galaxy.tools.actions import DefaultToolAction 40from galaxy.tools.actions.data_source import DataSourceToolAction 41from galaxy.tools.actions.data_manager import DataManagerToolAction 42from galaxy.tools.deps import build_dependency_manager 43from galaxy.tools.deps.requirements import parse_requirements_from_xml 44from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings 45from galaxy.tools.parameters.basic import (BaseURLToolParameter, 46 DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter, 47 SelectToolParameter, ToolParameter, UnvalidatedValue, 48 IntegerToolParameter, FloatToolParameter) 49from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, UploadDataset 50from galaxy.tools.parameters.input_translation import ToolInputTranslator 51from galaxy.tools.parameters.output import ToolOutputActionGroup 52from galaxy.tools.parameters.validation import LateValidationError 53from galaxy.tools.filters import FilterFactory 54from galaxy.tools.test import parse_tests_elem 55from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string 56from galaxy.util.bunch import Bunch 57from galaxy.util.expressions import ExpressionContext 58from galaxy.util.hash_util import hmac_new 59from galaxy.util.none_like import NoneDataset 60from galaxy.util.odict import odict 61from galaxy.util.template import fill_template 62from galaxy.web import url_for 63from galaxy.web.form_builder import SelectField 64from galaxy.model.item_attrs import Dictifiable 65from galaxy.model import Workflow 66from tool_shed.util import shed_util_common as suc 67from .loader import load_tool, template_macro_params 68from .wrappers import ( 69 ToolParameterValueWrapper, 70 RawObjectWrapper, 71 LibraryDatasetValueWrapper, 72 InputValueWrapper, 73 SelectToolParameterWrapper, 74 DatasetFilenameWrapper, 75 DatasetListWrapper, 76) 77 78 79log = logging.getLogger( __name__ ) 80 81WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' ) 82 83 84class ToolNotFoundException( Exception ): 85 pass 86 87 88def to_dict_helper( obj, kwargs ): 89 """ Helper function that provides the appropriate kwargs to to_dict an object. """ 90 91 # Label.to_dict cannot have kwargs. 92 if isinstance( obj, ToolSectionLabel ): 93 kwargs = {} 94 95 return obj.to_dict( **kwargs ) 96 97 98class ToolBox( object, Dictifiable ): 99 """Container for a collection of tools""" 100 101 def __init__( self, config_filenames, tool_root_dir, app ): 102 """ 103 Create a toolbox from the config files named by `config_filenames`, using 104 `tool_root_dir` as the base directory for finding individual tool config files. 105 """ 106 # The shed_tool_confs list contains dictionaries storing information about the tools defined in each 107 # shed-related shed_tool_conf.xml file. 108 self.shed_tool_confs = [] 109 self.tools_by_id = {} 110 self.workflows_by_id = {} 111 # In-memory dictionary that defines the layout of the tool panel. 112 self.tool_panel = odict() 113 self.index = 0 114 self.data_manager_tools = odict() 115 # File that contains the XML section and tool tags from all tool panel config files integrated into a 116 # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator 117 # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. 118 self.integrated_tool_panel_config = os.path.join( app.config.root, 'integrated_tool_panel.xml' ) 119 # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. 120 self.integrated_tool_panel = odict() 121 self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0 122 if self.integrated_tool_panel_config_has_contents: 123 self.load_integrated_tool_panel_keys() 124 # The following refers to the tool_path config setting for backward compatibility. The shed-related 125 # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag. 126 self.tool_root_dir = tool_root_dir 127 self.app = app 128 self.filter_factory = FilterFactory( self ) 129 self.init_dependency_manager() 130 config_filenames = listify( config_filenames ) 131 for config_filename in config_filenames: 132 if os.path.isdir( config_filename ): 133 directory_contents = sorted( os.listdir( config_filename ) ) 134 directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ] 135 config_filenames.remove( config_filename ) 136 config_filenames.extend( directory_config_files ) 137 for config_filename in config_filenames: 138 try: 139 self.init_tools( config_filename ) 140 except: 141 log.exception( "Error loading tools defined in config %s", config_filename ) 142 if self.integrated_tool_panel_config_has_contents: 143 # Load self.tool_panel based on the order in self.integrated_tool_panel. 144 self.load_tool_panel() 145 if app.config.update_integrated_tool_panel: 146 # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. 147 # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel 148 # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel 149 # will be False when things like functional tests are the caller. 150 self.fix_integrated_tool_panel_dict() 151 self.write_integrated_tool_panel_config_file() 152 153 def fix_integrated_tool_panel_dict( self ): 154 # HACK: instead of fixing after the fact, I suggest some combination of: 155 # 1) adjusting init_tools() and called methods to get this right 156 # 2) redesigning the code and/or data structure used to read/write integrated_tool_panel.xml 157 for key, value in self.integrated_tool_panel.iteritems(): 158 if isinstance( value, ToolSection ): 159 for section_key, section_value in value.elems.iteritems(): 160 if section_value is None: 161 if isinstance( section_value, Tool ): 162 tool_id = section_key[5:] 163 value.elems[section_key] = self.tools_by_id.get( tool_id ) 164 elif isinstance( section_value, Workflow ): 165 workflow_id = section_key[9:] 166 value.elems[section_key] = self.workflows_by_id.get( workflow_id ) 167 168 def init_tools( self, config_filename ): 169 """ 170 Read the configuration file and load each tool. The following tags are currently supported: 171 172 .. raw:: xml 173 174 <toolbox> 175 <tool file="data_source/upload.xml"/> # tools outside sections 176 <label text="Basic Tools" id="basic_tools" /> # labels outside sections 177 <workflow id="529fd61ab1c6cc36" /> # workflows outside sections 178 <section name="Get Data" id="getext"> # sections 179 <tool file="data_source/biomart.xml" /> # tools inside sections 180 <label text="In Section" id="in_section" /> # labels inside sections 181 <workflow id="adb5f5c93f827949" /> # workflows inside sections 182 </section> 183 </toolbox> 184 185 """ 186 if self.app.config.get_bool( 'enable_tool_tags', False ): 187 log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" ) 188 self.sa_session.query( self.app.model.ToolTagAssociation ).delete() 189 self.sa_session.flush() 190 log.info( "Parsing the tool configuration %s" % config_filename ) 191 tree = parse_xml( config_filename ) 192 root = tree.getroot() 193 tool_path = root.get( 'tool_path' ) 194 if tool_path: 195 # We're parsing a shed_tool_conf file since we have a tool_path attribute. 196 parsing_shed_tool_conf = True 197 # Keep an in-memory list of xml elements to enable persistence of the changing tool config. 198 config_elems = [] 199 else: 200 parsing_shed_tool_conf = False 201 # Default to backward compatible config setting. 202 tool_path = self.tool_root_dir 203 # Only load the panel_dict under certain conditions. 204 load_panel_dict = not self.integrated_tool_panel_config_has_contents 205 for _, elem in enumerate( root ): 206 index = self.index 207 self.index += 1 208 if parsing_shed_tool_conf: 209 config_elems.append( elem ) 210 if elem.tag == 'tool': 211 self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index ) 212 elif elem.tag == 'workflow': 213 self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) 214 elif elem.tag == 'section': 215 self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index ) 216 elif elem.tag == 'label': 217 self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index ) 218 if parsing_shed_tool_conf: 219 shed_tool_conf_dict = dict( config_filename=config_filename, 220 tool_path=tool_path, 221 config_elems=config_elems ) 222 self.shed_tool_confs.append( shed_tool_conf_dict ) 223 224 def get_shed_config_dict_by_filename( self, filename, default=None ): 225 for shed_config_dict in self.shed_tool_confs: 226 if shed_config_dict[ 'config_filename' ] == filename: 227 return shed_config_dict 228 return default 229 230 def __add_tool_to_tool_panel( self, tool, panel_component, section=False ): 231 # See if a version of this tool is already loaded into the tool panel. The value of panel_component 232 # will be a ToolSection (if the value of section=True) or self.tool_panel (if section=False). 233 tool_id = str( tool.id ) 234 tool = self.tools_by_id[ tool_id ] 235 if section: 236 panel_dict = panel_component.elems 237 else: 238 panel_dict = panel_component 239 already_loaded = False 240 loaded_version_key = None 241 lineage_id = None 242 for lineage_id in tool.lineage_ids: 243 if lineage_id in self.tools_by_id: 244 loaded_version_key = 'tool_%s' % lineage_id 245 if loaded_version_key in panel_dict: 246 already_loaded = True 247 break 248 if already_loaded: 249 if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ): 250 key = 'tool_%s' % tool.id 251 index = panel_dict.keys().index( loaded_version_key ) 252 del panel_dict[ loaded_version_key ] 253 panel_dict.insert( index, key, tool ) 254 log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) ) 255 else: 256 inserted = False 257 key = 'tool_%s' % tool.id 258 # The value of panel_component is the in-memory tool panel dictionary. 259 for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): 260 if key == integrated_panel_key: 261 panel_dict.insert( index, key, tool ) 262 if not inserted: 263 inserted = True 264 if not inserted: 265 # Check the tool's installed versions. 266 for lineage_id in tool.lineage_ids: 267 lineage_id_key = 'tool_%s' % lineage_id 268 for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ): 269 if lineage_id_key == integrated_panel_key: 270 panel_dict.insert( index, key, tool ) 271 if not inserted: 272 inserted = True 273 if not inserted: 274 if tool.guid is None or \ 275 tool.tool_shed is None or \ 276 tool.repository_name is None or \ 277 tool.repository_owner is None or \ 278 tool.installed_changeset_revision is None: 279 # We have a tool that was not installed from the Tool Shed, but is also not yet defined in 280 # integrated_tool_panel.xml, so append it to the tool panel. 281 panel_dict[ key ] = tool 282 log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) ) 283 else: 284 # We are in the process of installing the tool. 285 tool_version = self.__get_tool_version( tool_id ) 286 tool_lineage_ids = tool_version.get_version_ids( self.app, reverse=True ) 287 for lineage_id in tool_lineage_ids: 288 if lineage_id in self.tools_by_id: 289 loaded_version_key = 'tool_%s' % lineage_id 290 if loaded_version_key in panel_dict: 291 if not already_loaded: 292 already_loaded = True 293 if not already_loaded: 294 # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel. 295 panel_dict[ key ] = tool 296 log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) ) 297 298 def load_tool_panel( self ): 299 for key, val in self.integrated_tool_panel.items(): 300 if isinstance( val, Tool ): 301 tool_id = key.replace( 'tool_', '', 1 ) 302 if tool_id in self.tools_by_id: 303 self.__add_tool_to_tool_panel( val, self.tool_panel, section=False ) 304 elif isinstance( val, Workflow ): 305 workflow_id = key.replace( 'workflow_', '', 1 ) 306 if workflow_id in self.workflows_by_id: 307 workflow = self.workflows_by_id[ workflow_id ] 308 self.tool_panel[ key ] = workflow 309 log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) 310 elif isinstance( val, ToolSectionLabel ): 311 self.tool_panel[ key ] = val 312 elif isinstance( val, ToolSection ): 313 elem = ElementTree.Element( 'section' ) 314 elem.attrib[ 'id' ] = val.id or '' 315 elem.attrib[ 'name' ] = val.name or '' 316 elem.attrib[ 'version' ] = val.version or '' 317 section = ToolSection( elem ) 318 log.debug( "Loading section: %s" % elem.get( 'name' ) ) 319 for section_key, section_val in val.elems.items(): 320 if isinstance( section_val, Tool ): 321 tool_id = section_key.replace( 'tool_', '', 1 ) 322 if tool_id in self.tools_by_id: 323 self.__add_tool_to_tool_panel( section_val, section, section=True ) 324 elif isinstance( section_val, Workflow ): 325 workflow_id = section_key.replace( 'workflow_', '', 1 ) 326 if workflow_id in self.workflows_by_id: 327 workflow = self.workflows_by_id[ workflow_id ] 328 section.elems[ section_key ] = workflow 329 log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) ) 330 elif isinstance( section_val, ToolSectionLabel ): 331 if section_val: 332 section.elems[ section_key ] = section_val 333 log.debug( "Loaded label: %s" % ( section_val.text ) ) 334 self.tool_panel[ key ] = section 335 336 def load_integrated_tool_panel_keys( self ): 337 """ 338 Load the integrated tool panel keys, setting values for tools and workflows to None. The values will 339 be reset when the various tool panel config files are parsed, at which time the tools and workflows are 340 loaded. 341 """ 342 tree = parse_xml( self.integrated_tool_panel_config ) 343 root = tree.getroot() 344 for elem in root: 345 if elem.tag == 'tool': 346 key = 'tool_%s' % elem.get( 'id' ) 347 self.integrated_tool_panel[ key ] = None 348 elif elem.tag == 'workflow': 349 key = 'workflow_%s' % elem.get( 'id' ) 350 self.integrated_tool_panel[ key ] = None 351 elif elem.tag == 'section': 352 section = ToolSection( elem ) 353 for section_elem in elem: 354 if section_elem.tag == 'tool': 355 key = 'tool_%s' % section_elem.get( 'id' ) 356 section.elems[ key ] = None 357 elif section_elem.tag == 'workflow': 358 key = 'workflow_%s' % section_elem.get( 'id' ) 359 section.elems[ key ] = None 360 elif section_elem.tag == 'label': 361 key = 'label_%s' % section_elem.get( 'id' ) 362 section.elems[ key ] = None 363 key = elem.get( 'id' ) 364 self.integrated_tool_panel[ key ] = section 365 elif elem.tag == 'label': 366 key = 'label_%s' % elem.get( 'id' ) 367 self.integrated_tool_panel[ key ] = None 368 369 def write_integrated_tool_panel_config_file( self ): 370 """ 371 Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators 372 use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. 373 """ 374 fd, filename = tempfile.mkstemp() 375 os.write( fd, '<?xml version="1.0"?>\n' ) 376 os.write( fd, '<toolbox>\n' ) 377 for key, item in self.integrated_tool_panel.items(): 378 if item: 379 if isinstance( item, Tool ): 380 os.write( fd, ' <tool id="%s" />\n' % item.id ) 381 elif isinstance( item, Workflow ): 382 os.write( fd, ' <workflow id="%s" />\n' % item.id ) 383 elif isinstance( item, ToolSectionLabel ): 384 label_id = item.id or '' 385 label_text = item.text or '' 386 label_version = item.version or '' 387 os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) 388 elif isinstance( item, ToolSection ): 389 section_id = item.id or '' 390 section_name = item.name or '' 391 section_version = item.version or '' 392 os.write( fd, ' <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) ) 393 for section_key, section_item in item.elems.items(): 394 if isinstance( section_item, Tool ): 395 if section_item: 396 os.write( fd, ' <tool id="%s" />\n' % section_item.id ) 397 elif isinstance( section_item, Workflow ): 398 if section_item: 399 os.write( fd, ' <workflow id="%s" />\n' % section_item.id ) 400 elif isinstance( section_item, ToolSectionLabel ): 401 if section_item: 402 label_id = section_item.id or '' 403 label_text = section_item.text or '' 404 label_version = section_item.version or '' 405 os.write( fd, ' <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) ) 406 os.write( fd, ' </section>\n' ) 407 os.write( fd, '</toolbox>\n' ) 408 os.close( fd ) 409 shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) ) 410 os.chmod( self.integrated_tool_panel_config, 0644 ) 411 412 def get_tool( self, tool_id, tool_version=None, get_all_versions=False ): 413 """Attempt to locate a tool in the tool box.""" 414 if tool_id in self.tools_by_id and not get_all_versions: 415 #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) 416 return self.tools_by_id[ tool_id ] 417 #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions 418 rval = [] 419 tv = self.__get_tool_version( tool_id ) 420 if tv: 421 tool_version_ids = tv.get_version_ids( self.app ) 422 for tool_version_id in tool_version_ids: 423 if tool_version_id in self.tools_by_id: 424 rval.append( self.tools_by_id[ tool_version_id ] ) 425 if not rval: 426 #still no tool, do a deeper search and try to match by old ids 427 for tool in self.tools_by_id.itervalues(): 428 if tool.old_id == tool_id: 429 rval.append( tool ) 430 if rval: 431 if get_all_versions: 432 return rval 433 else: 434 if tool_version: 435 #return first tool with matching version 436 for tool in rval: 437 if tool.version == tool_version: 438 return tool 439 #No tool matches by version, simply return the first available tool found 440 return rval[0] 441 #We now likely have a Toolshed guid passed in, but no supporting database entries 442 #If the tool exists by exact id and is loaded then provide exact match within a list 443 if tool_id in self.tools_by_id: 444 return[ self.tools_by_id[ tool_id ] ] 445 return None 446 447 def get_loaded_tools_by_lineage( self, tool_id ): 448 """Get all loaded tools associated by lineage to the tool whose id is tool_id.""" 449 tv = self.__get_tool_version( tool_id ) 450 if tv: 451 tool_version_ids = tv.get_version_ids( self.app ) 452 available_tool_versions = [] 453 for tool_version_id in tool_version_ids: 454 if tool_version_id in self.tools_by_id: 455 available_tool_versions.append( self.tools_by_id[ tool_version_id ] ) 456 return available_tool_versions 457 else: 458 if tool_id in self.tools_by_id: 459 tool = self.tools_by_id[ tool_id ] 460 return [ tool ] 461 return [] 462 463 def __get_tool_version( self, tool_id ): 464 """Return a ToolVersion if one exists for the tool_id""" 465 return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \ 466 .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \ 467 .first() 468 469 def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ): 470 return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \ 471 .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed, 472 self.app.install_model.ToolShedRepository.table.c.name == name, 473 self.app.install_model.ToolShedRepository.table.c.owner == owner, 474 self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \ 475 .first() 476 477 def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ): 478 """ 479 Retrieve all loaded versions of a tool from the toolbox and return a select list enabling 480 selection of a different version, the list of the tool's loaded versions, and the specified tool. 481 """ 482 toolbox = self 483 tool_version_select_field = None 484 tools = [] 485 tool = None 486 # Backwards compatibility for datasource tools that have default tool_id configured, but which 487 # are now using only GALAXY_URL. 488 tool_ids = listify( tool_id ) 489 for tool_id in tool_ids: 490 if get_loaded_tools_by_lineage: 491 tools = toolbox.get_loaded_tools_by_lineage( tool_id ) 492 else: 493 tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True ) 494 if tools: 495 tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False ) 496 if len( tools ) > 1: 497 tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected ) 498 break 499 return tool_version_select_field, tools, tool 500 501 def build_tool_version_select_field( self, tools, tool_id, set_selected ): 502 """Build a SelectField whose options are the ids for the received list of tools.""" 503 options = [] 504 refresh_on_change_values = [] 505 for tool in tools: 506 options.insert( 0, ( tool.version, tool.id ) ) 507 refresh_on_change_values.append( tool.id ) 508 select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values ) 509 for option_tup in options: 510 selected = set_selected and option_tup[ 1 ] == tool_id 511 if selected: 512 select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True ) 513 else: 514 select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] ) 515 return select_field 516 517 def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ): 518 try: 519 path = elem.get( "file" ) 520 repository_id = None 521 if guid is None: 522 tool_shed_repository = None 523 can_load_into_panel_dict = True 524 else: 525 # The tool is contained in an installed tool shed repository, so load 526 # the tool only if the repository has not been marked deleted. 527 tool_shed = elem.find( "tool_shed" ).text 528 repository_name = elem.find( "repository_name" ).text 529 repository_owner = elem.find( "repository_owner" ).text 530 installed_changeset_revision_elem = elem.find( "installed_changeset_revision" ) 531 if installed_changeset_revision_elem is None: 532 # Backward compatibility issue - the tag used to be named 'changeset_revision'. 533 installed_changeset_revision_elem = elem.find( "changeset_revision" ) 534 installed_changeset_revision = installed_changeset_revision_elem.text 535 tool_shed_repository = self.__get_tool_shed_repository( tool_shed, repository_name, repository_owner, installed_changeset_revision ) 536 if tool_shed_repository: 537 # Only load tools if the repository is not deactivated or uninstalled. 538 can_load_into_panel_dict = not tool_shed_repository.deleted 539 repository_id = self.app.security.encode_id( tool_shed_repository.id ) 540 else: 541 # If there is not yet a tool_shed_repository record, we're in the process of installing 542 # a new repository, so any included tools can be loaded into the tool panel. 543 can_load_into_panel_dict = True 544 tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id ) 545 key = 'tool_%s' % str( tool.id ) 546 if can_load_into_panel_dict: 547 if guid is not None: 548 tool.tool_shed = tool_shed 549 tool.repository_name = repository_name 550 tool.repository_owner = repository_owner 551 tool.installed_changeset_revision = installed_changeset_revision 552 tool.guid = guid 553 tool.version = elem.find( "version" ).text 554 # Make sure the tool has a tool_version. 555 if not self.__get_tool_version( tool.id ): 556 tool_version = self.app.install_model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository ) 557 self.app.install_model.context.add( tool_version ) 558 self.app.install_model.context.flush() 559 # Load the tool's lineage ids. 560 tool.lineage_ids = tool.tool_version.get_version_ids( self.app ) 561 if self.app.config.get_bool( 'enable_tool_tags', False ): 562 tag_names = elem.get( "tags", "" ).split( "," ) 563 for tag_name in tag_names: 564 if tag_name == '': 565 continue 566 tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first() 567 if not tag: 568 tag = self.app.model.Tag( name=tag_name ) 569 self.sa_session.add( tag ) 570 self.sa_session.flush() 571 tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) 572 self.sa_session.add( tta ) 573 self.sa_session.flush() 574 else: 575 for tagged_tool in tag.tagged_tools: 576 if tagged_tool.tool_id == tool.id: 577 break 578 else: 579 tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id ) 580 self.sa_session.add( tta ) 581 self.sa_session.flush() 582 # Allow for the same tool to be loaded into multiple places in the tool panel. We have to handle 583 # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy 584 # administrator has retrieved updates to the installed repository. In this case, the tool may have 585 # been updated, but the version was not changed, so the tool should always be reloaded here. We used 586 # to only load the tool if it's it was not found in self.tools_by_id, but performing that check did 587 # not enable this scenario. 588 self.tools_by_id[ tool.id ] = tool 589 if load_panel_dict: 590 self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) ) 591 # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file. 592 if key in integrated_panel_dict or index is None: 593 integrated_panel_dict[ key ] = tool 594 else: 595 integrated_panel_dict.insert( index, key, tool ) 596 except: 597 log.exception( "Error reading tool from path: %s" % path ) 598 599 def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): 600 try: 601 # TODO: should id be encoded? 602 workflow_id = elem.get( 'id' ) 603 workflow = self.load_workflow( workflow_id ) 604 self.workflows_by_id[ workflow_id ] = workflow 605 key = 'workflow_' + workflow_id 606 if load_panel_dict: 607 panel_dict[ key ] = workflow 608 # Always load workflows into the integrated_panel_dict. 609 if key in integrated_panel_dict or index is None: 610 integrated_panel_dict[ key ] = workflow 611 else: 612 integrated_panel_dict.insert( index, key, workflow ) 613 except: 614 log.exception( "Error loading workflow: %s" % workflow_id ) 615 616 def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ): 617 label = ToolSectionLabel( elem ) 618 key = 'label_' + label.id 619 if load_panel_dict: 620 panel_dict[ key ] = label 621 if key in integrated_panel_dict or index is None: 622 integrated_panel_dict[ key ] = label 623 else: 624 integrated_panel_dict.insert( index, key, label ) 625 626 def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ): 627 key = elem.get( "id" ) 628 if key in self.tool_panel: 629 section = self.tool_panel[ key ] 630 elems = section.elems 631 else: 632 section = ToolSection( elem ) 633 elems = section.elems 634 if key in self.integrated_tool_panel: 635 integrated_section = self.integrated_tool_panel[ key ] 636 integrated_elems = integrated_section.elems 637 else: 638 integrated_section = ToolSection( elem ) 639 integrated_elems = integrated_section.elems 640 for sub_index, sub_elem in enumerate( elem ): 641 if sub_elem.tag == 'tool': 642 self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index ) 643 elif sub_elem.tag == 'workflow': 644 self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) 645 elif sub_elem.tag == 'label': 646 self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index ) 647 if load_panel_dict: 648 self.tool_panel[ key ] = section 649 # Always load sections into the integrated_tool_panel. 650 if key in self.integrated_tool_panel or index is None: 651 self.integrated_tool_panel[ key ] = integrated_section 652 else: 653 self.integrated_tool_panel.insert( index, key, integrated_section ) 654 655 def load_tool( self, config_file, guid=None, repository_id=None, **kwds ): 656 """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" 657 # Parse XML configuration file and get the root element 658 tree = load_tool( config_file ) 659 root = tree.getroot() 660 # Allow specifying a different tool subclass to instantiate 661 if root.find( "type" ) is not None: 662 type_elem = root.find( "type" ) 663 module = type_elem.get( 'module', 'galaxy.tools' ) 664 cls = type_elem.get( 'class' ) 665 mod = __import__( module, globals(), locals(), [cls] ) 666 ToolClass = getattr( mod, cls ) 667 elif root.get( 'tool_type', None ) is not None: 668 ToolClass = tool_types.get( root.get( 'tool_type' ) ) 669 else: 670 ToolClass = Tool 671 return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds ) 672 673 def reload_tool_by_id( self, tool_id ): 674 """ 675 Attempt to reload the tool identified by 'tool_id', if successful 676 replace the old tool. 677 """ 678 if tool_id not in self.tools_by_id: 679 message = "No tool with id %s" % tool_id 680 status = 'error' 681 else: 682 old_tool = self.tools_by_id[ tool_id ] 683 new_tool = self.load_tool( old_tool.config_file ) 684 # The tool may have been installed from a tool shed, so set the tool shed attributes. 685 # Since the tool version may have changed, we don't override it here. 686 new_tool.id = old_tool.id 687 new_tool.guid = old_tool.guid 688 new_tool.tool_shed = old_tool.tool_shed 689 new_tool.repository_name = old_tool.repository_name 690 new_tool.repository_owner = old_tool.repository_owner 691 new_tool.installed_changeset_revision = old_tool.installed_changeset_revision 692 new_tool.old_id = old_tool.old_id 693 # Replace old_tool with new_tool in self.tool_panel 694 tool_key = 'tool_' + tool_id 695 for key, val in self.tool_panel.items(): 696 if key == tool_key: 697 self.tool_panel[ key ] = new_tool 698 break 699 elif key.startswith( 'section' ): 700 if tool_key in val.elems: 701 self.tool_panel[ key ].elems[ tool_key ] = new_tool 702 break 703 self.tools_by_id[ tool_id ] = new_tool 704 message = "Reloaded the tool:<br/>" 705 message += "<b>name:</b> %s<br/>" % old_tool.name 706 message += "<b>id:</b> %s<br/>" % old_tool.id 707 message += "<b>version:</b> %s" % old_tool.version 708 status = 'done' 709 return message, status 710 711 def remove_tool_by_id( self, tool_id ): 712 """ 713 Attempt to remove the tool identified by 'tool_id'. 714 """ 715 if tool_id not in self.tools_by_id: 716 message = "No tool with id %s" % tool_id 717 status = 'error' 718 else: 719 tool = self.tools_by_id[ tool_id ] 720 del self.tools_by_id[ tool_id ] 721 tool_key = 'tool_' + tool_id 722 for key, val in self.tool_panel.items(): 723 if key == tool_key: 724 del self.tool_panel[ key ] 725 break 726 elif key.startswith( 'section' ): 727 if tool_key in val.elems: 728 del self.tool_panel[ key ].elems[ tool_key ] 729 break 730 if tool_id in self.data_manager_tools: 731 del self.data_manager_tools[ tool_id ] 732 #TODO: do we need to manually remove from the integrated panel here? 733 message = "Removed the tool:<br/>" 734 message += "<b>name:</b> %s<br/>" % tool.name 735 message += "<b>id:</b> %s<br/>" % tool.id 736 message += "<b>version:</b> %s" % tool.version 737 status = 'done' 738 return message, status 739 740 def load_workflow( self, workflow_id ): 741 """ 742 Return an instance of 'Workflow' identified by `id`, 743 which is encoded in the tool panel. 744 """ 745 id = self.app.security.decode_id( workflow_id ) 746 stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id ) 747 return stored.latest_workflow 748 749 def init_dependency_manager( self ): 750 self.dependency_manager = build_dependency_manager( self.app.config ) 751 752 @property 753 def sa_session( self ): 754 """ 755 Returns a SQLAlchemy session 756 """ 757 return self.app.model.context 758 759 def to_dict( self, trans, in_panel=True, **kwds ): 760 """ 761 to_dict toolbox. 762 """ 763 764 context = Bunch( toolbox=self, trans=trans, **kwds ) 765 if in_panel: 766 panel_elts = [ val for val in self.tool_panel.itervalues() ] 767 768 filters = self.filter_factory.build_filters( trans, **kwds ) 769 770 filtered_panel_elts = [] 771 for index, elt in enumerate( panel_elts ): 772 elt = _filter_for_panel( elt, filters, context ) 773 if elt: 774 filtered_panel_elts.append( elt ) 775 panel_elts = filtered_panel_elts 776 777 # Produce panel. 778 rval = [] 779 kwargs = dict( 780 trans=trans, 781 link_details=True 782 ) 783 for elt in panel_elts: 784 rval.append( to_dict_helper( elt, kwargs ) ) 785 else: 786 tools = [] 787 for id, tool in self.tools_by_id.items(): 788 tools.append( tool.to_dict( trans, link_details=True ) ) 789 rval = tools 790 791 return rval 792 793 794def _filter_for_panel( item, filters, context ): 795 """ 796 Filters tool panel elements so that only those that are compatible 797 with provided filters are kept. 798 """ 799 def _apply_filter( filter_item, filter_list ): 800 for filter_method in filter_list: 801 if not filter_method( context, filter_item ): 802 return False 803 return True 804 if isinstance( item, Tool ): 805 if _apply_filter( item, filters[ 'tool' ] ): 806 return item 807 elif isinstance( item, ToolSectionLabel ): 808 if _apply_filter( item, filters[ 'label' ] ): 809 return item 810 elif isinstance( item, ToolSection ): 811 # Filter section item-by-item. Only show a label if there are 812 # non-filtered tools below it. 813 814 if _apply_filter( item, filters[ 'section' ] ): 815 cur_label_key = None 816 tools_under_label = False 817 filtered_elems = item.elems.copy() 818 for key, section_item in item.elems.items(): 819 if isinstance( section_item, Tool ): 820 # Filter tool. 821 if _apply_filter( section_item, filters[ 'tool' ] ): 822 tools_under_label = True 823 else: 824 del filtered_elems[ key ] 825 elif isinstance( section_item, ToolSectionLabel ): 826 # If there is a label and it does not have tools, 827 # remove it. 828 if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ): 829 del filtered_elems[ cur_label_key ] 830 831 # Reset attributes for new label. 832 cur_label_key = key 833 tools_under_label = False 834 835 # Handle last label. 836 if cur_label_key and not tools_under_label: 837 del filtered_elems[ cur_label_key ] 838 839 # Only return section if there are elements. 840 if len( filtered_elems ) != 0: 841 copy = item.copy() 842 copy.elems = filtered_elems 843 return copy 844 845 return None 846 847 848class ToolSection( object, Dictifiable ): 849 """ 850 A group of tools with similar type/purpose that will be displayed as a 851 group in the user interface. 852 """ 853 854 dict_collection_visible_keys = ( 'id', 'name', 'version' ) 855 856 def __init__( self, elem=None ): 857 f = lambda elem, val: elem is not None and elem.get( val ) or '' 858 self.name = f( elem, 'name' ) 859 self.id = f( elem, 'id' ) 860 self.version = f( elem, 'version' ) 861 self.elems = odict() 862 863 def copy( self ): 864 copy = ToolSection() 865 copy.name = self.name 866 copy.id = self.id 867 copy.version = self.version 868 copy.elems = self.elems.copy() 869 return copy 870 871 def to_dict( self, trans, link_details=False ): 872 """ Return a dict that includes section's attributes. """ 873 874 section_dict = super( ToolSection, self ).to_dict() 875 section_elts = [] 876 kwargs = dict( 877 trans=trans, 878 link_details=link_details 879 ) 880 for elt in self.elems.values(): 881 section_elts.append( to_dict_helper( elt, kwargs ) ) 882 section_dict[ 'elems' ] = section_elts 883 884 return section_dict 885 886 887class ToolSectionLabel( object, Dictifiable ): 888 """ 889 A label for a set of tools that can be displayed above groups of tools 890 and sections in the user interface 891 """ 892 893 dict_collection_visible_keys = ( 'id', 'text', 'version' ) 894 895 def __init__( self, elem ): 896 self.text = elem.get( "text" ) 897 self.id = elem.get( "id" ) 898 self.version = elem.get( "version" ) or '' 899 900 901class DefaultToolState( object ): 902 """ 903 Keeps track of the state of a users interaction with a tool between 904 requests. The default tool state keeps track of the current page (for 905 multipage "wizard" tools) and the values of all 906 """ 907 def __init__( self ): 908 self.page = 0 909 self.rerun_remap_job_id = None 910 self.inputs = None 911 912 def encode( self, tool, app, secure=True ): 913 """ 914 Convert the data to a string 915 """ 916 # Convert parameters to a dictionary of strings, and save curent 917 # page in that dict 918 value = params_to_strings( tool.inputs, self.inputs, app ) 919 value["__page__"] = self.page 920 value["__rerun_remap_job_id__"] = self.rerun_remap_job_id 921 value = json.dumps( value ) 922 # Make it secure 923 if secure: 924 a = hmac_new( app.config.tool_secret, value ) 925 b = binascii.hexlify( value ) 926 return "%s:%s" % ( a, b ) 927 else: 928 return value 929 930 def decode( self, value, tool, app, secure=True ): 931 """ 932 Restore the state from a string 933 """ 934 if secure: 935 # Extract and verify hash 936 a, b = value.split( ":" ) 937 value = binascii.unhexlify( b ) 938 test = hmac_new( app.config.tool_secret, value ) 939 assert a == test 940 # Restore from string 941 values = json_fix( json.loads( value ) ) 942 self.page = values.pop( "__page__" ) 943 if '__rerun_remap_job_id__' in values: 944 self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" ) 945 else: 946 self.rerun_remap_job_id = None 947 self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True ) 948 949 950class ToolOutput( object, Dictifiable ): 951 """ 952 Represents an output datasets produced by a tool. For backward 953 compatibility this behaves as if it were the tuple:: 954 955 (format, metadata_source, parent) 956 """ 957 958 dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' ) 959 960 def __init__( self, name, format=None, format_source=None, metadata_source=None, 961 parent=None, label=None, filters=None, actions=None, hidden=False ): 962 self.name = name 963 self.format = format 964 self.format_source = format_source 965 self.metadata_source = metadata_source 966 self.parent = parent 967 self.label = label 968 self.filters = filters or [] 969 self.actions = actions 970 self.hidden = hidden 971 972 # Tuple emulation 973 974 def __len__( self ): 975 return 3 976 977 def __getitem__( self, index ): 978 if index == 0: 979 return self.format 980 elif index == 1: 981 return self.metadata_source 982 elif index == 2: 983 return self.parent 984 else: 985 raise IndexError( index ) 986 987 def __iter__( self ): 988 return iter( ( self.format, self.metadata_source, self.parent ) ) 989 990 991class Tool( object, Dictifiable ): 992 """ 993 Represents a computational tool that can be executed through Galaxy. 994 """ 995 996 tool_type = 'default' 997 requires_setting_metadata = True 998 default_tool_action = DefaultToolAction 999 dict_collection_visible_keys = ( 'id', 'name', 'version', 'description' ) 1000 1001 def __init__( self, config_file, root, app, guid=None, repository_id=None ): 1002 """Load a tool from the config named by `config_file`""" 1003 # Determine the full path of the directory where the tool config is 1004 self.config_file = config_file 1005 self.tool_dir = os.path.dirname( config_file ) 1006 self.app = app 1007 self.repository_id = repository_id 1008 #setup initial attribute values 1009 self.inputs = odict() 1010 self.stdio_exit_codes = list() 1011 self.stdio_regexes = list() 1012 self.inputs_by_page = list() 1013 self.display_by_page = list() 1014 self.action = '/tool_runner/index' 1015 self.target = 'galaxy_main' 1016 self.method = 'post' 1017 self.check_values = True 1018 self.nginx_upload = False 1019 self.input_required = False 1020 self.display_interface = True 1021 self.require_login = False 1022 self.rerun = False 1023 # Define a place to keep track of all input These 1024 # differ from the inputs dictionary in that inputs can be page 1025 # elements like conditionals, but input_params are basic form 1026 # parameters like SelectField objects. This enables us to more 1027 # easily ensure that parameter dependencies like index files or 1028 # tool_data_table_conf.xml entries exist. 1029 self.input_params = [] 1030 # Attributes of tools installed from Galaxy tool sheds. 1031 self.tool_shed = None 1032 self.repository_name = None 1033 self.repository_owner = None 1034 self.installed_changeset_revision = None 1035 # The tool.id value will be the value of guid, but we'll keep the 1036 # guid attribute since it is useful to have. 1037 self.guid = guid 1038 self…
Large files files are truncated, but you can click here to view the full file