PageRenderTime 100ms CodeModel.GetById 21ms app.highlight 62ms RepoModel.GetById 1ms app.codeStats 1ms

/lib/galaxy/tools/__init__.py

https://bitbucket.org/cistrome/cistrome-harvard/
Python | 3260 lines | 3210 code | 12 blank | 38 comment | 89 complexity | 766fca0cbb3c5eeef2edff23518a8b96 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1"""
   2Classes encapsulating galaxy tools and tool configuration.
   3"""
   4
   5import binascii
   6import glob
   7import json
   8import logging
   9import os
  10import pipes
  11import re
  12import shutil
  13import sys
  14import tempfile
  15import threading
  16import traceback
  17import types
  18import urllib
  19
  20from math import isinf
  21
  22from galaxy import eggs
  23eggs.require( "MarkupSafe" )  # MarkupSafe must load before mako
  24eggs.require( "Mako" )
  25eggs.require( "elementtree" )
  26eggs.require( "Paste" )
  27eggs.require( "SQLAlchemy >= 0.4" )
  28
  29from cgi import FieldStorage
  30from elementtree import ElementTree
  31from mako.template import Template
  32from paste import httpexceptions
  33from sqlalchemy import and_
  34
  35from galaxy import jobs, model
  36from galaxy.jobs.error_level import StdioErrorLevel
  37from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
  38from galaxy.jobs import ParallelismInfo
  39from galaxy.tools.actions import DefaultToolAction
  40from galaxy.tools.actions.data_source import DataSourceToolAction
  41from galaxy.tools.actions.data_manager import DataManagerToolAction
  42from galaxy.tools.deps import build_dependency_manager
  43from galaxy.tools.deps.requirements import parse_requirements_from_xml
  44from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings
  45from galaxy.tools.parameters.basic import (BaseURLToolParameter,
  46                                           DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter,
  47                                           SelectToolParameter, ToolParameter, UnvalidatedValue,
  48                                           IntegerToolParameter, FloatToolParameter)
  49from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, UploadDataset
  50from galaxy.tools.parameters.input_translation import ToolInputTranslator
  51from galaxy.tools.parameters.output import ToolOutputActionGroup
  52from galaxy.tools.parameters.validation import LateValidationError
  53from galaxy.tools.filters import FilterFactory
  54from galaxy.tools.test import parse_tests_elem
  55from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string
  56from galaxy.util.bunch import Bunch
  57from galaxy.util.expressions import ExpressionContext
  58from galaxy.util.hash_util import hmac_new
  59from galaxy.util.none_like import NoneDataset
  60from galaxy.util.odict import odict
  61from galaxy.util.template import fill_template
  62from galaxy.web import url_for
  63from galaxy.web.form_builder import SelectField
  64from galaxy.model.item_attrs import Dictifiable
  65from galaxy.model import Workflow
  66from tool_shed.util import shed_util_common as suc
  67from .loader import load_tool, template_macro_params
  68from .wrappers import (
  69    ToolParameterValueWrapper,
  70    RawObjectWrapper,
  71    LibraryDatasetValueWrapper,
  72    InputValueWrapper,
  73    SelectToolParameterWrapper,
  74    DatasetFilenameWrapper,
  75    DatasetListWrapper,
  76)
  77
  78
  79log = logging.getLogger( __name__ )
  80
  81WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' )
  82
  83
  84class ToolNotFoundException( Exception ):
  85    pass
  86
  87
  88def to_dict_helper( obj, kwargs ):
  89    """ Helper function that provides the appropriate kwargs to to_dict an object. """
  90
  91    # Label.to_dict cannot have kwargs.
  92    if isinstance( obj, ToolSectionLabel ):
  93        kwargs = {}
  94
  95    return obj.to_dict( **kwargs )
  96
  97
  98class ToolBox( object, Dictifiable ):
  99    """Container for a collection of tools"""
 100
 101    def __init__( self, config_filenames, tool_root_dir, app ):
 102        """
 103        Create a toolbox from the config files named by `config_filenames`, using
 104        `tool_root_dir` as the base directory for finding individual tool config files.
 105        """
 106        # The shed_tool_confs list contains dictionaries storing information about the tools defined in each
 107        # shed-related shed_tool_conf.xml file.
 108        self.shed_tool_confs = []
 109        self.tools_by_id = {}
 110        self.workflows_by_id = {}
 111        # In-memory dictionary that defines the layout of the tool panel.
 112        self.tool_panel = odict()
 113        self.index = 0
 114        self.data_manager_tools = odict()
 115        # File that contains the XML section and tool tags from all tool panel config files integrated into a
 116        # single file that defines the tool panel layout.  This file can be changed by the Galaxy administrator
 117        # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel.
 118        self.integrated_tool_panel_config = os.path.join( app.config.root, 'integrated_tool_panel.xml' )
 119        # In-memory dictionary that defines the layout of the tool_panel.xml file on disk.
 120        self.integrated_tool_panel = odict()
 121        self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0
 122        if self.integrated_tool_panel_config_has_contents:
 123            self.load_integrated_tool_panel_keys()
 124        # The following refers to the tool_path config setting for backward compatibility.  The shed-related
 125        # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
 126        self.tool_root_dir = tool_root_dir
 127        self.app = app
 128        self.filter_factory = FilterFactory( self )
 129        self.init_dependency_manager()
 130        config_filenames = listify( config_filenames )
 131        for config_filename in config_filenames:
 132            if os.path.isdir( config_filename ):
 133                directory_contents = sorted( os.listdir( config_filename ) )
 134                directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ]
 135                config_filenames.remove( config_filename )
 136                config_filenames.extend( directory_config_files )
 137        for config_filename in config_filenames:
 138            try:
 139                self.init_tools( config_filename )
 140            except:
 141                log.exception( "Error loading tools defined in config %s", config_filename )
 142        if self.integrated_tool_panel_config_has_contents:
 143            # Load self.tool_panel based on the order in self.integrated_tool_panel.
 144            self.load_tool_panel()
 145        if app.config.update_integrated_tool_panel:
 146            # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
 147            # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
 148            # config files, adding or removing locally developed tools or workflows.  The value of integrated_tool_panel
 149            # will be False when things like functional tests are the caller.
 150            self.fix_integrated_tool_panel_dict()
 151            self.write_integrated_tool_panel_config_file()
 152
 153    def fix_integrated_tool_panel_dict( self ):
 154        # HACK: instead of fixing after the fact, I suggest some combination of:
 155        #  1) adjusting init_tools() and called methods to get this right
 156        #  2) redesigning the code and/or data structure used to read/write integrated_tool_panel.xml
 157        for key, value in self.integrated_tool_panel.iteritems():
 158            if isinstance( value, ToolSection ):
 159                for section_key, section_value in value.elems.iteritems():
 160                    if section_value is None:
 161                        if isinstance( section_value, Tool ):
 162                            tool_id = section_key[5:]
 163                            value.elems[section_key] = self.tools_by_id.get( tool_id )
 164                        elif isinstance( section_value, Workflow ):
 165                            workflow_id = section_key[9:]
 166                            value.elems[section_key] = self.workflows_by_id.get( workflow_id )
 167
 168    def init_tools( self, config_filename ):
 169        """
 170        Read the configuration file and load each tool.  The following tags are currently supported:
 171
 172        .. raw:: xml
 173
 174            <toolbox>
 175                <tool file="data_source/upload.xml"/>            # tools outside sections
 176                <label text="Basic Tools" id="basic_tools" />    # labels outside sections
 177                <workflow id="529fd61ab1c6cc36" />               # workflows outside sections
 178                <section name="Get Data" id="getext">            # sections
 179                    <tool file="data_source/biomart.xml" />      # tools inside sections
 180                    <label text="In Section" id="in_section" />  # labels inside sections
 181                    <workflow id="adb5f5c93f827949" />           # workflows inside sections
 182                </section>
 183            </toolbox>
 184
 185        """
 186        if self.app.config.get_bool( 'enable_tool_tags', False ):
 187            log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" )
 188            self.sa_session.query( self.app.model.ToolTagAssociation ).delete()
 189            self.sa_session.flush()
 190        log.info( "Parsing the tool configuration %s" % config_filename )
 191        tree = parse_xml( config_filename )
 192        root = tree.getroot()
 193        tool_path = root.get( 'tool_path' )
 194        if tool_path:
 195            # We're parsing a shed_tool_conf file since we have a tool_path attribute.
 196            parsing_shed_tool_conf = True
 197            # Keep an in-memory list of xml elements to enable persistence of the changing tool config.
 198            config_elems = []
 199        else:
 200            parsing_shed_tool_conf = False
 201            # Default to backward compatible config setting.
 202            tool_path = self.tool_root_dir
 203        # Only load the panel_dict under certain conditions.
 204        load_panel_dict = not self.integrated_tool_panel_config_has_contents
 205        for _, elem in enumerate( root ):
 206            index = self.index
 207            self.index += 1
 208            if parsing_shed_tool_conf:
 209                config_elems.append( elem )
 210            if elem.tag == 'tool':
 211                self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index )
 212            elif elem.tag == 'workflow':
 213                self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
 214            elif elem.tag == 'section':
 215                self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
 216            elif elem.tag == 'label':
 217                self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
 218        if parsing_shed_tool_conf:
 219            shed_tool_conf_dict = dict( config_filename=config_filename,
 220                                        tool_path=tool_path,
 221                                        config_elems=config_elems )
 222            self.shed_tool_confs.append( shed_tool_conf_dict )
 223
 224    def get_shed_config_dict_by_filename( self, filename, default=None ):
 225        for shed_config_dict in self.shed_tool_confs:
 226            if shed_config_dict[ 'config_filename' ] == filename:
 227                return shed_config_dict
 228        return default
 229
 230    def __add_tool_to_tool_panel( self, tool, panel_component, section=False ):
 231        # See if a version of this tool is already loaded into the tool panel.  The value of panel_component
 232        # will be a ToolSection (if the value of section=True) or self.tool_panel (if section=False).
 233        tool_id = str( tool.id )
 234        tool = self.tools_by_id[ tool_id ]
 235        if section:
 236            panel_dict = panel_component.elems
 237        else:
 238            panel_dict = panel_component
 239        already_loaded = False
 240        loaded_version_key = None
 241        lineage_id = None
 242        for lineage_id in tool.lineage_ids:
 243            if lineage_id in self.tools_by_id:
 244                loaded_version_key = 'tool_%s' % lineage_id
 245                if loaded_version_key in panel_dict:
 246                    already_loaded = True
 247                    break
 248        if already_loaded:
 249            if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ):
 250                key = 'tool_%s' % tool.id
 251                index = panel_dict.keys().index( loaded_version_key )
 252                del panel_dict[ loaded_version_key ]
 253                panel_dict.insert( index, key, tool )
 254                log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) )
 255        else:
 256            inserted = False
 257            key = 'tool_%s' % tool.id
 258            # The value of panel_component is the in-memory tool panel dictionary.
 259            for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ):
 260                if key == integrated_panel_key:
 261                    panel_dict.insert( index, key, tool )
 262                    if not inserted:
 263                        inserted = True
 264            if not inserted:
 265                # Check the tool's installed versions.
 266                for lineage_id in tool.lineage_ids:
 267                    lineage_id_key = 'tool_%s' % lineage_id
 268                    for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ):
 269                        if lineage_id_key == integrated_panel_key:
 270                            panel_dict.insert( index, key, tool )
 271                            if not inserted:
 272                                inserted = True
 273                if not inserted:
 274                    if tool.guid is None or \
 275                        tool.tool_shed is None or \
 276                        tool.repository_name is None or \
 277                        tool.repository_owner is None or \
 278                        tool.installed_changeset_revision is None:
 279                        # We have a tool that was not installed from the Tool Shed, but is also not yet defined in
 280                        # integrated_tool_panel.xml, so append it to the tool panel.
 281                        panel_dict[ key ] = tool
 282                        log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) )
 283                    else:
 284                        # We are in the process of installing the tool.
 285                        tool_version = self.__get_tool_version( tool_id )
 286                        tool_lineage_ids = tool_version.get_version_ids( self.app, reverse=True )
 287                        for lineage_id in tool_lineage_ids:
 288                            if lineage_id in self.tools_by_id:
 289                                loaded_version_key = 'tool_%s' % lineage_id
 290                                if loaded_version_key in panel_dict:
 291                                    if not already_loaded:
 292                                        already_loaded = True
 293                        if not already_loaded:
 294                            # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel.
 295                            panel_dict[ key ] = tool
 296                            log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) )
 297
 298    def load_tool_panel( self ):
 299        for key, val in self.integrated_tool_panel.items():
 300            if isinstance( val, Tool ):
 301                tool_id = key.replace( 'tool_', '', 1 )
 302                if tool_id in self.tools_by_id:
 303                    self.__add_tool_to_tool_panel( val, self.tool_panel, section=False )
 304            elif isinstance( val, Workflow ):
 305                workflow_id = key.replace( 'workflow_', '', 1 )
 306                if workflow_id in self.workflows_by_id:
 307                    workflow = self.workflows_by_id[ workflow_id ]
 308                    self.tool_panel[ key ] = workflow
 309                    log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
 310            elif isinstance( val, ToolSectionLabel ):
 311                self.tool_panel[ key ] = val
 312            elif isinstance( val, ToolSection ):
 313                elem = ElementTree.Element( 'section' )
 314                elem.attrib[ 'id' ] = val.id or ''
 315                elem.attrib[ 'name' ] = val.name or ''
 316                elem.attrib[ 'version' ] = val.version or ''
 317                section = ToolSection( elem )
 318                log.debug( "Loading section: %s" % elem.get( 'name' ) )
 319                for section_key, section_val in val.elems.items():
 320                    if isinstance( section_val, Tool ):
 321                        tool_id = section_key.replace( 'tool_', '', 1 )
 322                        if tool_id in self.tools_by_id:
 323                            self.__add_tool_to_tool_panel( section_val, section, section=True )
 324                    elif isinstance( section_val, Workflow ):
 325                        workflow_id = section_key.replace( 'workflow_', '', 1 )
 326                        if workflow_id in self.workflows_by_id:
 327                            workflow = self.workflows_by_id[ workflow_id ]
 328                            section.elems[ section_key ] = workflow
 329                            log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
 330                    elif isinstance( section_val, ToolSectionLabel ):
 331                        if section_val:
 332                            section.elems[ section_key ] = section_val
 333                            log.debug( "Loaded label: %s" % ( section_val.text ) )
 334                self.tool_panel[ key ] = section
 335
 336    def load_integrated_tool_panel_keys( self ):
 337        """
 338        Load the integrated tool panel keys, setting values for tools and workflows to None.  The values will
 339        be reset when the various tool panel config files are parsed, at which time the tools and workflows are
 340        loaded.
 341        """
 342        tree = parse_xml( self.integrated_tool_panel_config )
 343        root = tree.getroot()
 344        for elem in root:
 345            if elem.tag == 'tool':
 346                key = 'tool_%s' % elem.get( 'id' )
 347                self.integrated_tool_panel[ key ] = None
 348            elif elem.tag == 'workflow':
 349                key = 'workflow_%s' % elem.get( 'id' )
 350                self.integrated_tool_panel[ key ] = None
 351            elif elem.tag == 'section':
 352                section = ToolSection( elem )
 353                for section_elem in elem:
 354                    if section_elem.tag == 'tool':
 355                        key = 'tool_%s' % section_elem.get( 'id' )
 356                        section.elems[ key ] = None
 357                    elif section_elem.tag == 'workflow':
 358                        key = 'workflow_%s' % section_elem.get( 'id' )
 359                        section.elems[ key ] = None
 360                    elif section_elem.tag == 'label':
 361                        key = 'label_%s' % section_elem.get( 'id' )
 362                        section.elems[ key ] = None
 363                key = elem.get( 'id' )
 364                self.integrated_tool_panel[ key ] = section
 365            elif elem.tag == 'label':
 366                key = 'label_%s' % elem.get( 'id' )
 367                self.integrated_tool_panel[ key ] = None
 368
 369    def write_integrated_tool_panel_config_file( self ):
 370        """
 371        Write the current in-memory version of the integrated_tool_panel.xml file to disk.  Since Galaxy administrators
 372        use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right.
 373        """
 374        fd, filename = tempfile.mkstemp()
 375        os.write( fd, '<?xml version="1.0"?>\n' )
 376        os.write( fd, '<toolbox>\n' )
 377        for key, item in self.integrated_tool_panel.items():
 378            if item:
 379                if isinstance( item, Tool ):
 380                    os.write( fd, '    <tool id="%s" />\n' % item.id )
 381                elif isinstance( item, Workflow ):
 382                    os.write( fd, '    <workflow id="%s" />\n' % item.id )
 383                elif isinstance( item, ToolSectionLabel ):
 384                    label_id = item.id or ''
 385                    label_text = item.text or ''
 386                    label_version = item.version or ''
 387                    os.write( fd, '    <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
 388                elif isinstance( item, ToolSection ):
 389                    section_id = item.id or ''
 390                    section_name = item.name or ''
 391                    section_version = item.version or ''
 392                    os.write( fd, '    <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) )
 393                    for section_key, section_item in item.elems.items():
 394                        if isinstance( section_item, Tool ):
 395                            if section_item:
 396                                os.write( fd, '        <tool id="%s" />\n' % section_item.id )
 397                        elif isinstance( section_item, Workflow ):
 398                            if section_item:
 399                                os.write( fd, '        <workflow id="%s" />\n' % section_item.id )
 400                        elif isinstance( section_item, ToolSectionLabel ):
 401                            if section_item:
 402                                label_id = section_item.id or ''
 403                                label_text = section_item.text or ''
 404                                label_version = section_item.version or ''
 405                                os.write( fd, '        <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
 406                    os.write( fd, '    </section>\n' )
 407        os.write( fd, '</toolbox>\n' )
 408        os.close( fd )
 409        shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) )
 410        os.chmod( self.integrated_tool_panel_config, 0644 )
 411
 412    def get_tool( self, tool_id, tool_version=None, get_all_versions=False ):
 413        """Attempt to locate a tool in the tool box."""
 414        if tool_id in self.tools_by_id and not get_all_versions:
 415            #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid)
 416            return self.tools_by_id[ tool_id ]
 417        #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions
 418        rval = []
 419        tv = self.__get_tool_version( tool_id )
 420        if tv:
 421            tool_version_ids = tv.get_version_ids( self.app )
 422            for tool_version_id in tool_version_ids:
 423                if tool_version_id in self.tools_by_id:
 424                    rval.append( self.tools_by_id[ tool_version_id ] )
 425        if not rval:
 426            #still no tool, do a deeper search and try to match by old ids
 427            for tool in self.tools_by_id.itervalues():
 428                if tool.old_id == tool_id:
 429                    rval.append( tool )
 430        if rval:
 431            if get_all_versions:
 432                return rval
 433            else:
 434                if tool_version:
 435                    #return first tool with matching version
 436                    for tool in rval:
 437                        if tool.version == tool_version:
 438                            return tool
 439                #No tool matches by version, simply return the first available tool found
 440                return rval[0]
 441        #We now likely have a Toolshed guid passed in, but no supporting database entries
 442        #If the tool exists by exact id and is loaded then provide exact match within a list
 443        if tool_id in self.tools_by_id:
 444            return[ self.tools_by_id[ tool_id ] ]
 445        return None
 446
 447    def get_loaded_tools_by_lineage( self, tool_id ):
 448        """Get all loaded tools associated by lineage to the tool whose id is tool_id."""
 449        tv = self.__get_tool_version( tool_id )
 450        if tv:
 451            tool_version_ids = tv.get_version_ids( self.app )
 452            available_tool_versions = []
 453            for tool_version_id in tool_version_ids:
 454                if tool_version_id in self.tools_by_id:
 455                    available_tool_versions.append( self.tools_by_id[ tool_version_id ] )
 456            return available_tool_versions
 457        else:
 458            if tool_id in self.tools_by_id:
 459                tool = self.tools_by_id[ tool_id ]
 460                return [ tool ]
 461        return []
 462
 463    def __get_tool_version( self, tool_id ):
 464        """Return a ToolVersion if one exists for the tool_id"""
 465        return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \
 466                                             .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \
 467                                             .first()
 468
 469    def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ):
 470        return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \
 471                              .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed,
 472                                             self.app.install_model.ToolShedRepository.table.c.name == name,
 473                                             self.app.install_model.ToolShedRepository.table.c.owner == owner,
 474                                             self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \
 475                              .first()
 476
 477    def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ):
 478        """
 479        Retrieve all loaded versions of a tool from the toolbox and return a select list enabling
 480        selection of a different version, the list of the tool's loaded versions, and the specified tool.
 481        """
 482        toolbox = self
 483        tool_version_select_field = None
 484        tools = []
 485        tool = None
 486        # Backwards compatibility for datasource tools that have default tool_id configured, but which
 487        # are now using only GALAXY_URL.
 488        tool_ids = listify( tool_id )
 489        for tool_id in tool_ids:
 490            if get_loaded_tools_by_lineage:
 491                tools = toolbox.get_loaded_tools_by_lineage( tool_id )
 492            else:
 493                tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True )
 494            if tools:
 495                tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False )
 496                if len( tools ) > 1:
 497                    tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected )
 498                break
 499        return tool_version_select_field, tools, tool
 500
 501    def build_tool_version_select_field( self, tools, tool_id, set_selected ):
 502        """Build a SelectField whose options are the ids for the received list of tools."""
 503        options = []
 504        refresh_on_change_values = []
 505        for tool in tools:
 506            options.insert( 0, ( tool.version, tool.id ) )
 507            refresh_on_change_values.append( tool.id )
 508        select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values )
 509        for option_tup in options:
 510            selected = set_selected and option_tup[ 1 ] == tool_id
 511            if selected:
 512                select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True )
 513            else:
 514                select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] )
 515        return select_field
 516
 517    def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
 518        try:
 519            path = elem.get( "file" )
 520            repository_id = None
 521            if guid is None:
 522                tool_shed_repository = None
 523                can_load_into_panel_dict = True
 524            else:
 525                # The tool is contained in an installed tool shed repository, so load
 526                # the tool only if the repository has not been marked deleted.
 527                tool_shed = elem.find( "tool_shed" ).text
 528                repository_name = elem.find( "repository_name" ).text
 529                repository_owner = elem.find( "repository_owner" ).text
 530                installed_changeset_revision_elem = elem.find( "installed_changeset_revision" )
 531                if installed_changeset_revision_elem is None:
 532                    # Backward compatibility issue - the tag used to be named 'changeset_revision'.
 533                    installed_changeset_revision_elem = elem.find( "changeset_revision" )
 534                installed_changeset_revision = installed_changeset_revision_elem.text
 535                tool_shed_repository = self.__get_tool_shed_repository( tool_shed, repository_name, repository_owner, installed_changeset_revision )
 536                if tool_shed_repository:
 537                    # Only load tools if the repository is not deactivated or uninstalled.
 538                    can_load_into_panel_dict = not tool_shed_repository.deleted
 539                    repository_id = self.app.security.encode_id( tool_shed_repository.id )
 540                else:
 541                    # If there is not yet a tool_shed_repository record, we're in the process of installing
 542                    # a new repository, so any included tools can be loaded into the tool panel.
 543                    can_load_into_panel_dict = True
 544            tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id )
 545            key = 'tool_%s' % str( tool.id )
 546            if can_load_into_panel_dict:
 547                if guid is not None:
 548                    tool.tool_shed = tool_shed
 549                    tool.repository_name = repository_name
 550                    tool.repository_owner = repository_owner
 551                    tool.installed_changeset_revision = installed_changeset_revision
 552                    tool.guid = guid
 553                    tool.version = elem.find( "version" ).text
 554                # Make sure the tool has a tool_version.
 555                if not self.__get_tool_version( tool.id ):
 556                    tool_version = self.app.install_model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository )
 557                    self.app.install_model.context.add( tool_version )
 558                    self.app.install_model.context.flush()
 559                # Load the tool's lineage ids.
 560                tool.lineage_ids = tool.tool_version.get_version_ids( self.app )
 561                if self.app.config.get_bool( 'enable_tool_tags', False ):
 562                    tag_names = elem.get( "tags", "" ).split( "," )
 563                    for tag_name in tag_names:
 564                        if tag_name == '':
 565                            continue
 566                        tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first()
 567                        if not tag:
 568                            tag = self.app.model.Tag( name=tag_name )
 569                            self.sa_session.add( tag )
 570                            self.sa_session.flush()
 571                            tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
 572                            self.sa_session.add( tta )
 573                            self.sa_session.flush()
 574                        else:
 575                            for tagged_tool in tag.tagged_tools:
 576                                if tagged_tool.tool_id == tool.id:
 577                                    break
 578                            else:
 579                                tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
 580                                self.sa_session.add( tta )
 581                                self.sa_session.flush()
 582                # Allow for the same tool to be loaded into multiple places in the tool panel.  We have to handle
 583                # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
 584                # administrator has retrieved updates to the installed repository.  In this case, the tool may have
 585                # been updated, but the version was not changed, so the tool should always be reloaded here.  We used
 586                # to only load the tool if it's it was not found in self.tools_by_id, but performing that check did
 587                # not enable this scenario.
 588                self.tools_by_id[ tool.id ] = tool
 589                if load_panel_dict:
 590                    self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
 591            # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
 592            if key in integrated_panel_dict or index is None:
 593                integrated_panel_dict[ key ] = tool
 594            else:
 595                integrated_panel_dict.insert( index, key, tool )
 596        except:
 597            log.exception( "Error reading tool from path: %s" % path )
 598
 599    def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
 600        try:
 601            # TODO: should id be encoded?
 602            workflow_id = elem.get( 'id' )
 603            workflow = self.load_workflow( workflow_id )
 604            self.workflows_by_id[ workflow_id ] = workflow
 605            key = 'workflow_' + workflow_id
 606            if load_panel_dict:
 607                panel_dict[ key ] = workflow
 608            # Always load workflows into the integrated_panel_dict.
 609            if key in integrated_panel_dict or index is None:
 610                integrated_panel_dict[ key ] = workflow
 611            else:
 612                integrated_panel_dict.insert( index, key, workflow )
 613        except:
 614            log.exception( "Error loading workflow: %s" % workflow_id )
 615
 616    def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
 617        label = ToolSectionLabel( elem )
 618        key = 'label_' + label.id
 619        if load_panel_dict:
 620            panel_dict[ key ] = label
 621        if key in integrated_panel_dict or index is None:
 622            integrated_panel_dict[ key ] = label
 623        else:
 624            integrated_panel_dict.insert( index, key, label )
 625
 626    def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ):
 627        key = elem.get( "id" )
 628        if key in self.tool_panel:
 629            section = self.tool_panel[ key ]
 630            elems = section.elems
 631        else:
 632            section = ToolSection( elem )
 633            elems = section.elems
 634        if key in self.integrated_tool_panel:
 635            integrated_section = self.integrated_tool_panel[ key ]
 636            integrated_elems = integrated_section.elems
 637        else:
 638            integrated_section = ToolSection( elem )
 639            integrated_elems = integrated_section.elems
 640        for sub_index, sub_elem in enumerate( elem ):
 641            if sub_elem.tag == 'tool':
 642                self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index )
 643            elif sub_elem.tag == 'workflow':
 644                self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
 645            elif sub_elem.tag == 'label':
 646                self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
 647        if load_panel_dict:
 648            self.tool_panel[ key ] = section
 649        # Always load sections into the integrated_tool_panel.
 650        if key in self.integrated_tool_panel or index is None:
 651            self.integrated_tool_panel[ key ] = integrated_section
 652        else:
 653            self.integrated_tool_panel.insert( index, key, integrated_section )
 654
 655    def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
 656        """Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
 657        # Parse XML configuration file and get the root element
 658        tree = load_tool( config_file )
 659        root = tree.getroot()
 660        # Allow specifying a different tool subclass to instantiate
 661        if root.find( "type" ) is not None:
 662            type_elem = root.find( "type" )
 663            module = type_elem.get( 'module', 'galaxy.tools' )
 664            cls = type_elem.get( 'class' )
 665            mod = __import__( module, globals(), locals(), [cls] )
 666            ToolClass = getattr( mod, cls )
 667        elif root.get( 'tool_type', None ) is not None:
 668            ToolClass = tool_types.get( root.get( 'tool_type' ) )
 669        else:
 670            ToolClass = Tool
 671        return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )
 672
 673    def reload_tool_by_id( self, tool_id ):
 674        """
 675        Attempt to reload the tool identified by 'tool_id', if successful
 676        replace the old tool.
 677        """
 678        if tool_id not in self.tools_by_id:
 679            message = "No tool with id %s" % tool_id
 680            status = 'error'
 681        else:
 682            old_tool = self.tools_by_id[ tool_id ]
 683            new_tool = self.load_tool( old_tool.config_file )
 684            # The tool may have been installed from a tool shed, so set the tool shed attributes.
 685            # Since the tool version may have changed, we don't override it here.
 686            new_tool.id = old_tool.id
 687            new_tool.guid = old_tool.guid
 688            new_tool.tool_shed = old_tool.tool_shed
 689            new_tool.repository_name = old_tool.repository_name
 690            new_tool.repository_owner = old_tool.repository_owner
 691            new_tool.installed_changeset_revision = old_tool.installed_changeset_revision
 692            new_tool.old_id = old_tool.old_id
 693            # Replace old_tool with new_tool in self.tool_panel
 694            tool_key = 'tool_' + tool_id
 695            for key, val in self.tool_panel.items():
 696                if key == tool_key:
 697                    self.tool_panel[ key ] = new_tool
 698                    break
 699                elif key.startswith( 'section' ):
 700                    if tool_key in val.elems:
 701                        self.tool_panel[ key ].elems[ tool_key ] = new_tool
 702                        break
 703            self.tools_by_id[ tool_id ] = new_tool
 704            message = "Reloaded the tool:<br/>"
 705            message += "<b>name:</b> %s<br/>" % old_tool.name
 706            message += "<b>id:</b> %s<br/>" % old_tool.id
 707            message += "<b>version:</b> %s" % old_tool.version
 708            status = 'done'
 709        return message, status
 710
 711    def remove_tool_by_id( self, tool_id ):
 712        """
 713        Attempt to remove the tool identified by 'tool_id'.
 714        """
 715        if tool_id not in self.tools_by_id:
 716            message = "No tool with id %s" % tool_id
 717            status = 'error'
 718        else:
 719            tool = self.tools_by_id[ tool_id ]
 720            del self.tools_by_id[ tool_id ]
 721            tool_key = 'tool_' + tool_id
 722            for key, val in self.tool_panel.items():
 723                if key == tool_key:
 724                    del self.tool_panel[ key ]
 725                    break
 726                elif key.startswith( 'section' ):
 727                    if tool_key in val.elems:
 728                        del self.tool_panel[ key ].elems[ tool_key ]
 729                        break
 730            if tool_id in self.data_manager_tools:
 731                del self.data_manager_tools[ tool_id ]
 732            #TODO: do we need to manually remove from the integrated panel here?
 733            message = "Removed the tool:<br/>"
 734            message += "<b>name:</b> %s<br/>" % tool.name
 735            message += "<b>id:</b> %s<br/>" % tool.id
 736            message += "<b>version:</b> %s" % tool.version
 737            status = 'done'
 738        return message, status
 739
 740    def load_workflow( self, workflow_id ):
 741        """
 742        Return an instance of 'Workflow' identified by `id`,
 743        which is encoded in the tool panel.
 744        """
 745        id = self.app.security.decode_id( workflow_id )
 746        stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id )
 747        return stored.latest_workflow
 748
 749    def init_dependency_manager( self ):
 750        self.dependency_manager = build_dependency_manager( self.app.config )
 751
 752    @property
 753    def sa_session( self ):
 754        """
 755        Returns a SQLAlchemy session
 756        """
 757        return self.app.model.context
 758
 759    def to_dict( self, trans, in_panel=True, **kwds ):
 760        """
 761        to_dict toolbox.
 762        """
 763
 764        context = Bunch( toolbox=self, trans=trans, **kwds )
 765        if in_panel:
 766            panel_elts = [ val for val in self.tool_panel.itervalues() ]
 767
 768            filters = self.filter_factory.build_filters( trans, **kwds )
 769
 770            filtered_panel_elts = []
 771            for index, elt in enumerate( panel_elts ):
 772                elt = _filter_for_panel( elt, filters, context )
 773                if elt:
 774                    filtered_panel_elts.append( elt )
 775            panel_elts = filtered_panel_elts
 776
 777            # Produce panel.
 778            rval = []
 779            kwargs = dict(
 780                trans=trans,
 781                link_details=True
 782            )
 783            for elt in panel_elts:
 784                rval.append( to_dict_helper( elt, kwargs ) )
 785        else:
 786            tools = []
 787            for id, tool in self.tools_by_id.items():
 788                tools.append( tool.to_dict( trans, link_details=True ) )
 789            rval = tools
 790
 791        return rval
 792
 793
 794def _filter_for_panel( item, filters, context ):
 795    """
 796    Filters tool panel elements so that only those that are compatible
 797    with provided filters are kept.
 798    """
 799    def _apply_filter( filter_item, filter_list ):
 800        for filter_method in filter_list:
 801            if not filter_method( context, filter_item ):
 802                return False
 803        return True
 804    if isinstance( item, Tool ):
 805        if _apply_filter( item, filters[ 'tool' ] ):
 806            return item
 807    elif isinstance( item, ToolSectionLabel ):
 808        if _apply_filter( item, filters[ 'label' ] ):
 809            return item
 810    elif isinstance( item, ToolSection ):
 811        # Filter section item-by-item. Only show a label if there are
 812        # non-filtered tools below it.
 813
 814        if _apply_filter( item, filters[ 'section' ] ):
 815            cur_label_key = None
 816            tools_under_label = False
 817            filtered_elems = item.elems.copy()
 818            for key, section_item in item.elems.items():
 819                if isinstance( section_item, Tool ):
 820                    # Filter tool.
 821                    if _apply_filter( section_item, filters[ 'tool' ] ):
 822                        tools_under_label = True
 823                    else:
 824                        del filtered_elems[ key ]
 825                elif isinstance( section_item, ToolSectionLabel ):
 826                    # If there is a label and it does not have tools,
 827                    # remove it.
 828                    if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
 829                        del filtered_elems[ cur_label_key ]
 830
 831                    # Reset attributes for new label.
 832                    cur_label_key = key
 833                    tools_under_label = False
 834
 835            # Handle last label.
 836            if cur_label_key and not tools_under_label:
 837                del filtered_elems[ cur_label_key ]
 838
 839            # Only return section if there are elements.
 840            if len( filtered_elems ) != 0:
 841                copy = item.copy()
 842                copy.elems = filtered_elems
 843                return copy
 844
 845    return None
 846
 847
 848class ToolSection( object, Dictifiable ):
 849    """
 850    A group of tools with similar type/purpose that will be displayed as a
 851    group in the user interface.
 852    """
 853
 854    dict_collection_visible_keys = ( 'id', 'name', 'version' )
 855
 856    def __init__( self, elem=None ):
 857        f = lambda elem, val: elem is not None and elem.get( val ) or ''
 858        self.name = f( elem, 'name' )
 859        self.id = f( elem, 'id' )
 860        self.version = f( elem, 'version' )
 861        self.elems = odict()
 862
 863    def copy( self ):
 864        copy = ToolSection()
 865        copy.name = self.name
 866        copy.id = self.id
 867        copy.version = self.version
 868        copy.elems = self.elems.copy()
 869        return copy
 870
 871    def to_dict( self, trans, link_details=False ):
 872        """ Return a dict that includes section's attributes. """
 873
 874        section_dict = super( ToolSection, self ).to_dict()
 875        section_elts = []
 876        kwargs = dict(
 877            trans=trans,
 878            link_details=link_details
 879        )
 880        for elt in self.elems.values():
 881            section_elts.append( to_dict_helper( elt, kwargs ) )
 882        section_dict[ 'elems' ] = section_elts
 883
 884        return section_dict
 885
 886
 887class ToolSectionLabel( object, Dictifiable ):
 888    """
 889    A label for a set of tools that can be displayed above groups of tools
 890    and sections in the user interface
 891    """
 892
 893    dict_collection_visible_keys = ( 'id', 'text', 'version' )
 894
 895    def __init__( self, elem ):
 896        self.text = elem.get( "text" )
 897        self.id = elem.get( "id" )
 898        self.version = elem.get( "version" ) or ''
 899
 900
 901class DefaultToolState( object ):
 902    """
 903    Keeps track of the state of a users interaction with a tool between
 904    requests. The default tool state keeps track of the current page (for
 905    multipage "wizard" tools) and the values of all
 906    """
 907    def __init__( self ):
 908        self.page = 0
 909        self.rerun_remap_job_id = None
 910        self.inputs = None
 911
 912    def encode( self, tool, app, secure=True ):
 913        """
 914        Convert the data to a string
 915        """
 916        # Convert parameters to a dictionary of strings, and save curent
 917        # page in that dict
 918        value = params_to_strings( tool.inputs, self.inputs, app )
 919        value["__page__"] = self.page
 920        value["__rerun_remap_job_id__"] = self.rerun_remap_job_id
 921        value = json.dumps( value )
 922        # Make it secure
 923        if secure:
 924            a = hmac_new( app.config.tool_secret, value )
 925            b = binascii.hexlify( value )
 926            return "%s:%s" % ( a, b )
 927        else:
 928            return value
 929
 930    def decode( self, value, tool, app, secure=True ):
 931        """
 932        Restore the state from a string
 933        """
 934        if secure:
 935            # Extract and verify hash
 936            a, b = value.split( ":" )
 937            value = binascii.unhexlify( b )
 938            test = hmac_new( app.config.tool_secret, value )
 939            assert a == test
 940        # Restore from string
 941        values = json_fix( json.loads( value ) )
 942        self.page = values.pop( "__page__" )
 943        if '__rerun_remap_job_id__' in values:
 944            self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" )
 945        else:
 946            self.rerun_remap_job_id = None
 947        self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True )
 948
 949
 950class ToolOutput( object, Dictifiable ):
 951    """
 952    Represents an output datasets produced by a tool. For backward
 953    compatibility this behaves as if it were the tuple::
 954
 955      (format, metadata_source, parent)
 956    """
 957
 958    dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' )
 959
 960    def __init__( self, name, format=None, format_source=None, metadata_source=None,
 961                  parent=None, label=None, filters=None, actions=None, hidden=False ):
 962        self.name = name
 963        self.format = format
 964        self.format_source = format_source
 965        self.metadata_source = metadata_source
 966        self.parent = parent
 967        self.label = label
 968        self.filters = filters or []
 969        self.actions = actions
 970        self.hidden = hidden
 971
 972    # Tuple emulation
 973
 974    def __len__( self ):
 975        return 3
 976
 977    def __getitem__( self, index ):
 978        if index == 0:
 979            return self.format
 980        elif index == 1:
 981            return self.metadata_source
 982        elif index == 2:
 983            return self.parent
 984        else:
 985            raise IndexError( index )
 986
 987    def __iter__( self ):
 988        return iter( ( self.format, self.metadata_source, self.parent ) )
 989
 990
 991class Tool( object, Dictifiable ):
 992    """
 993    Represents a computational tool that can be executed through Galaxy.
 994    """
 995
 996    tool_type = 'default'
 997    requires_setting_metadata = True
 998    default_tool_action = DefaultToolAction
 999    dict_collection_visible_keys = ( 'id', 'name', 'version', 'description' )
1000
1001    def __init__( self, config_file, root, app, guid=None, repository_id=None ):
1002        """Load a tool from the config named by `config_file`"""
1003        # Determine the full path of the directory where the tool config is
1004        self.config_file = config_file
1005        self.tool_dir = os.path.dirname( config_file )
1006        self.app = app
1007        self.repository_id = repository_id
1008        #setup initial attribute values
1009        self.inputs = odict()
1010        self.stdio_exit_codes = list()
1011        self.stdio_regexes = list()
1012        self.inputs_by_page = list()
1013        self.display_by_page = list()
1014        self.action = '/tool_runner/index'
1015        self.target = 'galaxy_main'
1016        self.method = 'post'
1017        self.check_values = True
1018        self.nginx_upload = False
1019        self.input_required = False
1020        self.display_interface = True
1021        self.require_login = False
1022        self.rerun = False
1023        # Define a place to keep track of all input   These
1024        # differ from the inputs dictionary in that inputs can be page
1025        # elements like conditionals, but input_params are basic form
1026        # parameters like SelectField objects.  This enables us to more
1027        # easily ensure that parameter dependencies like index files or
1028        # tool_data_table_conf.xml entries exist.
1029        self.input_params = []
1030        # Attributes of tools installed from Galaxy tool sheds.
1031        self.tool_shed = None
1032        self.repository_name = None
1033        self.repository_owner = None
1034        self.installed_changeset_revision = None
1035        # The tool.id value will be the value of guid, but we'll keep the
1036        # guid attribute since it is useful to have.
1037        self.guid = guid
1038        self

Large files files are truncated, but you can click here to view the full file