"""
Classes encapsulating galaxy tools and tool configuration.
"""

import binascii
import glob
import json
import logging
import os
import pipes
import re
import shutil
import sys
import tempfile
import threading
import traceback
import types
import urllib

from math import isinf

from galaxy import eggs
eggs.require( "MarkupSafe" )  # MarkupSafe must load before mako
eggs.require( "Mako" )
eggs.require( "elementtree" )
eggs.require( "Paste" )
eggs.require( "SQLAlchemy >= 0.4" )

from cgi import FieldStorage
from elementtree import ElementTree
from mako.template import Template
from paste import httpexceptions
from sqlalchemy import and_

from galaxy import jobs, model
from galaxy.jobs.error_level import StdioErrorLevel
from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
from galaxy.jobs import ParallelismInfo
from galaxy.tools.actions import DefaultToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.data_manager import DataManagerToolAction
from galaxy.tools.deps import build_dependency_manager
from galaxy.tools.deps.requirements import parse_requirements_from_xml
from galaxy.tools.parameters import check_param, params_from_strings, params_to_strings
from galaxy.tools.parameters.basic import (BaseURLToolParameter,
                                           DataToolParameter, HiddenToolParameter, LibraryDatasetToolParameter,
                                           SelectToolParameter, ToolParameter, UnvalidatedValue,
                                           IntegerToolParameter, FloatToolParameter)
from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, UploadDataset
from galaxy.tools.parameters.input_translation import ToolInputTranslator
from galaxy.tools.parameters.output import ToolOutputActionGroup
from galaxy.tools.parameters.validation import LateValidationError
from galaxy.tools.filters import FilterFactory
from galaxy.tools.test import parse_tests_elem
from galaxy.util import listify, parse_xml, rst_to_html, string_as_bool, string_to_object, xml_text, xml_to_string
from galaxy.util.bunch import Bunch
from galaxy.util.expressions import ExpressionContext
from galaxy.util.hash_util import hmac_new
from galaxy.util.none_like import NoneDataset
from galaxy.util.odict import odict
from galaxy.util.template import fill_template
from galaxy.web import url_for
from galaxy.web.form_builder import SelectField
from galaxy.model.item_attrs import Dictifiable
from galaxy.model import Workflow
from tool_shed.util import shed_util_common as suc
from .loader import load_tool, template_macro_params
from .wrappers import (
    ToolParameterValueWrapper,
    RawObjectWrapper,
    LibraryDatasetValueWrapper,
    InputValueWrapper,
    SelectToolParameterWrapper,
    DatasetFilenameWrapper,
    DatasetListWrapper,
)


log = logging.getLogger( __name__ )

WORKFLOW_PARAMETER_REGULAR_EXPRESSION = re.compile( '''\$\{.+?\}''' )


class ToolNotFoundException( Exception ):
    pass


def to_dict_helper( obj, kwargs ):
    """ Helper function that provides the appropriate kwargs to to_dict an object. """

    # Label.to_dict cannot have kwargs.
    if isinstance( obj, ToolSectionLabel ):
        kwargs = {}

    return obj.to_dict( **kwargs )


class ToolBox( object, Dictifiable ):
    """Container for a collection of tools"""

    def __init__( self, config_filenames, tool_root_dir, app ):
        """
        Create a toolbox from the config files named by `config_filenames`, using
        `tool_root_dir` as the base directory for finding individual tool config files.
        """
        # The shed_tool_confs list contains dictionaries storing information about the tools defined in each
        # shed-related shed_tool_conf.xml file.
        self.shed_tool_confs = []
        self.tools_by_id = {}
        self.workflows_by_id = {}
        # In-memory dictionary that defines the layout of the tool panel.
        self.tool_panel = odict()
        self.index = 0
        self.data_manager_tools = odict()
        # File that contains the XML section and tool tags from all tool panel config files integrated into a
        # single file that defines the tool panel layout.  This file can be changed by the Galaxy administrator
        # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel.
        self.integrated_tool_panel_config = os.path.join( app.config.root, 'integrated_tool_panel.xml' )
        # In-memory dictionary that defines the layout of the tool_panel.xml file on disk.
        self.integrated_tool_panel = odict()
        self.integrated_tool_panel_config_has_contents = os.path.exists( self.integrated_tool_panel_config ) and os.stat( self.integrated_tool_panel_config ).st_size > 0
        if self.integrated_tool_panel_config_has_contents:
            self.load_integrated_tool_panel_keys()
        # The following refers to the tool_path config setting for backward compatibility.  The shed-related
        # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the <toolbox> tag.
        self.tool_root_dir = tool_root_dir
        self.app = app
        self.filter_factory = FilterFactory( self )
        self.init_dependency_manager()
        config_filenames = listify( config_filenames )
        for config_filename in config_filenames:
            if os.path.isdir( config_filename ):
                directory_contents = sorted( os.listdir( config_filename ) )
                directory_config_files = [ config_file for config_file in directory_contents if config_file.endswith( ".xml" ) ]
                config_filenames.remove( config_filename )
                config_filenames.extend( directory_config_files )
        for config_filename in config_filenames:
            try:
                self.init_tools( config_filename )
            except:
                log.exception( "Error loading tools defined in config %s", config_filename )
        if self.integrated_tool_panel_config_has_contents:
            # Load self.tool_panel based on the order in self.integrated_tool_panel.
            self.load_tool_panel()
        if app.config.update_integrated_tool_panel:
            # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file.
            # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel
            # config files, adding or removing locally developed tools or workflows.  The value of integrated_tool_panel
            # will be False when things like functional tests are the caller.
            self.fix_integrated_tool_panel_dict()
            self.write_integrated_tool_panel_config_file()

    def fix_integrated_tool_panel_dict( self ):
        # HACK: instead of fixing after the fact, I suggest some combination of:
        #  1) adjusting init_tools() and called methods to get this right
        #  2) redesigning the code and/or data structure used to read/write integrated_tool_panel.xml
        for key, value in self.integrated_tool_panel.iteritems():
            if isinstance( value, ToolSection ):
                for section_key, section_value in value.elems.iteritems():
                    if section_value is None:
                        if isinstance( section_value, Tool ):
                            tool_id = section_key[5:]
                            value.elems[section_key] = self.tools_by_id.get( tool_id )
                        elif isinstance( section_value, Workflow ):
                            workflow_id = section_key[9:]
                            value.elems[section_key] = self.workflows_by_id.get( workflow_id )

    def init_tools( self, config_filename ):
        """
        Read the configuration file and load each tool.  The following tags are currently supported:

        .. raw:: xml

            <toolbox>
                <tool file="data_source/upload.xml"/>            # tools outside sections
                <label text="Basic Tools" id="basic_tools" />    # labels outside sections
                <workflow id="529fd61ab1c6cc36" />               # workflows outside sections
                <section name="Get Data" id="getext">            # sections
                    <tool file="data_source/biomart.xml" />      # tools inside sections
                    <label text="In Section" id="in_section" />  # labels inside sections
                    <workflow id="adb5f5c93f827949" />           # workflows inside sections
                </section>
            </toolbox>

        """
        if self.app.config.get_bool( 'enable_tool_tags', False ):
            log.info("removing all tool tag associations (" + str( self.sa_session.query( self.app.model.ToolTagAssociation ).count() ) + ")" )
            self.sa_session.query( self.app.model.ToolTagAssociation ).delete()
            self.sa_session.flush()
        log.info( "Parsing the tool configuration %s" % config_filename )
        tree = parse_xml( config_filename )
        root = tree.getroot()
        tool_path = root.get( 'tool_path' )
        if tool_path:
            # We're parsing a shed_tool_conf file since we have a tool_path attribute.
            parsing_shed_tool_conf = True
            # Keep an in-memory list of xml elements to enable persistence of the changing tool config.
            config_elems = []
        else:
            parsing_shed_tool_conf = False
            # Default to backward compatible config setting.
            tool_path = self.tool_root_dir
        # Only load the panel_dict under certain conditions.
        load_panel_dict = not self.integrated_tool_panel_config_has_contents
        for _, elem in enumerate( root ):
            index = self.index
            self.index += 1
            if parsing_shed_tool_conf:
                config_elems.append( elem )
            if elem.tag == 'tool':
                self.load_tool_tag_set( elem, self.tool_panel, self.integrated_tool_panel, tool_path, load_panel_dict, guid=elem.get( 'guid' ), index=index )
            elif elem.tag == 'workflow':
                self.load_workflow_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
            elif elem.tag == 'section':
                self.load_section_tag_set( elem, tool_path, load_panel_dict, index=index )
            elif elem.tag == 'label':
                self.load_label_tag_set( elem, self.tool_panel, self.integrated_tool_panel, load_panel_dict, index=index )
        if parsing_shed_tool_conf:
            shed_tool_conf_dict = dict( config_filename=config_filename,
                                        tool_path=tool_path,
                                        config_elems=config_elems )
            self.shed_tool_confs.append( shed_tool_conf_dict )

    def get_shed_config_dict_by_filename( self, filename, default=None ):
        for shed_config_dict in self.shed_tool_confs:
            if shed_config_dict[ 'config_filename' ] == filename:
                return shed_config_dict
        return default

    def __add_tool_to_tool_panel( self, tool, panel_component, section=False ):
        # See if a version of this tool is already loaded into the tool panel.  The value of panel_component
        # will be a ToolSection (if the value of section=True) or self.tool_panel (if section=False).
        tool_id = str( tool.id )
        tool = self.tools_by_id[ tool_id ]
        if section:
            panel_dict = panel_component.elems
        else:
            panel_dict = panel_component
        already_loaded = False
        loaded_version_key = None
        lineage_id = None
        for lineage_id in tool.lineage_ids:
            if lineage_id in self.tools_by_id:
                loaded_version_key = 'tool_%s' % lineage_id
                if loaded_version_key in panel_dict:
                    already_loaded = True
                    break
        if already_loaded:
            if tool.lineage_ids.index( tool_id ) > tool.lineage_ids.index( lineage_id ):
                key = 'tool_%s' % tool.id
                index = panel_dict.keys().index( loaded_version_key )
                del panel_dict[ loaded_version_key ]
                panel_dict.insert( index, key, tool )
                log.debug( "Loaded tool id: %s, version: %s into tool panel." % ( tool.id, tool.version ) )
        else:
            inserted = False
            key = 'tool_%s' % tool.id
            # The value of panel_component is the in-memory tool panel dictionary.
            for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ):
                if key == integrated_panel_key:
                    panel_dict.insert( index, key, tool )
                    if not inserted:
                        inserted = True
            if not inserted:
                # Check the tool's installed versions.
                for lineage_id in tool.lineage_ids:
                    lineage_id_key = 'tool_%s' % lineage_id
                    for index, integrated_panel_key in enumerate( self.integrated_tool_panel.keys() ):
                        if lineage_id_key == integrated_panel_key:
                            panel_dict.insert( index, key, tool )
                            if not inserted:
                                inserted = True
                if not inserted:
                    if tool.guid is None or \
                        tool.tool_shed is None or \
                        tool.repository_name is None or \
                        tool.repository_owner is None or \
                        tool.installed_changeset_revision is None:
                        # We have a tool that was not installed from the Tool Shed, but is also not yet defined in
                        # integrated_tool_panel.xml, so append it to the tool panel.
                        panel_dict[ key ] = tool
                        log.debug( "Loaded tool id: %s, version: %s into tool panel.." % ( tool.id, tool.version ) )
                    else:
                        # We are in the process of installing the tool.
                        tool_version = self.__get_tool_version( tool_id )
                        tool_lineage_ids = tool_version.get_version_ids( self.app, reverse=True )
                        for lineage_id in tool_lineage_ids:
                            if lineage_id in self.tools_by_id:
                                loaded_version_key = 'tool_%s' % lineage_id
                                if loaded_version_key in panel_dict:
                                    if not already_loaded:
                                        already_loaded = True
                        if not already_loaded:
                            # If the tool is not defined in integrated_tool_panel.xml, append it to the tool panel.
                            panel_dict[ key ] = tool
                            log.debug( "Loaded tool id: %s, version: %s into tool panel...." % ( tool.id, tool.version ) )

    def load_tool_panel( self ):
        for key, val in self.integrated_tool_panel.items():
            if isinstance( val, Tool ):
                tool_id = key.replace( 'tool_', '', 1 )
                if tool_id in self.tools_by_id:
                    self.__add_tool_to_tool_panel( val, self.tool_panel, section=False )
            elif isinstance( val, Workflow ):
                workflow_id = key.replace( 'workflow_', '', 1 )
                if workflow_id in self.workflows_by_id:
                    workflow = self.workflows_by_id[ workflow_id ]
                    self.tool_panel[ key ] = workflow
                    log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
            elif isinstance( val, ToolSectionLabel ):
                self.tool_panel[ key ] = val
            elif isinstance( val, ToolSection ):
                elem = ElementTree.Element( 'section' )
                elem.attrib[ 'id' ] = val.id or ''
                elem.attrib[ 'name' ] = val.name or ''
                elem.attrib[ 'version' ] = val.version or ''
                section = ToolSection( elem )
                log.debug( "Loading section: %s" % elem.get( 'name' ) )
                for section_key, section_val in val.elems.items():
                    if isinstance( section_val, Tool ):
                        tool_id = section_key.replace( 'tool_', '', 1 )
                        if tool_id in self.tools_by_id:
                            self.__add_tool_to_tool_panel( section_val, section, section=True )
                    elif isinstance( section_val, Workflow ):
                        workflow_id = section_key.replace( 'workflow_', '', 1 )
                        if workflow_id in self.workflows_by_id:
                            workflow = self.workflows_by_id[ workflow_id ]
                            section.elems[ section_key ] = workflow
                            log.debug( "Loaded workflow: %s %s" % ( workflow_id, workflow.name ) )
                    elif isinstance( section_val, ToolSectionLabel ):
                        if section_val:
                            section.elems[ section_key ] = section_val
                            log.debug( "Loaded label: %s" % ( section_val.text ) )
                self.tool_panel[ key ] = section

    def load_integrated_tool_panel_keys( self ):
        """
        Load the integrated tool panel keys, setting values for tools and workflows to None.  The values will
        be reset when the various tool panel config files are parsed, at which time the tools and workflows are
        loaded.
        """
        tree = parse_xml( self.integrated_tool_panel_config )
        root = tree.getroot()
        for elem in root:
            if elem.tag == 'tool':
                key = 'tool_%s' % elem.get( 'id' )
                self.integrated_tool_panel[ key ] = None
            elif elem.tag == 'workflow':
                key = 'workflow_%s' % elem.get( 'id' )
                self.integrated_tool_panel[ key ] = None
            elif elem.tag == 'section':
                section = ToolSection( elem )
                for section_elem in elem:
                    if section_elem.tag == 'tool':
                        key = 'tool_%s' % section_elem.get( 'id' )
                        section.elems[ key ] = None
                    elif section_elem.tag == 'workflow':
                        key = 'workflow_%s' % section_elem.get( 'id' )
                        section.elems[ key ] = None
                    elif section_elem.tag == 'label':
                        key = 'label_%s' % section_elem.get( 'id' )
                        section.elems[ key ] = None
                key = elem.get( 'id' )
                self.integrated_tool_panel[ key ] = section
            elif elem.tag == 'label':
                key = 'label_%s' % elem.get( 'id' )
                self.integrated_tool_panel[ key ] = None

    def write_integrated_tool_panel_config_file( self ):
        """
        Write the current in-memory version of the integrated_tool_panel.xml file to disk.  Since Galaxy administrators
        use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right.
        """
        fd, filename = tempfile.mkstemp()
        os.write( fd, '<?xml version="1.0"?>\n' )
        os.write( fd, '<toolbox>\n' )
        for key, item in self.integrated_tool_panel.items():
            if item:
                if isinstance( item, Tool ):
                    os.write( fd, '    <tool id="%s" />\n' % item.id )
                elif isinstance( item, Workflow ):
                    os.write( fd, '    <workflow id="%s" />\n' % item.id )
                elif isinstance( item, ToolSectionLabel ):
                    label_id = item.id or ''
                    label_text = item.text or ''
                    label_version = item.version or ''
                    os.write( fd, '    <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
                elif isinstance( item, ToolSection ):
                    section_id = item.id or ''
                    section_name = item.name or ''
                    section_version = item.version or ''
                    os.write( fd, '    <section id="%s" name="%s" version="%s">\n' % ( section_id, section_name, section_version ) )
                    for section_key, section_item in item.elems.items():
                        if isinstance( section_item, Tool ):
                            if section_item:
                                os.write( fd, '        <tool id="%s" />\n' % section_item.id )
                        elif isinstance( section_item, Workflow ):
                            if section_item:
                                os.write( fd, '        <workflow id="%s" />\n' % section_item.id )
                        elif isinstance( section_item, ToolSectionLabel ):
                            if section_item:
                                label_id = section_item.id or ''
                                label_text = section_item.text or ''
                                label_version = section_item.version or ''
                                os.write( fd, '        <label id="%s" text="%s" version="%s" />\n' % ( label_id, label_text, label_version ) )
                    os.write( fd, '    </section>\n' )
        os.write( fd, '</toolbox>\n' )
        os.close( fd )
        shutil.move( filename, os.path.abspath( self.integrated_tool_panel_config ) )
        os.chmod( self.integrated_tool_panel_config, 0644 )

    def get_tool( self, tool_id, tool_version=None, get_all_versions=False ):
        """Attempt to locate a tool in the tool box."""
        if tool_id in self.tools_by_id and not get_all_versions:
            #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid)
            return self.tools_by_id[ tool_id ]
        #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions
        rval = []
        tv = self.__get_tool_version( tool_id )
        if tv:
            tool_version_ids = tv.get_version_ids( self.app )
            for tool_version_id in tool_version_ids:
                if tool_version_id in self.tools_by_id:
                    rval.append( self.tools_by_id[ tool_version_id ] )
        if not rval:
            #still no tool, do a deeper search and try to match by old ids
            for tool in self.tools_by_id.itervalues():
                if tool.old_id == tool_id:
                    rval.append( tool )
        if rval:
            if get_all_versions:
                return rval
            else:
                if tool_version:
                    #return first tool with matching version
                    for tool in rval:
                        if tool.version == tool_version:
                            return tool
                #No tool matches by version, simply return the first available tool found
                return rval[0]
        #We now likely have a Toolshed guid passed in, but no supporting database entries
        #If the tool exists by exact id and is loaded then provide exact match within a list
        if tool_id in self.tools_by_id:
            return[ self.tools_by_id[ tool_id ] ]
        return None

    def get_loaded_tools_by_lineage( self, tool_id ):
        """Get all loaded tools associated by lineage to the tool whose id is tool_id."""
        tv = self.__get_tool_version( tool_id )
        if tv:
            tool_version_ids = tv.get_version_ids( self.app )
            available_tool_versions = []
            for tool_version_id in tool_version_ids:
                if tool_version_id in self.tools_by_id:
                    available_tool_versions.append( self.tools_by_id[ tool_version_id ] )
            return available_tool_versions
        else:
            if tool_id in self.tools_by_id:
                tool = self.tools_by_id[ tool_id ]
                return [ tool ]
        return []

    def __get_tool_version( self, tool_id ):
        """Return a ToolVersion if one exists for the tool_id"""
        return self.app.install_model.context.query( self.app.install_model.ToolVersion ) \
                                             .filter( self.app.install_model.ToolVersion.table.c.tool_id == tool_id ) \
                                             .first()

    def __get_tool_shed_repository( self, tool_shed, name, owner, installed_changeset_revision ):
        return self.app.install_model.context.query( self.app.install_model.ToolShedRepository ) \
                              .filter( and_( self.app.install_model.ToolShedRepository.table.c.tool_shed == tool_shed,
                                             self.app.install_model.ToolShedRepository.table.c.name == name,
                                             self.app.install_model.ToolShedRepository.table.c.owner == owner,
                                             self.app.install_model.ToolShedRepository.table.c.installed_changeset_revision == installed_changeset_revision ) ) \
                              .first()

    def get_tool_components( self, tool_id, tool_version=None, get_loaded_tools_by_lineage=False, set_selected=False ):
        """
        Retrieve all loaded versions of a tool from the toolbox and return a select list enabling
        selection of a different version, the list of the tool's loaded versions, and the specified tool.
        """
        toolbox = self
        tool_version_select_field = None
        tools = []
        tool = None
        # Backwards compatibility for datasource tools that have default tool_id configured, but which
        # are now using only GALAXY_URL.
        tool_ids = listify( tool_id )
        for tool_id in tool_ids:
            if get_loaded_tools_by_lineage:
                tools = toolbox.get_loaded_tools_by_lineage( tool_id )
            else:
                tools = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=True )
            if tools:
                tool = toolbox.get_tool( tool_id, tool_version=tool_version, get_all_versions=False )
                if len( tools ) > 1:
                    tool_version_select_field = self.build_tool_version_select_field( tools, tool.id, set_selected )
                break
        return tool_version_select_field, tools, tool

    def build_tool_version_select_field( self, tools, tool_id, set_selected ):
        """Build a SelectField whose options are the ids for the received list of tools."""
        options = []
        refresh_on_change_values = []
        for tool in tools:
            options.insert( 0, ( tool.version, tool.id ) )
            refresh_on_change_values.append( tool.id )
        select_field = SelectField( name='tool_id', refresh_on_change=True, refresh_on_change_values=refresh_on_change_values )
        for option_tup in options:
            selected = set_selected and option_tup[ 1 ] == tool_id
            if selected:
                select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ], selected=True )
            else:
                select_field.add_option( 'version %s' % option_tup[ 0 ], option_tup[ 1 ] )
        return select_field

    def load_tool_tag_set( self, elem, panel_dict, integrated_panel_dict, tool_path, load_panel_dict, guid=None, index=None ):
        try:
            path = elem.get( "file" )
            repository_id = None
            if guid is None:
                tool_shed_repository = None
                can_load_into_panel_dict = True
            else:
                # The tool is contained in an installed tool shed repository, so load
                # the tool only if the repository has not been marked deleted.
                tool_shed = elem.find( "tool_shed" ).text
                repository_name = elem.find( "repository_name" ).text
                repository_owner = elem.find( "repository_owner" ).text
                installed_changeset_revision_elem = elem.find( "installed_changeset_revision" )
                if installed_changeset_revision_elem is None:
                    # Backward compatibility issue - the tag used to be named 'changeset_revision'.
                    installed_changeset_revision_elem = elem.find( "changeset_revision" )
                installed_changeset_revision = installed_changeset_revision_elem.text
                tool_shed_repository = self.__get_tool_shed_repository( tool_shed, repository_name, repository_owner, installed_changeset_revision )
                if tool_shed_repository:
                    # Only load tools if the repository is not deactivated or uninstalled.
                    can_load_into_panel_dict = not tool_shed_repository.deleted
                    repository_id = self.app.security.encode_id( tool_shed_repository.id )
                else:
                    # If there is not yet a tool_shed_repository record, we're in the process of installing
                    # a new repository, so any included tools can be loaded into the tool panel.
                    can_load_into_panel_dict = True
            tool = self.load_tool( os.path.join( tool_path, path ), guid=guid, repository_id=repository_id )
            key = 'tool_%s' % str( tool.id )
            if can_load_into_panel_dict:
                if guid is not None:
                    tool.tool_shed = tool_shed
                    tool.repository_name = repository_name
                    tool.repository_owner = repository_owner
                    tool.installed_changeset_revision = installed_changeset_revision
                    tool.guid = guid
                    tool.version = elem.find( "version" ).text
                # Make sure the tool has a tool_version.
                if not self.__get_tool_version( tool.id ):
                    tool_version = self.app.install_model.ToolVersion( tool_id=tool.id, tool_shed_repository=tool_shed_repository )
                    self.app.install_model.context.add( tool_version )
                    self.app.install_model.context.flush()
                # Load the tool's lineage ids.
                tool.lineage_ids = tool.tool_version.get_version_ids( self.app )
                if self.app.config.get_bool( 'enable_tool_tags', False ):
                    tag_names = elem.get( "tags", "" ).split( "," )
                    for tag_name in tag_names:
                        if tag_name == '':
                            continue
                        tag = self.sa_session.query( self.app.model.Tag ).filter_by( name=tag_name ).first()
                        if not tag:
                            tag = self.app.model.Tag( name=tag_name )
                            self.sa_session.add( tag )
                            self.sa_session.flush()
                            tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
                            self.sa_session.add( tta )
                            self.sa_session.flush()
                        else:
                            for tagged_tool in tag.tagged_tools:
                                if tagged_tool.tool_id == tool.id:
                                    break
                            else:
                                tta = self.app.model.ToolTagAssociation( tool_id=tool.id, tag_id=tag.id )
                                self.sa_session.add( tta )
                                self.sa_session.flush()
                # Allow for the same tool to be loaded into multiple places in the tool panel.  We have to handle
                # the case where the tool is contained in a repository installed from the tool shed, and the Galaxy
                # administrator has retrieved updates to the installed repository.  In this case, the tool may have
                # been updated, but the version was not changed, so the tool should always be reloaded here.  We used
                # to only load the tool if it's it was not found in self.tools_by_id, but performing that check did
                # not enable this scenario.
                self.tools_by_id[ tool.id ] = tool
                if load_panel_dict:
                    self.__add_tool_to_tool_panel( tool, panel_dict, section=isinstance( panel_dict, ToolSection ) )
            # Always load the tool into the integrated_panel_dict, or it will not be included in the integrated_tool_panel.xml file.
            if key in integrated_panel_dict or index is None:
                integrated_panel_dict[ key ] = tool
            else:
                integrated_panel_dict.insert( index, key, tool )
        except:
            log.exception( "Error reading tool from path: %s" % path )

    def load_workflow_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
        try:
            # TODO: should id be encoded?
            workflow_id = elem.get( 'id' )
            workflow = self.load_workflow( workflow_id )
            self.workflows_by_id[ workflow_id ] = workflow
            key = 'workflow_' + workflow_id
            if load_panel_dict:
                panel_dict[ key ] = workflow
            # Always load workflows into the integrated_panel_dict.
            if key in integrated_panel_dict or index is None:
                integrated_panel_dict[ key ] = workflow
            else:
                integrated_panel_dict.insert( index, key, workflow )
        except:
            log.exception( "Error loading workflow: %s" % workflow_id )

    def load_label_tag_set( self, elem, panel_dict, integrated_panel_dict, load_panel_dict, index=None ):
        label = ToolSectionLabel( elem )
        key = 'label_' + label.id
        if load_panel_dict:
            panel_dict[ key ] = label
        if key in integrated_panel_dict or index is None:
            integrated_panel_dict[ key ] = label
        else:
            integrated_panel_dict.insert( index, key, label )

    def load_section_tag_set( self, elem, tool_path, load_panel_dict, index=None ):
        key = elem.get( "id" )
        if key in self.tool_panel:
            section = self.tool_panel[ key ]
            elems = section.elems
        else:
            section = ToolSection( elem )
            elems = section.elems
        if key in self.integrated_tool_panel:
            integrated_section = self.integrated_tool_panel[ key ]
            integrated_elems = integrated_section.elems
        else:
            integrated_section = ToolSection( elem )
            integrated_elems = integrated_section.elems
        for sub_index, sub_elem in enumerate( elem ):
            if sub_elem.tag == 'tool':
                self.load_tool_tag_set( sub_elem, elems, integrated_elems, tool_path, load_panel_dict, guid=sub_elem.get( 'guid' ), index=sub_index )
            elif sub_elem.tag == 'workflow':
                self.load_workflow_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
            elif sub_elem.tag == 'label':
                self.load_label_tag_set( sub_elem, elems, integrated_elems, load_panel_dict, index=sub_index )
        if load_panel_dict:
            self.tool_panel[ key ] = section
        # Always load sections into the integrated_tool_panel.
        if key in self.integrated_tool_panel or index is None:
            self.integrated_tool_panel[ key ] = integrated_section
        else:
            self.integrated_tool_panel.insert( index, key, integrated_section )

    def load_tool( self, config_file, guid=None, repository_id=None, **kwds ):
        """Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
        # Parse XML configuration file and get the root element
        tree = load_tool( config_file )
        root = tree.getroot()
        # Allow specifying a different tool subclass to instantiate
        if root.find( "type" ) is not None:
            type_elem = root.find( "type" )
            module = type_elem.get( 'module', 'galaxy.tools' )
            cls = type_elem.get( 'class' )
            mod = __import__( module, globals(), locals(), [cls] )
            ToolClass = getattr( mod, cls )
        elif root.get( 'tool_type', None ) is not None:
            ToolClass = tool_types.get( root.get( 'tool_type' ) )
        else:
            ToolClass = Tool
        return ToolClass( config_file, root, self.app, guid=guid, repository_id=repository_id, **kwds )

    def reload_tool_by_id( self, tool_id ):
        """
        Attempt to reload the tool identified by 'tool_id', if successful
        replace the old tool.
        """
        if tool_id not in self.tools_by_id:
            message = "No tool with id %s" % tool_id
            status = 'error'
        else:
            old_tool = self.tools_by_id[ tool_id ]
            new_tool = self.load_tool( old_tool.config_file )
            # The tool may have been installed from a tool shed, so set the tool shed attributes.
            # Since the tool version may have changed, we don't override it here.
            new_tool.id = old_tool.id
            new_tool.guid = old_tool.guid
            new_tool.tool_shed = old_tool.tool_shed
            new_tool.repository_name = old_tool.repository_name
            new_tool.repository_owner = old_tool.repository_owner
            new_tool.installed_changeset_revision = old_tool.installed_changeset_revision
            new_tool.old_id = old_tool.old_id
            # Replace old_tool with new_tool in self.tool_panel
            tool_key = 'tool_' + tool_id
            for key, val in self.tool_panel.items():
                if key == tool_key:
                    self.tool_panel[ key ] = new_tool
                    break
                elif key.startswith( 'section' ):
                    if tool_key in val.elems:
                        self.tool_panel[ key ].elems[ tool_key ] = new_tool
                        break
            self.tools_by_id[ tool_id ] = new_tool
            message = "Reloaded the tool:<br/>"
            message += "<b>name:</b> %s<br/>" % old_tool.name
            message += "<b>id:</b> %s<br/>" % old_tool.id
            message += "<b>version:</b> %s" % old_tool.version
            status = 'done'
        return message, status

    def remove_tool_by_id( self, tool_id ):
        """
        Attempt to remove the tool identified by 'tool_id'.
        """
        if tool_id not in self.tools_by_id:
            message = "No tool with id %s" % tool_id
            status = 'error'
        else:
            tool = self.tools_by_id[ tool_id ]
            del self.tools_by_id[ tool_id ]
            tool_key = 'tool_' + tool_id
            for key, val in self.tool_panel.items():
                if key == tool_key:
                    del self.tool_panel[ key ]
                    break
                elif key.startswith( 'section' ):
                    if tool_key in val.elems:
                        del self.tool_panel[ key ].elems[ tool_key ]
                        break
            if tool_id in self.data_manager_tools:
                del self.data_manager_tools[ tool_id ]
            #TODO: do we need to manually remove from the integrated panel here?
            message = "Removed the tool:<br/>"
            message += "<b>name:</b> %s<br/>" % tool.name
            message += "<b>id:</b> %s<br/>" % tool.id
            message += "<b>version:</b> %s" % tool.version
            status = 'done'
        return message, status

    def load_workflow( self, workflow_id ):
        """
        Return an instance of 'Workflow' identified by `id`,
        which is encoded in the tool panel.
        """
        id = self.app.security.decode_id( workflow_id )
        stored = self.app.model.context.query( self.app.model.StoredWorkflow ).get( id )
        return stored.latest_workflow

    def init_dependency_manager( self ):
        self.dependency_manager = build_dependency_manager( self.app.config )

    @property
    def sa_session( self ):
        """
        Returns a SQLAlchemy session
        """
        return self.app.model.context

    def to_dict( self, trans, in_panel=True, **kwds ):
        """
        to_dict toolbox.
        """

        context = Bunch( toolbox=self, trans=trans, **kwds )
        if in_panel:
            panel_elts = [ val for val in self.tool_panel.itervalues() ]

            filters = self.filter_factory.build_filters( trans, **kwds )

            filtered_panel_elts = []
            for index, elt in enumerate( panel_elts ):
                elt = _filter_for_panel( elt, filters, context )
                if elt:
                    filtered_panel_elts.append( elt )
            panel_elts = filtered_panel_elts

            # Produce panel.
            rval = []
            kwargs = dict(
                trans=trans,
                link_details=True
            )
            for elt in panel_elts:
                rval.append( to_dict_helper( elt, kwargs ) )
        else:
            tools = []
            for id, tool in self.tools_by_id.items():
                tools.append( tool.to_dict( trans, link_details=True ) )
            rval = tools

        return rval


def _filter_for_panel( item, filters, context ):
    """
    Filters tool panel elements so that only those that are compatible
    with provided filters are kept.
    """
    def _apply_filter( filter_item, filter_list ):
        for filter_method in filter_list:
            if not filter_method( context, filter_item ):
                return False
        return True
    if isinstance( item, Tool ):
        if _apply_filter( item, filters[ 'tool' ] ):
            return item
    elif isinstance( item, ToolSectionLabel ):
        if _apply_filter( item, filters[ 'label' ] ):
            return item
    elif isinstance( item, ToolSection ):
        # Filter section item-by-item. Only show a label if there are
        # non-filtered tools below it.

        if _apply_filter( item, filters[ 'section' ] ):
            cur_label_key = None
            tools_under_label = False
            filtered_elems = item.elems.copy()
            for key, section_item in item.elems.items():
                if isinstance( section_item, Tool ):
                    # Filter tool.
                    if _apply_filter( section_item, filters[ 'tool' ] ):
                        tools_under_label = True
                    else:
                        del filtered_elems[ key ]
                elif isinstance( section_item, ToolSectionLabel ):
                    # If there is a label and it does not have tools,
                    # remove it.
                    if ( cur_label_key and not tools_under_label ) or not _apply_filter( section_item, filters[ 'label' ] ):
                        del filtered_elems[ cur_label_key ]

                    # Reset attributes for new label.
                    cur_label_key = key
                    tools_under_label = False

            # Handle last label.
            if cur_label_key and not tools_under_label:
                del filtered_elems[ cur_label_key ]

            # Only return section if there are elements.
            if len( filtered_elems ) != 0:
                copy = item.copy()
                copy.elems = filtered_elems
                return copy

    return None


class ToolSection( object, Dictifiable ):
    """
    A group of tools with similar type/purpose that will be displayed as a
    group in the user interface.
    """

    dict_collection_visible_keys = ( 'id', 'name', 'version' )

    def __init__( self, elem=None ):
        f = lambda elem, val: elem is not None and elem.get( val ) or ''
        self.name = f( elem, 'name' )
        self.id = f( elem, 'id' )
        self.version = f( elem, 'version' )
        self.elems = odict()

    def copy( self ):
        copy = ToolSection()
        copy.name = self.name
        copy.id = self.id
        copy.version = self.version
        copy.elems = self.elems.copy()
        return copy

    def to_dict( self, trans, link_details=False ):
        """ Return a dict that includes section's attributes. """

        section_dict = super( ToolSection, self ).to_dict()
        section_elts = []
        kwargs = dict(
            trans=trans,
            link_details=link_details
        )
        for elt in self.elems.values():
            section_elts.append( to_dict_helper( elt, kwargs ) )
        section_dict[ 'elems' ] = section_elts

        return section_dict


class ToolSectionLabel( object, Dictifiable ):
    """
    A label for a set of tools that can be displayed above groups of tools
    and sections in the user interface
    """

    dict_collection_visible_keys = ( 'id', 'text', 'version' )

    def __init__( self, elem ):
        self.text = elem.get( "text" )
        self.id = elem.get( "id" )
        self.version = elem.get( "version" ) or ''


class DefaultToolState( object ):
    """
    Keeps track of the state of a users interaction with a tool between
    requests. The default tool state keeps track of the current page (for
    multipage "wizard" tools) and the values of all
    """
    def __init__( self ):
        self.page = 0
        self.rerun_remap_job_id = None
        self.inputs = None

    def encode( self, tool, app, secure=True ):
        """
        Convert the data to a string
        """
        # Convert parameters to a dictionary of strings, and save curent
        # page in that dict
        value = params_to_strings( tool.inputs, self.inputs, app )
        value["__page__"] = self.page
        value["__rerun_remap_job_id__"] = self.rerun_remap_job_id
        value = json.dumps( value )
        # Make it secure
        if secure:
            a = hmac_new( app.config.tool_secret, value )
            b = binascii.hexlify( value )
            return "%s:%s" % ( a, b )
        else:
            return value

    def decode( self, value, tool, app, secure=True ):
        """
        Restore the state from a string
        """
        if secure:
            # Extract and verify hash
            a, b = value.split( ":" )
            value = binascii.unhexlify( b )
            test = hmac_new( app.config.tool_secret, value )
            assert a == test
        # Restore from string
        values = json_fix( json.loads( value ) )
        self.page = values.pop( "__page__" )
        if '__rerun_remap_job_id__' in values:
            self.rerun_remap_job_id = values.pop( "__rerun_remap_job_id__" )
        else:
            self.rerun_remap_job_id = None
        self.inputs = params_from_strings( tool.inputs, values, app, ignore_errors=True )


class ToolOutput( object, Dictifiable ):
    """
    Represents an output datasets produced by a tool. For backward
    compatibility this behaves as if it were the tuple::

      (format, metadata_source, parent)
    """

    dict_collection_visible_keys = ( 'name', 'format', 'label', 'hidden' )

    def __init__( self, name, format=None, format_source=None, metadata_source=None,
                  parent=None, label=None, filters=None, actions=None, hidden=False ):
        self.name = name
        self.format = format
        self.format_source = format_source
        self.metadata_source = metadata_source
        self.parent = parent
        self.label = label
        self.filters = filters or []
        self.actions = actions
        self.hidden = hidden

    # Tuple emulation

    def __len__( self ):
        return 3

    def __getitem__( self, index ):
        if index == 0:
            return self.format
        elif index == 1:
            return self.metadata_source
        elif index == 2:
            return self.parent
        else:
            raise IndexError( index )

    def __iter__( self ):
        return iter( ( self.format, self.metadata_source, self.parent ) )


class Tool( object, Dictifiable ):
    """
    Represents a computational tool that can be executed through Galaxy.
    """

    tool_type = 'default'
    requires_setting_metadata = True
    default_tool_action = DefaultToolAction
    dict_collection_visible_keys = ( 'id', 'name', 'version', 'description' )

    def __init__( self, config_file, root, app, guid=None, repository_id=None ):
        """Load a tool from the config named by `config_file`"""
        # Determine the full path of the directory where the tool config is
        self.config_file = config_file
        self.tool_dir = os.path.dirname( config_file )
        self.app = app
        self.repository_id = repository_id
        #setup initial attribute values
        self.inputs = odict()
        self.stdio_exit_codes = list()
        self.stdio_regexes = list()
        self.inputs_by_page = list()
        self.display_by_page = list()
        self.action = '/tool_runner/index'
        self.target = 'galaxy_main'
        self.method = 'post'
        self.check_values = True
        self.nginx_upload = False
        self.input_required = False
        self.display_interface = True
        self.require_login = False
        self.rerun = False
        # Define a place to keep track of all input   These
        # differ from the inputs dictionary in that inputs can be page
        # elements like conditionals, but input_params are basic form
        # parameters like SelectField objects.  This enables us to more
        # easily ensure that parameter dependencies like index files or
        # tool_data_table_conf.xml entries exist.
        self.input_params = []
        # Attributes of tools installed from Galaxy tool sheds.
        self.tool_shed = None
        self.repository_name = None
        self.repository_owner = None
        self.installed_changeset_revision = None
        # The tool.id value will be the value of guid, but we'll keep the
        # guid attribute since it is useful to have.
        self.guid = guid
        self…