/lib/galaxy/web/controllers/requests_common.py

from galaxy.web.base.controller import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.model.orm import *
from galaxy import model, util
from galaxy.util.odict import odict
from galaxy.web.form_builder import *
from galaxy.security.validate_user_input import validate_email
import logging, os, csv

log = logging.getLogger( __name__ )

class RequestsGrid( grids.Grid ):
    # Custom column types
    class NameColumn( grids.TextColumn ):
        def get_value( self, trans, grid, request ):
            return request.name
    class DescriptionColumn( grids.TextColumn ):
        def get_value(self, trans, grid, request):
            return request.desc
    class SamplesColumn( grids.GridColumn ):
        def get_value(self, trans, grid, request):
            return str( len( request.samples ) )
    class TypeColumn( grids.TextColumn ):
        def get_value( self, trans, grid, request ):
            return request.type.name
    class StateColumn( grids.StateColumn ):
        def get_value(self, trans, grid, request ):
            state = request.state
            if state == request.states.REJECTED:
                state_color = 'error'
            elif state == request.states.NEW:
                state_color = 'new'
            elif state == request.states.SUBMITTED:
                state_color = 'running'
            elif state == request.states.COMPLETE:
                state_color = 'ok'
            else:
                state_color = state
            return '<div class="count-box state-color-%s">%s</div>' % ( state_color, state )
        def filter( self, trans, user, query, column_filter ):
            """ Modify query to filter request by state. """
            if column_filter == "All":
                return query
            if column_filter:
                return query.join( model.RequestEvent.table ) \
                            .filter( self.model_class.table.c.id == model.RequestEvent.table.c.request_id ) \
                            .filter( model.RequestEvent.table.c.state == column_filter ) \
                            .filter( model.RequestEvent.table.c.id.in_( select( columns=[ func.max( model.RequestEvent.table.c.id ) ],
                                                                                from_obj=model.RequestEvent.table,
                                                                                group_by=model.RequestEvent.table.c.request_id ) ) )
        
    # Grid definition
    title = "Sequencing Requests"
    template = "requests/grid.mako"
    model_class = model.Request
    default_sort_key = "-update_time"
    num_rows_per_page = 50
    use_paging = True
    default_filter = dict( state="All", deleted="False" )
    columns = [
        NameColumn( "Name", 
                    key="name", 
                    link=( lambda item: dict( operation="view_request", id=item.id ) ),
                    attach_popup=True, 
                    filterable="advanced" ),
        DescriptionColumn( "Description",
                           key='desc',
                           filterable="advanced" ),
        SamplesColumn( "Samples", 
                       link=( lambda item: iff( item.deleted, None, dict( operation="edit_samples", id=item.id ) ) ) ),
        TypeColumn( "Type",
                    link=( lambda item: iff( item.deleted, None, dict( operation="view_type", id=item.type.id ) ) ) ),
        grids.GridColumn( "Last Updated", key="update_time", format=time_ago ),
        grids.DeletedColumn( "Deleted", 
                             key="deleted", 
                             visible=False, 
                             filterable="advanced" ),
        StateColumn( "State", 
                     key='state',
                     filterable="advanced",
                     link=( lambda item: iff( item.deleted, None, dict( operation="view_request_history", id=item.id ) ) )
                   )
    ]
    columns.append( grids.MulticolFilterColumn( "Search", 
                                                cols_to_filter=[ columns[0], columns[1] ], 
                                                key="free-text-search",
                                                visible=False,
                                                filterable="standard" ) )
    operations = [
        grids.GridOperation( "Submit",
                             allow_multiple=False,
                             condition=( lambda item: not item.deleted and item.is_unsubmitted and item.samples ),
                             confirm="Samples cannot be added to this request after it is submitted. Click OK to submit."  )
        ]

class RequestsCommon( BaseUIController, UsesFormDefinitions ):
    @web.json
    def sample_state_updates( self, trans, ids=None, states=None ):
        # Avoid caching
        trans.response.headers['Pragma'] = 'no-cache'
        trans.response.headers['Expires'] = '0'
        # Create new HTML for any that have changed
        rval = {}
        if ids is not None and states is not None:
            ids = map( int, ids.split( "," ) )
            states = states.split( "," )
            for id, state in zip( ids, states ):
                sample = trans.sa_session.query( self.app.model.Sample ).get( id )
                if sample.state.name != state:
                    rval[ id ] = { "state": sample.state.name,
                                   "html_state": unicode( trans.fill_template( "requests/common/sample_state.mako",
                                                                               sample=sample),
                                                                               'utf-8' ) }
        return rval
    @web.json
    def sample_datasets_updates( self, trans, ids=None, datasets=None ):
        # Avoid caching
        trans.response.headers['Pragma'] = 'no-cache'
        trans.response.headers['Expires'] = '0'
        # Create new HTML for any that have changed
        rval = {}
        if ids is not None and datasets is not None:
            ids = map( int, ids.split( "," ) )
            number_of_datasets_list = map(int, datasets.split( "," ) )
            for id, number_of_datasets in zip( ids, number_of_datasets_list ):
                sample = trans.sa_session.query( self.app.model.Sample ).get( id )
                if len(sample.datasets) != number_of_datasets:
                    rval[ id ] = { "datasets": len( sample.datasets ),
                                   "html_datasets": unicode( trans.fill_template( "requests/common/sample_datasets.mako",
                                                                                  sample=sample),
                                                                                  'utf-8' ) }
        return rval
    @web.json
    def dataset_transfer_status_updates( self, trans, ids=None, transfer_status_list=None ):
        # Avoid caching
        trans.response.headers['Pragma'] = 'no-cache'
        trans.response.headers['Expires'] = '0'
        # Create new HTML for any that have changed
        rval = {}
        if ids is not None and transfer_status_list is not None:
            ids = ids.split( "," )
            transfer_status_list = transfer_status_list.split( "," )
            for id, transfer_status in zip( ids, transfer_status_list ):
                sample_dataset = trans.sa_session.query( self.app.model.SampleDataset ).get( trans.security.decode_id( id ) )
                if sample_dataset.status != transfer_status:
                    rval[ id ] = { "status": sample_dataset.status,
                                   "html_status": unicode( trans.fill_template( "requests/common/sample_dataset_transfer_status.mako",
                                                                                sample_dataset=sample_dataset),
                                                                                'utf-8' ) }
        return rval
    @web.expose
    @web.require_login( "create sequencing requests" )
    def create_request( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        message = util.restore_text( params.get( 'message', '' ) )
        status = params.get( 'status', 'done' )
        request_type_id = params.get( 'request_type_id', 'none' )
        if request_type_id != 'none':
            request_type = trans.sa_session.query( trans.model.RequestType ).get( trans.security.decode_id( request_type_id ) )
        else:
            request_type = None
        # user_id will not be 'none' if an admin user is submitting this request on behalf of another user
        # and they selected that user's id from the user_id SelectField.
        user_id_encoded = True
        user_id = params.get( 'user_id', 'none' )
        if user_id != 'none':
            try:
                user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) )
            except TypeError, e:
                # We must have an email address rather than an encoded user id
                # This is because the galaxy.base.js creates a search+select box 
                # when there are more than 20 items in a SelectField.
                user = trans.sa_session.query( trans.model.User ) \
                                       .filter( trans.model.User.table.c.email==util.restore_text( user_id ) ) \
                                       .first()
                user_id_encoded = False
                
        elif not is_admin:
            user = trans.user
        else:
            user = None
        if params.get( 'create_request_button', False ) or params.get( 'add_sample_button', False ):
            name = util.restore_text( params.get( 'name', '' ) )
            if is_admin and user_id == 'none':
                message = 'Select the user on behalf of whom you are submitting this request.'
                status = 'error'
            elif user is None:
                message = 'Invalid user ID (%s)' % str(user_id)
                status = 'error'
            # when creating a request from the user perspective, check if the 
            # user has access permission to this request_type 
            elif cntrller == 'requests' and not trans.app.security_agent.can_access_request_type( user.all_roles(), request_type ):
                message = '%s does not have access permission to the "%s" request type.' % ( user.email, request_type.name )
                status = 'error'
            elif not name:
                message = 'Enter the name of the request.'
                status = 'error'
            else:
                request = self.__save_request( trans, cntrller, **kwd )
                message = 'The sequencing request has been created.'
                if params.get( 'create_request_button', False ):
                    return trans.response.send_redirect( web.url_for( controller=cntrller,
                                                                      action='browse_requests',
                                                                      message=message ,
                                                                      status='done' ) )
                elif params.get( 'add_sample_button', False ):
                    request_id = trans.security.encode_id( request.id )
                    return self.add_sample( trans, cntrller, request_id, **kwd )
        request_type_select_field = self.__build_request_type_id_select_field( trans, selected_value=request_type_id )
        # Widgets to be rendered on the request form
        widgets = []
        if request_type is not None or status == 'error':
            # Either the user selected a request_type or an error exists on the form.
            widgets.append( dict( label='Name of the Experiment', 
                                  widget=TextField( 'name', 40, util.restore_text( params.get( 'name', ''  ) ) ), 
                                  helptext='(Required)') )
            widgets.append( dict( label='Description', 
                                  widget=TextField( 'desc', 40, util.restore_text( params.get( 'desc', ''  ) )), 
                                  helptext='(Optional)') )
            if request_type is not None:
                widgets += request_type.request_form.get_widgets( user, **kwd )
        # In case there is an error on the form, make sure to populate widget fields with anything the user
        # may have already entered.
        widgets = self.populate_widgets_from_kwd( trans, widgets, **kwd )
        if request_type is not None or status == 'error':
            # Either the user selected a request_type or an error exists on the form.
            if is_admin:
                if not user_id_encoded and user:
                    selected_user_id = trans.security.encode_id( user.id )
                else:
                    selected_user_id = user_id
                user_widget = dict( label='Select user',
                                    widget=self.__build_user_id_select_field( trans, selected_value=selected_user_id ),
                                    helptext='Submit the request on behalf of the selected user (Required)')
                widgets = [ user_widget ] + widgets
        return trans.fill_template( '/requests/common/create_request.mako',
                                    cntrller=cntrller,
                                    request_type_select_field=request_type_select_field,
                                    request_type_select_field_selected=request_type_id,                               
                                    widgets=widgets,
                                    message=message,
                                    status=status )
    @web.expose
    @web.require_login( "view request" )
    def view_request( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        sample_state_id = params.get( 'sample_state_id', None )
        # Build a list of sample widgets (based on the attributes of each sample) for display.
        displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd )
        request_widgets = self.__get_request_widgets( trans, request.id )
        return trans.fill_template( '/requests/common/view_request.mako',
                                    cntrller=cntrller, 
                                    request=request,
                                    request_widgets=request_widgets,
                                    displayable_sample_widgets=displayable_sample_widgets,
                                    status=status,
                                    message=message )
    @web.expose
    @web.require_login( "edit sequencing requests" )
    def edit_basic_request_info( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        name = util.restore_text( params.get( 'name', '' ) )
        desc = util.restore_text( params.get( 'desc', ''  ) )
        if params.get( 'edit_basic_request_info_button', False ):
            if not name:
                status = 'error'
                message = 'Enter the name of the request'
            else:
                request = self.__save_request( trans, cntrller, request=request, **kwd )
                message = 'The changes made to request (%s) have been saved.' % request.name
        # Widgets to be rendered on the request form
        widgets = []
        widgets.append( dict( label='Name', 
                              widget=TextField( 'name', 40, request.name ), 
                              helptext='(Required)' ) )
        widgets.append( dict( label='Description', 
                              widget=TextField( 'desc', 40, request.desc ), 
                              helptext='(Optional)' ) )
        widgets = widgets + request.type.request_form.get_widgets( request.user, request.values.content, **kwd )
        # In case there is an error on the form, make sure to populate widget fields with anything the user
        # may have already entered.
        widgets = self.populate_widgets_from_kwd( trans, widgets, **kwd )
        return trans.fill_template( 'requests/common/edit_basic_request_info.mako',
                                    cntrller=cntrller,
                                    request_type=request.type,
                                    request=request,
                                    widgets=widgets,
                                    message=message,
                                    status=status )
    def __save_request( self, trans, cntrller, request=None, **kwd ):
        """
        Saves changes to an existing request, or creates a new 
        request if received request is None.
        """
        params = util.Params( kwd )
        request_type_id = params.get( 'request_type_id', None )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        if request is None:
            # We're creating a new request, so we need the associated request_type
            request_type = trans.sa_session.query( trans.model.RequestType ).get( trans.security.decode_id( request_type_id ) )
            if is_admin:
                # The admin user is creating a request on behalf of another user
                user_id = params.get( 'user_id', '' )
                user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) )
            else:
                user = trans.user
        else:
            # We're saving changes to an existing request
            user = request.user
            request_type = request.type
        name = util.restore_text( params.get( 'name', '' ) )
        desc = util.restore_text( params.get( 'desc', '' ) )
        notification = dict( email=[ user.email ], sample_states=[ request_type.final_sample_state.id ], body='', subject='' )
        values = self.get_form_values( trans, user, request_type.request_form, **kwd )
        if request is None:
            form_values = trans.model.FormValues( request_type.request_form, values )
            trans.sa_session.add( form_values )
            # We're creating a new request
            request = trans.model.Request( name, desc, request_type, user, form_values, notification )
            trans.sa_session.add( request )
            trans.sa_session.flush()
            trans.sa_session.refresh( request )
            # Create an event with state 'New' for this new request
            comment = "Sequencing request created by %s" % trans.user.email
            if request.user != trans.user:
                comment += " on behalf of %s." % request.user.email
            event = trans.model.RequestEvent( request, request.states.NEW, comment )
            trans.sa_session.add( event )
            trans.sa_session.flush()
        else:
            # We're saving changes to an existing request
            request.name = name
            request.desc = desc
            request.type = request_type
            request.user = user
            request.notification = notification
            request.values.content = values
            trans.sa_session.add( request )
            trans.sa_session.add( request.values )
            trans.sa_session.flush()
        return request
    @web.expose
    @web.require_login( "submit sequencing requests" )
    def submit_request( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        request_id = params.get( 'id', None )
        message = util.restore_text( params.get( 'message', '' ) )
        status = util.restore_text( params.get( 'status', 'done' ) )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        ok = True
        if not request.samples:
            message = 'Add at least 1 sample to this request before submitting.'
            ok = False
        if ok:
            message = self.__validate_request( trans, cntrller, request )
        if message or not ok:
            return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                              action='edit_basic_request_info',
                                                              cntrller=cntrller,
                                                              id = request_id,
                                                              status='error',
                                                              message=message ) )
        # Change the request state to 'Submitted'
        comment = "Sequencing request submitted by %s" % trans.user.email
        if request.user != trans.user:
            comment += " on behalf of %s." % request.user.email
        event = trans.model.RequestEvent( request, request.states.SUBMITTED, comment )
        trans.sa_session.add( event )
        # Change the state of each of the samples of this request
        # request.type.states is the list of SampleState objects configured
        # by the admin for this RequestType.
        trans.sa_session.add( event )
        trans.sa_session.flush()
        # Samples will not have an associated SampleState until the request is submitted, at which
        # time all samples of the request will be set to the first SampleState configured for the
        # request's RequestType configured by the admin.
        initial_sample_state_after_request_submitted = request.type.states[0]
        for sample in request.samples:
            event_comment = 'Sequencing request submitted and sample state set to %s.' % request.type.states[0].name
            event = trans.model.SampleEvent( sample,
                                             initial_sample_state_after_request_submitted,
                                             event_comment )
            trans.sa_session.add( event )
        trans.sa_session.add( request )
        trans.sa_session.flush()
        request.send_email_notification( trans, initial_sample_state_after_request_submitted )
        message = 'The sequencing request has been submitted.'
        # show the request page after submitting the request 
        return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                          action='view_request',
                                                          cntrller=cntrller,
                                                          id=request_id,
                                                          status=status,
                                                          message=message ) )
    @web.expose
    @web.require_login( "edit samples" )
    def edit_samples( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        if params.get( 'cancel_changes_button', False ):
            return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                                          action='edit_samples',
                                                                          cntrller=cntrller,
                                                                          id=request_id ) )
        libraries = trans.app.security_agent.get_accessible_libraries( trans, request.user )
        # Build a list of sample widgets (based on the attributes of each sample) for display.
        displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd )
        encoded_selected_sample_ids = self.__get_encoded_selected_sample_ids( trans, request, **kwd )
        sample_operation = params.get( 'sample_operation', 'none' )
        def handle_error( **kwd ):
            kwd[ 'status' ] = 'error'
            return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                              action='edit_samples',
                                                              cntrller=cntrller,
                                                              **kwd ) )
        if not encoded_selected_sample_ids and sample_operation != 'none':
            # Probably occurred due to refresh_on_change...is there a better approach?
            kwd[ 'sample_operation' ] = 'none'
            message = 'Select at least one sample before selecting an operation.'
            kwd[ 'message' ] = message
            handle_error( **kwd )
        if params.get( 'save_samples_button', False ):
            if encoded_selected_sample_ids:
                # We need the list of displayable_sample_widgets to include the same number
                # of objects that that request.samples has so that we can enumerate over each
                # list without problems.  We have to be careful here since the user may have
                # used the multi-select check boxes when editing sample widgets, but didn't
                # select all of them.  We'll first get the set of samples corresponding to the
                # checked sample ids.
                samples = []
                selected_samples = []
                for encoded_sample_id in encoded_selected_sample_ids:
                    sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( encoded_sample_id ) )
                    selected_samples.append( sample )
                # Now build the list of samples, inserting None for samples that have not been checked.
                for sample in request.samples:
                    if sample in selected_samples:
                        samples.append( sample )
                    else:
                        samples.append( None )
                # The __save_samples method requires sample_widgets, not sample objects, so we'll get what we
                # need by calling __get_sample_widgets().  However, we need to take care here because __get_sample_widgets()
                # is used to populate the sample widget dicts from kwd, and the method assumes that a None object in the 
                # received list of samples should be populated from the db.  Since we're just re-using the method here to
                # change our list of samples into a list of sample widgets, we'll need to make sure to keep track of our
                # None objects.
                sample_widgets = [ obj for obj in samples ]
                sample_widgets = self.__get_sample_widgets( trans, request, sample_widgets, **kwd )
                # Replace each sample widget dict with a None object if necessary
                for index, obj in enumerate( samples ):
                    if obj is None:
                        sample_widgets[ index ] = None
            else:
                sample_widgets = displayable_sample_widgets
            return self.__save_samples( trans, cntrller, request, sample_widgets, saving_new_samples=False, **kwd )
        request_widgets = self.__get_request_widgets( trans, request.id )
        sample_copy_select_field = self.__build_copy_sample_select_field( trans, displayable_sample_widgets )
        libraries_select_field, folders_select_field = self.__build_library_and_folder_select_fields( trans,
                                                                                                      request.user,
                                                                                                      'sample_operation',
                                                                                                      libraries,
                                                                                                      None,
                                                                                                      **kwd )
        sample_operation_select_field = self.__build_sample_operation_select_field( trans, is_admin, request, sample_operation )
        sample_state_id = params.get( 'sample_state_id', None )
        sample_state_id_select_field = self.__build_sample_state_id_select_field( trans, request, sample_state_id )
        return trans.fill_template( '/requests/common/edit_samples.mako',
                                    cntrller=cntrller, 
                                    request=request,
                                    encoded_selected_sample_ids=encoded_selected_sample_ids,
                                    request_widgets=request_widgets,
                                    displayable_sample_widgets=displayable_sample_widgets,
                                    sample_copy_select_field=sample_copy_select_field, 
                                    libraries=libraries,
                                    sample_operation_select_field=sample_operation_select_field,
                                    libraries_select_field=libraries_select_field,
                                    folders_select_field=folders_select_field,
                                    sample_state_id_select_field=sample_state_id_select_field,
                                    status=status,
                                    message=message )
    @web.expose
    def update_sample_state(self, trans, cntrller, sample_ids, new_state, comment=None ):
        for sample_id in sample_ids:
            try:
                sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) )
            except:
                if cntrller == 'api':
                    trans.response.status = 400
                    return "Invalid sample id ( %s ) specified, unable to decode." % str( sample_id )
                else:
                    return invalid_id_redirect( trans, cntrller, sample_id, 'sample' )
            if comment is None:
                comment = 'Sample state set to %s' % str( new_state )
            event = trans.model.SampleEvent( sample, new_state, comment )
            trans.sa_session.add( event )
            trans.sa_session.flush()
        if cntrller == 'api':
            return 200, 'Done'        
    @web.expose
    @web.require_login( "delete sequencing requests" )
    def delete_request( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        id_list = util.listify( kwd.get( 'id', '' ) )
        message = util.restore_text( params.get( 'message', '' ) )
        status = util.restore_text( params.get( 'status', 'done' ) )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        num_deleted = 0
        not_deleted = []
        for id in id_list:
            ok_for_now = True
            try:
                # This block will handle bots that do not send valid request ids.
                request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( id ) )
            except:
                ok_for_now = False
            if ok_for_now:
                # We will only allow the request to be deleted by a non-admin user if not request.submitted
                if is_admin or not request.is_submitted:
                    request.deleted = True
                    trans.sa_session.add( request )
                    # Delete all the samples belonging to this request
                    for s in request.samples:
                        s.deleted = True
                        trans.sa_session.add( s )
                    comment = "Sequencing request marked deleted by %s." % trans.user.email
                    # There is no DELETED state for a request, so keep the current request state
                    event = trans.model.RequestEvent( request, request.state, comment )
                    trans.sa_session.add( event )
                    trans.sa_session.flush()
                    num_deleted += 1
                else:
                    not_deleted.append( request )
        message += '%i requests have been deleted.' % num_deleted
        if not_deleted:
            message += '  Contact the administrator to delete the following submitted requests: '
            for request in not_deleted:
                message += '%s, ' % request.name
            message = message.rstrip( ', ' )
        return trans.response.send_redirect( web.url_for( controller=cntrller,
                                                          action='browse_requests',
                                                          status=status,
                                                          message=message ) )
    @web.expose
    @web.require_login( "undelete sequencing requests" )
    def undelete_request( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        id_list = util.listify( kwd.get( 'id', '' ) )
        message = util.restore_text( params.get( 'message', '' ) )
        status = util.restore_text( params.get( 'status', 'done' ) )
        num_undeleted = 0
        for id in id_list:
            ok_for_now = True
            try:
                # This block will handle bots that do not send valid request ids.
                request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( id ) )
            except:
                ok_for_now = False
            if ok_for_now:
                request.deleted = False
                trans.sa_session.add( request )
                # Undelete all the samples belonging to this request
                for s in request.samples:
                    s.deleted = False
                    trans.sa_session.add( s )
                comment = "Sequencing request marked undeleted by %s." % trans.user.email
                event = trans.model.RequestEvent( request, request.state, comment )
                trans.sa_session.add( event )
                trans.sa_session.flush()
                num_undeleted += 1
        message += '%i requests have been undeleted.' % num_undeleted
        return trans.response.send_redirect( web.url_for( controller=cntrller,
                                                          action='browse_requests',
                                                          status=status,
                                                          message=message ) )
    @web.expose
    @web.require_login( "sequencing request history" )
    def view_request_history( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        return trans.fill_template( '/requests/common/view_request_history.mako', 
                                    cntrller=cntrller,
                                    request=request )
    @web.expose
    @web.require_login( "edit email notification settings" )
    def edit_email_settings( self, trans, cntrller, **kwd ):
        """
        Allow for changing the email notification settings where email is sent to a list of users
        whenever the request state changes to one selected for notification.
        """
        params = util.Params( kwd )
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        email_address = CheckboxField.is_checked( params.get( 'email_address', '' ) )
        additional_email_addresses = params.get( 'additional_email_addresses', '' )
        # Get the list of checked sample state CheckBoxFields
        checked_sample_states = []
        for index, sample_state in enumerate( request.type.states ):
            if CheckboxField.is_checked( params.get( 'sample_state_%i' % sample_state.id, '' ) ):
                checked_sample_states.append( sample_state.id )
        if additional_email_addresses:
            additional_email_addresses = additional_email_addresses.split( '\r\n' )
        if email_address or additional_email_addresses:
            # The user added 1 or more email addresses
            email_addresses = []
            if email_address:
                email_addresses.append( request.user.email )
            for email_address in additional_email_addresses:
                email_addresses.append( util.restore_text( email_address ) )
            # Make sure email addresses are valid
            err_msg = ''
            for email_address in email_addresses:
                err_msg += validate_email( trans, email_address, check_dup=False )
            if err_msg:
                status = 'error'
                message += err_msg
            else:
                request.notification = dict( email=email_addresses,
                                             sample_states=checked_sample_states, 
                                             body='',
                                             subject='' )
        else:
            # The user may have eliminated email addresses that were previously set
            request.notification = None
            if checked_sample_states:
                message = 'All sample states have been unchecked since no email addresses have been selected or entered.  '
        trans.sa_session.add( request )
        trans.sa_session.flush()
        trans.sa_session.refresh( request )
        message += 'The changes made to the email notification settings have been saved.'
        return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                          action='edit_basic_request_info',
                                                          cntrller=cntrller,
                                                          id=request_id,
                                                          message=message ,
                                                          status=status ) )
    @web.expose
    @web.require_login( "update sequencing request state" )
    def update_request_state( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        message = params.get( 'message', '' )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'request_id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        # Make sure all the samples of the current request have the same state
        common_state = request.samples_have_common_state
        if not common_state:
            # If the current request state is complete and one of its samples moved from
            # the final sample state, then move the request state to In-progress
            if request.is_complete:
                message = "At least 1 sample state moved from the final sample state, so now the request's state is (%s)" % request.states.SUBMITTED
                event = trans.model.RequestEvent( request, request.states.SUBMITTED, message )
                trans.sa_session.add( event )
                trans.sa_session.flush()
            if cntrller == 'api':
                return 200, message
        else:
            final_state = False
            request_type_state = request.type.final_sample_state
            if common_state.id == request_type_state.id:
                # since all the samples are in the final state, change the request state to 'Complete'
                comment = "All samples of this sequencing request are in the final sample state (%s). " % request_type_state.name
                state = request.states.COMPLETE
                final_state = True
            else:
                comment = "All samples of this sequencing request are in the (%s) sample state. " % common_state.name
                state = request.states.SUBMITTED
            event = trans.model.RequestEvent( request, state, comment )
            trans.sa_session.add( event )
            trans.sa_session.flush()
            # See if an email notification is configured to be sent when the samples are in this state.
            retval = request.send_email_notification( trans, common_state, final_state )
            if retval:
                message = comment + retval
            else:
                message = comment
            if cntrller == 'api':
                return 200, message
        return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                          action='edit_samples',
                                                          cntrller=cntrller,
                                                          id=request_id,
                                                          status=status,
                                                          message=message ) )
    @web.expose
    @web.require_login( "find samples" )
    def find_samples( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        samples_list = []
        results = ''
        if params.get( 'find_samples_button', False ):
            search_string = kwd.get( 'search_box', ''  )
            search_type = params.get( 'search_type', ''  )
            request_states = util.listify( params.get( 'request_states', '' ) )
            samples = []
            if search_type == 'bar_code':
                samples = trans.sa_session.query( trans.model.Sample ) \
                                          .filter( and_( trans.model.Sample.table.c.deleted==False,
                                                         func.lower( trans.model.Sample.table.c.bar_code ).like( "%" + search_string.lower() + "%" ) ) ) \
                                          .order_by( trans.model.Sample.table.c.create_time.desc() )
            elif search_type == 'sample name':
                samples = trans.sa_session.query( trans.model.Sample ) \
                                          .filter( and_( trans.model.Sample.table.c.deleted==False,
                                                         func.lower( trans.model.Sample.table.c.name ).like( "%" + search_string.lower() + "%" ) ) ) \
                                          .order_by( trans.model.Sample.table.c.create_time.desc() )
            elif search_type == 'dataset':
                samples = trans.sa_session.query( trans.model.Sample ) \
                                          .filter( and_( trans.model.Sample.table.c.deleted==False,
                                                         trans.model.SampleDataset.table.c.sample_id==trans.model.Sample.table.c.id,
                                                         func.lower( trans.model.SampleDataset.table.c.name ).like( "%" + search_string.lower() + "%" ) ) ) \
                                          .order_by( trans.model.Sample.table.c.create_time.desc() )
            elif search_type == 'form value':
                samples = []
                if search_string.find('=') != -1:
                    field_label, field_value = search_string.split('=')
                    all_samples = trans.sa_session.query( trans.model.Sample ) \
                                              .filter( trans.model.Sample.table.c.deleted==False ) \
                                              .order_by( trans.model.Sample.table.c.create_time.desc() )
                    for sample in all_samples:
                        # find the field in the sample form with the given label
                        for field in sample.request.type.sample_form.fields:
                            if field_label == field['label']:
                                # check if the value is equal to the value in the search string
                                if sample.values.content[ field['name'] ] == field_value:
                                    samples.append( sample )
            if is_admin:
                for s in samples:
                    if not s.request.deleted and s.request.state in request_states:
                        samples_list.append( s )
            else:
                for s in samples:
                    if s.request.user.id == trans.user.id and s.request.state in request_states and not s.request.deleted:
                        samples_list.append( s )
            results = 'There are %i samples matching the search parameters.' % len( samples_list )
        # Build the request_states SelectField
        selected_value = kwd.get( 'request_states', trans.model.Request.states.SUBMITTED )
        states = [ v for k, v in trans.model.Request.states.items() ]
        request_states = build_select_field( trans,
                                             states,
                                             'self',
                                             'request_states',
                                             selected_value=selected_value,
                                             refresh_on_change=False,
                                             multiple=True,
                                             display='checkboxes' )
        # Build the search_type SelectField
        selected_value = kwd.get( 'search_type', 'sample name' )
        types = [ 'sample name', 'bar_code', 'dataset', 'form value' ]
        search_type = build_select_field( trans, types, 'self', 'search_type', selected_value=selected_value, refresh_on_change=False )
        # Build the search_box TextField
        search_box = TextField( 'search_box', 50, kwd.get('search_box', '' ) )
        return trans.fill_template( '/requests/common/find_samples.mako', 
                                    cntrller=cntrller,
                                    request_states=request_states,
                                    samples=samples_list,
                                    search_type=search_type,
                                    results=results,
                                    search_box=search_box )
    @web.expose
    @web.require_login( "sample events" )
    def view_sample_history( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        status = params.get( 'status', 'done' )
        message = util.restore_text( params.get( 'message', '' ) )
        sample_id = params.get( 'sample_id', None )
        try:
            sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, sample_id, 'sample' )
        return trans.fill_template( '/requests/common/view_sample_history.mako', 
                                    cntrller=cntrller,
                                    sample=sample )
    @web.expose
    @web.require_login( "add samples" )
    def add_samples( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        request_id = params.get( 'id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        libraries = trans.app.security_agent.get_accessible_libraries( trans, request.user )
        # Build a list of sample widgets (based on the attributes of each sample) for display.
        displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd )
        if params.get( 'import_samples_button', False ):
            # Import sample field values from a csv file
            # TODO: should this be a mapper?
            workflows = [ w.latest_workflow for w in trans.user.stored_workflows if not w.deleted ]
            return self.__import_samples( trans, cntrller, request, displayable_sample_widgets, libraries, workflows, **kwd )
        elif params.get( 'add_sample_button', False ):
            return self.add_sample( trans, cntrller, request_id, **kwd )
        elif params.get( 'save_samples_button', False ):
            return self.__save_samples( trans, cntrller, request, displayable_sample_widgets, saving_new_samples=True, **kwd )
        request_widgets = self.__get_request_widgets( trans, request.id )
        sample_copy_select_field = self.__build_copy_sample_select_field( trans, displayable_sample_widgets )
        libraries_select_field, folders_select_field = self.__build_library_and_folder_select_fields( trans,
                                                                                                      request.user,
                                                                                                      'sample_operation',
                                                                                                      libraries,
                                                                                                      None,
                                                                                                      **kwd )
        return trans.fill_template( '/requests/common/add_samples.mako',
                                    cntrller=cntrller, 
                                    request=request,
                                    request_widgets=request_widgets,
                                    displayable_sample_widgets=displayable_sample_widgets,
                                    sample_copy_select_field=sample_copy_select_field, 
                                    libraries=libraries,
                                    libraries_select_field=libraries_select_field,
                                    folders_select_field=folders_select_field,
                                    status=status,
                                    message=message )
    @web.expose
    @web.require_login( "add sample" )
    def add_sample( self, trans, cntrller, request_id, **kwd ):
        params = util.Params( kwd )
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        # Get the widgets for rendering the request form
        request_widgets = self.__get_request_widgets( trans, request.id )
        displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd )
        if not displayable_sample_widgets:
            # Form field names are zero-based.
            sample_index = 0
        else:
            sample_index = len( displayable_sample_widgets )
        if params.get( 'add_sample_button', False ):
            libraries = trans.app.security_agent.get_accessible_libraries( trans, request.user )
            num_samples_to_add = int( params.get( 'num_sample_to_copy', 1 ) )
            # See if the user has selected a sample to copy.
            copy_sample_index = int( params.get( 'copy_sample_index', -1 ) )
            for index in range( num_samples_to_add ):
                field_values = {}
                if copy_sample_index != -1:
                    # The user has selected a sample to copy.
                    library_id = displayable_sample_widgets[ copy_sample_index][ 'library_select_field' ].get_selected( return_value=True )
                    folder_id = displayable_sample_widgets[ copy_sample_index ][ 'folder_select_field' ].get_selected( return_value=True )
                    name = displayable_sample_widgets[ copy_sample_index ][ 'name' ] + '_%i' % ( len( displayable_sample_widgets ) + 1 )
                    history_id = displayable_sample_widgets[ copy_sample_index ][ 'history_select_field' ].get_selected( return_value=True )
                    workflow_id = displayable_sample_widgets[ copy_sample_index ][ 'workflow_select_field' ][0].get_selected( return_value=True )
                    # DBTODO Do something nicer with the workflow fieldset.  Remove [0] indexing and copy mappings as well.
                    for field_name in displayable_sample_widgets[ copy_sample_index ][ 'field_values' ]:
                        field_values[ field_name ] = ''
                else:
                    # The user has not selected a sample to copy, just adding a new generic sample.
                    library_id = None
                    folder_id = None
                    history_id = None
                    workflow_id = None
                    name = 'Sample_%i' % ( len( displayable_sample_widgets ) + 1 )
                    for field in request.type.sample_form.fields:
                        field_values[ field[ 'name' ] ] = ''
                # Build the library_select_field and folder_select_field for the new sample being added.
                library_select_field, folder_select_field = self.__build_library_and_folder_select_fields( trans,
                                                                                                           user=request.user, 
                                                                                                           sample_index=len( displayable_sample_widgets ), 
                                                                                                           libraries=libraries,
                                                                                                           sample=None, 
                                                                                                           library_id=library_id,
                                                                                                           folder_id=folder_id,
                                                                                                           **kwd )
                history_select_field = self.__build_history_select_field( trans=trans,
                                                                          user=request.user,
                                                                          sample_index=len( displayable_sample_widgets ),
                                                                          history_id=history_id,
                                                                          **kwd )
                workflow_select_field = self.__build_workflow_select_field( trans=trans,
                                                                            user=request.user,
                                                                            request=request,
                                                                            sample_index=len( displayable_sample_widgets ),
                                                                            workflow_id=workflow_id,
                                                                            history_id=history_id,
                                                                            **kwd )
                # Append the new sample to the current list of samples for the request
                displayable_sample_widgets.append( dict( id=None,
                                                         name=name,
                                                         bar_code='',
                                                         library=None,
                                                         library_id=library_id,
                                                         history=None,
                                                         workflow=None,
                                                         history_select_field=history_select_field,
                                                         workflow_select_field=workflow_select_field,
                                                         folder=None,
                                                         folder_id=folder_id,
                                                         field_values=field_values,
                                                         library_select_field=library_select_field,
                                                         folder_select_field=folder_select_field ) )
        sample_copy_select_field = self.__build_copy_sample_select_field( trans, displayable_sample_widgets )
        return trans.fill_template( '/requests/common/add_samples.mako',
                                    cntrller=cntrller,
                                    request=request,
                                    request_widgets=request_widgets,
                                    displayable_sample_widgets=displayable_sample_widgets,
                                    sample_copy_select_field=sample_copy_select_field, 
                                    message=message,
                                    status=status )
    @web.expose
    @web.require_login( "view request" )
    def view_sample( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        sample_id = params.get( 'id', None )
        try:
            sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, sample_id, 'sample' )
        # See if we have any associated templates
        widgets = sample.get_template_widgets( trans )
        widget_fields_have_contents = self.widget_fields_have_contents( widgets )
        if is_admin:
            external_services = sample.populate_external_services( trans = trans )
        else:
            external_services = None
        return trans.fill_template( '/requests/common/view_sample.mako',
                                    cntrller=cntrller, 
                                    sample=sample,
                                    widgets=widgets,
                                    widget_fields_have_contents=widget_fields_have_contents,
                                    status=status,
                                    message=message,
                                    external_services=external_services )
    @web.expose
    @web.require_login( "delete sample from sequencing request" )
    def delete_sample( self, trans, cntrller, **kwd ):
        params = util.Params( kwd )
        status = params.get( 'status', 'done' )
        message = util.restore_text( params.get( 'message', '' ) )
        request_id = params.get( 'request_id', None )
        try:
            request = trans.sa_session.query( trans.model.Request ).get( trans.security.decode_id( request_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, request_id )
        displayable_sample_widgets = self.__get_sample_widgets( trans, request, request.samples, **kwd )
        sample_index = int( params.get( 'sample_id', 0 ) )
        sample_name = displayable_sample_widgets[sample_index]['name']
        sample = request.get_sample( sample_name )
        if sample:
            trans.sa_session.delete( sample.values )
            trans.sa_session.delete( sample )
            trans.sa_session.flush()
        message = 'Sample (%s) has been deleted.' % sample_name
        return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                          action='edit_samples',
                                                          cntrller=cntrller,
                                                          id=trans.security.encode_id( request.id ),
                                                          status=status,
                                                          message=message ) )
    @web.expose
    @web.require_login( "view data transfer page" )
    def view_sample_datasets( self, trans, cntrller, **kwd ):
        # The link on the number of selected datasets will only appear if there is at least 1 selected dataset.
        # If there are 0 selected datasets, there is no link, so this method will only be reached from the requests
        # controller if there are selected datasets.
        params = util.Params( kwd )
        message = util.restore_text( params.get( 'message', ''  ) )
        status = params.get( 'status', 'done' )
        is_admin = cntrller == 'requests_admin' and trans.user_is_admin()
        sample_id = params.get( 'sample_id', None )
        try:
            sample = trans.sa_session.query( trans.model.Sample ).get( trans.security.decode_id( sample_id ) )
        except:
            return invalid_id_redirect( trans, cntrller, sample_id, 'sample' )
        # See if a library and folder have been set for this sample.
        if is_admin and not sample.library or not sample.folder:
            status = 'error'
            message = "Select a target data library and folder for the sample before selecting the datasets."
            return trans.response.send_redirect( web.url_for( controller='requests_common',
                                                              action='edit_samples',
                                                              cntrller=cntrller,
                                                              id=trans.security.encode_id( sample.request.id ),
                                                              status=status,
                                                              message=message ) )
        transfer_status = params.get( 'transfer_status', None )
        if transfer_status in [ None, 'None' ]:
            title = 'All selected datasets for "%s"' % sample.name
            sample_datasets = sample.datasets
        elif transfer_status == trans.model.SampleDataset.transfer_status.IN_QUEUE:
            title = 'Datasets of "%s" that are in the transfer queue' % sample.name
            sample_datasets = sample.queued_dataset_files
        elif transfer_status == trans.model.SampleDataset.transfer_status.TRANSFERRING:
            title = 'Datasets of "%s" that are being transferred' % sample.name
            sample_datasets = sample.transferring_dataset_files
        elif transfer_status == trans.model.SampleDataset.transfer_status.ADD_TO_LIBRARY:
            title = 'Datasets of "%s" that are being added to the target data library' % sample.name
            sample_datasets = sample.adding_to_library_dataset_files
        elif transfer_status == trans.model.SampleDataset.transfer_status.COMPLETE:
            title = 'Datasets of "%s" that are available in the target data library' % sample.name
            sample_datasets = sample.transferred_dataset_files
        elif transfer_status == trans.model.SampleDataset.transfer_status.ERROR:
            title = 'Datasets of "%s" that resulted in a transfer error' % sample.name
            sample_datasets = sample.transfer_error_dataset_files
        return trans.fill_template( '/requests/common/view_sample_datasets.mako', 
                                    cntrller=cntrller,
                                    title=title,
                                    sample=sample,
                                    sample_datasets=sample_datasets,
                                    transfer_status=transfer_status,
                                    message=message,
                                    status=status )
    def __import_samples( self, trans, cntrller, request, displayable