PageRenderTime 97ms CodeModel.GetById 15ms app.highlight 70ms RepoModel.GetById 1ms app.codeStats 1ms

/lib/galaxy/web/controllers/dataset.py

https://bitbucket.org/cistrome/cistrome-harvard/
Python | 1170 lines | 1141 code | 17 blank | 12 comment | 73 complexity | bc297ac926443be24ae1de071316e11c MD5 | raw file

Large files files are truncated, but you can click here to view the full file

  1import logging, os, string, shutil, re, socket, mimetypes, urllib, tempfile, zipfile, glob, sys
  2
  3from galaxy.web.base.controller import *
  4from galaxy.web.framework.helpers import time_ago, iff, grids
  5from galaxy import util, datatypes, jobs, web, model
  6from cgi import escape, FieldStorage
  7from galaxy.datatypes.display_applications.util import encode_dataset_user, decode_dataset_user
  8from galaxy.util.sanitize_html import sanitize_html
  9from galaxy.util import inflector
 10from galaxy.model.item_attrs import *
 11from galaxy.model import LibraryDatasetDatasetAssociation, HistoryDatasetAssociation
 12from galaxy.web.framework.helpers import to_unicode
 13
 14import pkg_resources; 
 15pkg_resources.require( "Paste" )
 16import paste.httpexceptions
 17
 18if sys.version_info[:2] < ( 2, 6 ):
 19    zipfile.BadZipFile = zipfile.error
 20if sys.version_info[:2] < ( 2, 5 ):
 21    zipfile.LargeZipFile = zipfile.error
 22
 23tmpd = tempfile.mkdtemp()
 24comptypes=[]
 25ziptype = '32'
 26tmpf = os.path.join( tmpd, 'compression_test.zip' )
 27try:
 28    archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True )
 29    archive.close()
 30    comptypes.append( 'zip' )
 31    ziptype = '64'
 32except RuntimeError:
 33    log.exception( "Compression error when testing zip compression. This option will be disabled for library downloads." )
 34except (TypeError, zipfile.LargeZipFile):    # ZIP64 is only in Python2.5+.  Remove TypeError when 2.4 support is dropped
 35    log.warning( 'Max zip file size is 2GB, ZIP64 not supported' )    
 36    comptypes.append( 'zip' )
 37try:
 38    os.unlink( tmpf )
 39except OSError:
 40    pass
 41os.rmdir( tmpd )
 42
 43log = logging.getLogger( __name__ )
 44
 45error_report_template = """
 46GALAXY TOOL ERROR REPORT
 47------------------------
 48
 49This error report was sent from the Galaxy instance hosted on the server
 50"${host}"
 51-----------------------------------------------------------------------------
 52This is in reference to dataset id ${dataset_id} from history id ${history_id}
 53-----------------------------------------------------------------------------
 54You should be able to view the history containing the related history item
 55
 56${hid}: ${history_item_name} 
 57
 58by logging in as a Galaxy admin user to the Galaxy instance referenced above
 59and pointing your browser to the following link.
 60
 61${history_view_link}
 62-----------------------------------------------------------------------------
 63The user '${email}' provided the following information:
 64
 65${message}
 66-----------------------------------------------------------------------------
 67job id: ${job_id}
 68tool id: ${job_tool_id}
 69-----------------------------------------------------------------------------
 70job command line:
 71${job_command_line}
 72-----------------------------------------------------------------------------
 73job stderr:
 74${job_stderr}
 75-----------------------------------------------------------------------------
 76job stdout:
 77${job_stdout}
 78-----------------------------------------------------------------------------
 79job info:
 80${job_info}
 81-----------------------------------------------------------------------------
 82job traceback:
 83${job_traceback}
 84-----------------------------------------------------------------------------
 85(This is an automated message).
 86"""
 87
 88class HistoryDatasetAssociationListGrid( grids.Grid ):
 89    # Custom columns for grid.
 90    class HistoryColumn( grids.GridColumn ):
 91        def get_value( self, trans, grid, hda):
 92            return hda.history.name
 93            
 94    class StatusColumn( grids.GridColumn ):
 95        def get_value( self, trans, grid, hda ):
 96            if hda.deleted:
 97                return "deleted"
 98            return ""
 99        def get_accepted_filters( self ):
100            """ Returns a list of accepted filters for this column. """
101            accepted_filter_labels_and_vals = { "Active" : "False", "Deleted" : "True", "All": "All" }
102            accepted_filters = []
103            for label, val in accepted_filter_labels_and_vals.items():
104               args = { self.key: val }
105               accepted_filters.append( grids.GridColumnFilter( label, args) )
106            return accepted_filters
107
108    # Grid definition
109    title = "Saved Datasets"
110    model_class = model.HistoryDatasetAssociation
111    template='/dataset/grid.mako'
112    default_sort_key = "-update_time"
113    columns = [
114        grids.TextColumn( "Name", key="name", 
115                            # Link name to dataset's history.
116                            link=( lambda item: iff( item.history.deleted, None, dict( operation="switch", id=item.id ) ) ), filterable="advanced", attach_popup=True ),
117        HistoryColumn( "History", key="history", 
118                        link=( lambda item: iff( item.history.deleted, None, dict( operation="switch_history", id=item.id ) ) ) ),
119        grids.IndividualTagsColumn( "Tags", key="tags", model_tag_association_class=model.HistoryDatasetAssociationTagAssociation, filterable="advanced", grid_name="HistoryDatasetAssocationListGrid" ),
120        StatusColumn( "Status", key="deleted", attach_popup=False ),
121        grids.GridColumn( "Last Updated", key="update_time", format=time_ago ),
122    ]
123    columns.append( 
124        grids.MulticolFilterColumn(  
125        "Search", 
126        cols_to_filter=[ columns[0], columns[2] ], 
127        key="free-text-search", visible=False, filterable="standard" )
128                )
129    operations = [
130        grids.GridOperation( "Copy to current history", condition=( lambda item: not item.deleted ), async_compatible=False ),
131    ]
132    standard_filters = []
133    default_filter = dict( name="All", deleted="False", tags="All" )
134    preserve_state = False
135    use_paging = True
136    num_rows_per_page = 50
137    def build_initial_query( self, trans, **kwargs ):
138        # Show user's datasets that are not deleted, not in deleted histories, and not hidden.
139        # To filter HDAs by user, need to join model class/HDA and History table so that it is 
140        # possible to filter by user. However, for dictionary-based filtering to work, need a 
141        # primary table for the query.
142        return trans.sa_session.query( self.model_class ).select_from( self.model_class.table.join( model.History.table ) ) \
143                .filter( model.History.user == trans.user ) \
144                .filter( self.model_class.deleted==False ) \
145                .filter( model.History.deleted==False ) \
146                .filter( self.model_class.visible==True )
147        
148class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistory, UsesHistoryDatasetAssociation, UsesItemRatings ):
149        
150    stored_list_grid = HistoryDatasetAssociationListGrid()
151
152    @web.expose
153    def errors( self, trans, id ):
154        hda = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id )
155        return trans.fill_template( "dataset/errors.mako", hda=hda )
156    @web.expose
157    def stderr( self, trans, id ):
158        dataset = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id )
159        job = dataset.creating_job_associations[0].job
160        trans.response.set_content_type( 'text/plain' )
161        return job.stderr
162    @web.expose
163    def report_error( self, trans, id, email='', message="" ):
164        smtp_server = trans.app.config.smtp_server
165        if smtp_server is None:
166            return trans.show_error_message( "Mail is not configured for this galaxy instance" )
167        to_address = trans.app.config.error_email_to
168        if to_address is None:
169            return trans.show_error_message( "Error reporting has been disabled for this galaxy instance" )
170        # Get the dataset and associated job
171        hda = trans.sa_session.query( model.HistoryDatasetAssociation ).get( id )
172        job = hda.creating_job_associations[0].job
173        # Get the name of the server hosting the Galaxy instance from which this report originated
174        host = trans.request.host
175        history_view_link = "%s/ap/history/view?id=%s" % ( str( host ), trans.security.encode_id( hda.history_id ) )
176        # Build the email message
177        body = string.Template( error_report_template ) \
178            .safe_substitute( host=host,
179                              dataset_id=hda.dataset_id,
180                              history_id=hda.history_id,
181                              hid=hda.hid,
182                              history_item_name=hda.get_display_name(),
183                              history_view_link=history_view_link,
184                              job_id=job.id,
185                              job_tool_id=job.tool_id,
186                              job_command_line=job.command_line,
187                              job_stderr=job.stderr,
188                              job_stdout=job.stdout,
189                              job_info=job.info,
190                              job_traceback=job.traceback,
191                              email=email, 
192                              message=message )
193        frm = to_address
194        # Check email a bit
195        email = email.strip()
196        parts = email.split()
197        if len( parts ) == 1 and len( email ) > 0:
198            to = to_address + ", " + email
199        else:
200            to = to_address
201        subject = "Galaxy tool error report from " + email
202        # Send it
203        try:
204            util.send_mail( frm, to, subject, body, trans.app.config )
205            return trans.show_ok_message( "Your error report has been sent" )
206        except Exception, e:
207            return trans.show_error_message( "An error occurred sending the report by email: %s" % str( e ) )
208    
209    @web.expose
210    def default(self, trans, dataset_id=None, **kwd):
211        return 'This link may not be followed from within Galaxy.'
212    
213    @web.expose
214    def archive_composite_dataset( self, trans, data=None, **kwd ):
215        # save a composite object into a compressed archive for downloading
216        params = util.Params( kwd )
217        valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
218        outfname = data.name[0:150]
219        outfname = ''.join(c in valid_chars and c or '_' for c in outfname)
220        if (params.do_action == None):
221     	    params.do_action = 'zip' # default
222        msg = util.restore_text( params.get( 'msg', ''  ) )
223        messagetype = params.get( 'messagetype', 'done' )
224        if not data:
225            msg = "You must select at least one dataset"
226            messagetype = 'error'
227        else:
228            error = False
229            try:
230                if (params.do_action == 'zip'): 
231                    # Can't use mkstemp - the file must not exist first
232                    tmpd = tempfile.mkdtemp()
233                    tmpf = os.path.join( tmpd, 'library_download.' + params.do_action )
234                    if ziptype == '64':
235                        archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True )
236                    else:
237                        archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED )
238                    archive.add = lambda x, y: archive.write( x, y.encode('CP437') )
239                elif params.do_action == 'tgz':
240                    archive = util.streamball.StreamBall( 'w|gz' )
241                elif params.do_action == 'tbz':
242                    archive = util.streamball.StreamBall( 'w|bz2' )
243            except (OSError, zipfile.BadZipFile):
244                error = True
245                log.exception( "Unable to create archive for download" )
246                msg = "Unable to create archive for %s for download, please report this error" % outfname
247                messagetype = 'error'
248            if not error:
249                current_user_roles = trans.get_current_user_roles()
250                ext = data.extension
251                path = data.file_name
252                fname = os.path.split(path)[-1]
253                efp = data.extra_files_path
254                htmlname = os.path.splitext(outfname)[0]
255                if not htmlname.endswith(ext):
256                    htmlname = '%s_%s' % (htmlname,ext)
257                archname = '%s.html' % htmlname # fake the real nature of the html file
258                try:
259                    archive.add(data.file_name,archname)
260                except IOError:
261                    error = True
262                    log.exception( "Unable to add composite parent %s to temporary library download archive" % data.file_name)
263                    msg = "Unable to create archive for download, please report this error"
264                    messagetype = 'error'
265                for root, dirs, files in os.walk(efp):
266                    for fname in files:
267                        fpath = os.path.join(root,fname) 
268                        rpath = os.path.relpath(fpath,efp)
269                        try:
270                            archive.add( fpath,rpath )
271                        except IOError:
272                            error = True
273                            log.exception( "Unable to add %s to temporary library download archive" % rpath)
274                            msg = "Unable to create archive for download, please report this error"
275                            messagetype = 'error'
276                            continue
277                if not error:    
278                    if params.do_action == 'zip':
279                        archive.close()
280                        tmpfh = open( tmpf )
281                        # clean up now
282                        try:
283                            os.unlink( tmpf )
284                            os.rmdir( tmpd )
285                        except OSError:
286                            error = True
287                            msg = "Unable to remove temporary library download archive and directory"
288                            log.exception( msg )
289                            messagetype = 'error'
290                        if not error:
291                            trans.response.set_content_type( "application/x-zip-compressed" )
292                            trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.zip" % outfname 
293                            return tmpfh
294                    else:
295                        trans.response.set_content_type( "application/x-tar" )
296                        outext = 'tgz'
297                        if params.do_action == 'tbz':
298                            outext = 'tbz'
299                        trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (outfname,outext) 
300                        archive.wsgi_status = trans.response.wsgi_status()
301                        archive.wsgi_headeritems = trans.response.wsgi_headeritems()
302                        return archive.stream
303        return trans.show_error_message( msg )
304
305
306    @web.expose
307    def get_metadata_file(self, trans, hda_id, metadata_name):
308        """ Allows the downloading of metadata files associated with datasets (eg. bai index for bam files) """
309        data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( hda_id ) )
310        if not data or not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ):
311            return trans.show_error_message( "You are not allowed to access this dataset" )
312        
313        valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
314        fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
315        
316        file_ext = data.metadata.spec.get(metadata_name).get("file_ext", metadata_name)
317        trans.response.headers["Content-Type"] = "application/octet-stream"
318        trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy%s-[%s].%s" % (data.hid, fname, file_ext)
319        return open(data.metadata.get(metadata_name).file_name)
320        
321    @web.expose
322    def display(self, trans, dataset_id=None, preview=False, filename=None, to_ext=None, **kwd):
323        """Catches the dataset id and displays file contents as directed"""
324        composite_extensions = trans.app.datatypes_registry.get_composite_extensions( )
325        composite_extensions.append('html') # for archiving composite datatypes
326        # DEPRECATION: We still support unencoded ids for backward compatibility
327        try:
328            data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) )
329            if data is None:
330                raise ValueError( 'Invalid reference dataset id: %s.' % dataset_id )
331        except:
332            try:
333                data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( dataset_id ) )
334            except:
335                data = None
336        if not data:
337            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
338        if not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), data.dataset ):
339            return trans.show_error_message( "You are not allowed to access this dataset" )
340        
341        if data.state == trans.model.Dataset.states.UPLOAD:
342            return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." )
343        
344        if filename and filename != "index":
345            # For files in extra_files_path
346            file_path = os.path.join( data.extra_files_path, filename )
347            if os.path.exists( file_path ):
348                if os.path.isdir( file_path ):
349                    return trans.show_error_message( "Directory listing is not allowed." ) #TODO: Reconsider allowing listing of directories?
350                mime, encoding = mimetypes.guess_type( file_path )
351                if not mime:
352                    try:
353                        mime = trans.app.datatypes_registry.get_mimetype_by_extension( ".".split( file_path )[-1] )
354                    except:
355                        mime = "text/plain"
356                trans.response.set_content_type( mime )
357                return open( file_path )
358            else:
359                return trans.show_error_message( "Could not find '%s' on the extra files path %s." % ( filename, file_path ) )
360        
361        trans.response.set_content_type(data.get_mime())
362        trans.log_event( "Display dataset id: %s" % str( dataset_id ) )
363        
364        if to_ext or isinstance(data.datatype, datatypes.binary.Binary): # Saving the file, or binary file
365            if data.extension in composite_extensions:
366                return self.archive_composite_dataset( trans, data, **kwd )
367            else:                    
368                trans.response.headers['Content-Length'] = int( os.stat( data.file_name ).st_size )
369                if not to_ext:
370                    to_ext = data.extension
371                valid_chars = '.,^_-()[]0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
372                fname = ''.join(c in valid_chars and c or '_' for c in data.name)[0:150]
373                trans.response.headers["Content-Disposition"] = "attachment; filename=Galaxy%s-[%s].%s" % (data.hid, fname, to_ext)
374                return open( data.file_name )
375        if not os.path.exists( data.file_name ):
376            raise paste.httpexceptions.HTTPNotFound( "File Not Found (%s)." % data.file_name )
377        
378        max_peek_size = 1000000 # 1 MB
379        if not preview or isinstance(data.datatype, datatypes.images.Image) or os.stat( data.file_name ).st_size < max_peek_size:
380            return open( data.file_name )
381        else:
382            trans.response.set_content_type( "text/html" )
383            return trans.stream_template_mako( "/dataset/large_file.mako",
384                                            truncated_data = open( data.file_name ).read(max_peek_size),
385                                            data = data )
386
387    @web.expose
388    def edit(self, trans, dataset_id=None, filename=None, hid=None, **kwd):
389        """Allows user to modify parameters of an HDA."""
390        message = None
391        status = 'done'
392        refresh_frames = []
393        error = False
394        def __ok_to_edit_metadata( dataset_id ):
395            #prevent modifying metadata when dataset is queued or running as input/output
396            #This code could be more efficient, i.e. by using mappers, but to prevent slowing down loading a History panel, we'll leave the code here for now
397            for job_to_dataset_association in trans.sa_session.query( self.app.model.JobToInputDatasetAssociation ) \
398                                                              .filter_by( dataset_id=dataset_id ) \
399                                                              .all() \
400                                            + trans.sa_session.query( self.app.model.JobToOutputDatasetAssociation ) \
401                                                              .filter_by( dataset_id=dataset_id ) \
402                                                              .all():
403                if job_to_dataset_association.job.state not in [ job_to_dataset_association.job.states.OK, job_to_dataset_association.job.states.ERROR, job_to_dataset_association.job.states.DELETED ]:
404                    return False
405            return True
406        if hid is not None:
407            history = trans.get_history()
408            # TODO: hid handling
409            data = history.datasets[ int( hid ) - 1 ]
410            id = None
411        elif dataset_id is not None: 
412            id = trans.app.security.decode_id( dataset_id )
413            data = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id )
414        else:
415            trans.log_event( "dataset_id and hid are both None, cannot load a dataset to edit" )
416            return trans.show_error_message( "You must provide a history dataset id to edit" )
417        if data is None:
418            trans.log_event( "Problem retrieving dataset (encoded: %s, decoded: %s) with history id %s." % ( str( dataset_id ), str( id ), str( hid ) ) )
419            return trans.show_error_message( "History dataset id is invalid" )
420        if dataset_id is not None and data.history.user is not None and data.history.user != trans.user:
421            trans.log_event( "User attempted to edit an HDA they do not own (encoded: %s, decoded: %s)" % ( dataset_id, id ) )
422            # Do not reveal the dataset's existence
423            return trans.show_error_message( "History dataset id is invalid" )
424        current_user_roles = trans.get_current_user_roles()
425        if data.history.user and not data.dataset.has_manage_permissions_roles( trans ):
426            # Permission setting related to DATASET_MANAGE_PERMISSIONS was broken for a period of time,
427            # so it is possible that some Datasets have no roles associated with the DATASET_MANAGE_PERMISSIONS
428            # permission.  In this case, we'll reset this permission to the hda user's private role.
429            manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action )
430            permissions = { manage_permissions_action : [ trans.app.security_agent.get_private_user_role( data.history.user ) ] }
431            trans.app.security_agent.set_dataset_permission( data.dataset, permissions )        
432        if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
433            if data.state == trans.model.Dataset.states.UPLOAD:
434                return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." )
435            params = util.Params( kwd, sanitize=False )
436            if params.change:
437                # The user clicked the Save button on the 'Change data type' form
438                if data.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change:
439                    #prevent modifying datatype when dataset is queued or running as input/output
440                    if not __ok_to_edit_metadata( data.id ):
441                        message = "This dataset is currently being used as input or output.  You cannot change datatype until the jobs have completed or you have canceled them."
442                        error = True
443                    else:
444                        trans.app.datatypes_registry.change_datatype( data, params.datatype, set_meta = not trans.app.config.set_metadata_externally )
445                        trans.sa_session.flush()
446                        if trans.app.config.set_metadata_externally:
447                            trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data }, overwrite = False ) #overwrite is False as per existing behavior
448                        message = "Changed the type of dataset '%s' to %s" % ( to_unicode( data.name ), params.datatype )
449                        refresh_frames=['history']
450                else:
451                    message = "You are unable to change datatypes in this manner. Changing %s to %s is not allowed." % ( data.extension, params.datatype )
452                    error = True
453            elif params.save:
454                # The user clicked the Save button on the 'Edit Attributes' form
455                data.name  = params.name
456                data.info  = params.info
457                message = ''
458                if __ok_to_edit_metadata( data.id ):
459                    # The following for loop will save all metadata_spec items
460                    for name, spec in data.datatype.metadata_spec.items():
461                        if spec.get("readonly"):
462                            continue
463                        optional = params.get("is_"+name, None)
464                        other = params.get("or_"+name, None)
465                        if optional and optional == 'true':
466                            # optional element... == 'true' actually means it is NOT checked (and therefore omitted)
467                            setattr(data.metadata, name, None)
468                        else:
469                            if other:
470                                setattr( data.metadata, name, other )
471                            else:
472                                setattr( data.metadata, name, spec.unwrap( params.get (name, None) ) )
473                    data.datatype.after_setting_metadata( data )
474                    # Sanitize annotation before adding it.
475                    if params.annotation:
476                        annotation = sanitize_html( params.annotation, 'utf-8', 'text/html' )
477                        self.add_item_annotation( trans.sa_session, trans.get_user(), data, annotation )
478                    # If setting metadata previously failed and all required elements have now been set, clear the failed state.
479                    if data._state == trans.model.Dataset.states.FAILED_METADATA and not data.missing_meta():
480                        data._state = None
481                    trans.sa_session.flush()
482                    message = "Attributes updated%s" % message
483                    refresh_frames=['history']
484                else:
485                    trans.sa_session.flush()
486                    message = "Attributes updated, but metadata could not be changed because this dataset is currently being used as input or output. You must cancel or wait for these jobs to complete before changing metadata."
487                    status = "warning"
488                    refresh_frames=['history']
489            elif params.detect:
490                # The user clicked the Auto-detect button on the 'Edit Attributes' form
491                #prevent modifying metadata when dataset is queued or running as input/output
492                if not __ok_to_edit_metadata( data.id ):
493                    message = "This dataset is currently being used as input or output.  You cannot change metadata until the jobs have completed or you have canceled them."
494                    error = True
495                else:
496                    for name, spec in data.metadata.spec.items():
497                        # We need to be careful about the attributes we are resetting
498                        if name not in [ 'name', 'info', 'dbkey', 'base_name' ]:
499                            if spec.get( 'default' ):
500                                setattr( data.metadata, name, spec.unwrap( spec.get( 'default' ) ) )
501                    if trans.app.config.set_metadata_externally:
502                        message = 'Attributes have been queued to be updated'
503                        trans.app.datatypes_registry.set_external_metadata_tool.tool_action.execute( trans.app.datatypes_registry.set_external_metadata_tool, trans, incoming = { 'input1':data } )
504                    else:
505                        message = 'Attributes updated'
506                        data.set_meta()
507                        data.datatype.after_setting_metadata( data )
508                    trans.sa_session.flush()
509                    refresh_frames=['history']
510            elif params.convert_data:
511                target_type = kwd.get("target_type", None)
512                if target_type:
513                    message = data.datatype.convert_dataset(trans, data, target_type)
514                    refresh_frames=['history']
515            elif params.update_roles_button:
516                if not trans.user:
517                    return trans.show_error_message( "You must be logged in if you want to change permissions." )
518                if trans.app.security_agent.can_manage_dataset( current_user_roles, data.dataset ):
519                    access_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_ACCESS.action )
520                    manage_permissions_action = trans.app.security_agent.get_action( trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action )
521                    # The user associated the DATASET_ACCESS permission on the dataset with 1 or more roles.  We
522                    # need to ensure that they did not associate roles that would cause accessibility problems.
523                    permissions, in_roles, error, message = \
524                    trans.app.security_agent.derive_roles_from_access( trans, data.dataset.id, 'root', **kwd )
525                    if error:
526                        # Keep the original role associations for the DATASET_ACCESS permission on the dataset.
527                        permissions[ access_action ] = data.dataset.get_access_roles( trans )
528                        status = 'error'
529                    else:
530                        error = trans.app.security_agent.set_all_dataset_permissions( data.dataset, permissions )
531                        if error:
532                            message += error
533                            status = 'error'
534                        else:
535                            message = 'Your changes completed successfully.'
536                    trans.sa_session.refresh( data.dataset )
537                else:
538                    message = "You are not authorized to change this dataset's permissions"
539                    error = True
540            else:
541                if "dbkey" in data.datatype.metadata_spec and not data.metadata.dbkey:
542                    # Copy dbkey into metadata, for backwards compatability
543                    # This looks like it does nothing, but getting the dbkey
544                    # returns the metadata dbkey unless it is None, in which
545                    # case it resorts to the old dbkey.  Setting the dbkey
546                    # sets it properly in the metadata
547                    #### This is likely no longer required, since the dbkey exists entirely within metadata (the old_dbkey field is gone): REMOVE ME?
548                    data.metadata.dbkey = data.dbkey
549            # let's not overwrite the imported datatypes module with the variable datatypes?
550            # the built-in 'id' is overwritten in lots of places as well
551            ldatatypes = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ]
552            ldatatypes.sort()
553            all_roles = trans.app.security_agent.get_legitimate_roles( trans, data.dataset, 'root' )
554            if error:
555                status = 'error'
556            return trans.fill_template( "/dataset/edit_attributes.mako",
557                                        data=data,
558                                        data_annotation=self.get_item_annotation_str( trans.sa_session, trans.user, data ),
559                                        datatypes=ldatatypes,
560                                        current_user_roles=current_user_roles,
561                                        all_roles=all_roles,
562                                        message=message,
563                                        status=status,
564                                        dataset_id=dataset_id,
565                                        refresh_frames=refresh_frames )
566        else:
567            return trans.show_error_message( "You do not have permission to edit this dataset's ( id: %s ) information." % str( dataset_id ) )
568                        
569    @web.expose
570    @web.require_login( "see all available datasets" )
571    def list( self, trans, **kwargs ):
572        """List all available datasets"""
573        status = message = None
574
575        if 'operation' in kwargs:
576            operation = kwargs['operation'].lower()
577            hda_ids = util.listify( kwargs.get( 'id', [] ) )
578            
579            # Display no message by default
580            status, message = None, None
581
582            # Load the hdas and ensure they all belong to the current user
583            hdas = []
584            for encoded_hda_id in hda_ids:
585                hda_id = trans.security.decode_id( encoded_hda_id )
586                hda = trans.sa_session.query( model.HistoryDatasetAssociation ).filter_by( id=hda_id ).first()
587                if hda:
588                    # Ensure history is owned by current user
589                    if hda.history.user_id != None and trans.user:
590                        assert trans.user.id == hda.history.user_id, "HistoryDatasetAssocation does not belong to current user"
591                    hdas.append( hda )
592                else:
593                    log.warn( "Invalid history_dataset_association id '%r' passed to list", hda_id )
594
595            if hdas:
596                if operation == "switch" or operation == "switch_history":
597                    # Switch to a history that the HDA resides in.
598                    
599                    # Convert hda to histories.
600                    histories = []
601                    for hda in hdas:
602                        histories.append( hda.history )
603                        
604                    # Use history controller to switch the history. TODO: is this reasonable?
605                    status, message = trans.webapp.controllers['history']._list_switch( trans, histories )
606                    
607                    # Current history changed, refresh history frame; if switching to a dataset, set hda seek.
608                    trans.template_context['refresh_frames'] = ['history']
609                    if operation == "switch":
610                        hda_ids = [ trans.security.encode_id( hda.id ) for hda in hdas ]
611                        trans.template_context[ 'seek_hda_ids' ] = hda_ids
612                elif operation == "copy to current history":
613                    # Copy a dataset to the current history.
614                    target_histories = [ trans.get_history() ]
615                    status, message = self._copy_datasets( trans, hda_ids, target_histories )
616                    
617                    # Current history changed, refresh history frame.
618                    trans.template_context['refresh_frames'] = ['history']
619
620        # Render the list view
621        return self.stored_list_grid( trans, status=status, message=message, **kwargs )
622        
623    @web.expose
624    def imp( self, trans, dataset_id=None, **kwd ):
625        """ Import another user's dataset via a shared URL; dataset is added to user's current history. """
626        msg = ""
627        
628        # Set referer message.
629        referer = trans.request.referer
630        if referer is not "":
631            referer_message = "<a href='%s'>return to the previous page</a>" % referer
632        else:
633            referer_message = "<a href='%s'>go to Galaxy's start page</a>" % url_for( '/' )
634        
635        # Error checking.
636        if not dataset_id:
637            return trans.show_error_message( "You must specify a dataset to import. You can %s." % referer_message, use_panels=True )
638            
639        # Do import.
640        cur_history = trans.get_history( create=True )
641        status, message = self._copy_datasets( trans, [ dataset_id ], [ cur_history ], imported=True )
642        message = "Dataset imported. <br>You can <a href='%s'>start using the dataset</a> or %s." % ( url_for('/'),  referer_message )
643        return trans.show_message( message, type=status, use_panels=True )
644        
645    @web.expose
646    @web.json
647    @web.require_login( "use Galaxy datasets" )
648    def get_name_and_link_async( self, trans, id=None ):
649        """ Returns dataset's name and link. """
650        dataset = self.get_dataset( trans, id, False, True )
651        return_dict = { "name" : dataset.name, "link" : url_for( action="display_by_username_and_slug", username=dataset.history.user.username, slug=trans.security.encode_id( dataset.id ) ) }
652        return return_dict
653                
654    @web.expose
655    def get_embed_html_async( self, trans, id ):
656        """ Returns HTML for embedding a dataset in a page. """
657        dataset = self.get_dataset( trans, id, False, True )
658        if dataset:
659            return "Embedded Dataset '%s'" % dataset.name
660
661    @web.expose
662    @web.require_login( "use Galaxy datasets" )
663    def set_accessible_async( self, trans, id=None, accessible=False ):
664        """ Does nothing because datasets do not have an importable/accessible attribute. This method could potentially set another attribute. """
665        return
666        
667    @web.expose
668    @web.require_login( "rate items" )
669    @web.json
670    def rate_async( self, trans, id, rating ):
671        """ Rate a dataset asynchronously and return updated community data. """
672
673        dataset = self.get_dataset( trans, id, check_ownership=False, check_accessible=True )
674        if not dataset:
675            return trans.show_error_message( "The specified dataset does not exist." )
676
677        # Rate dataset.
678        dataset_rating = self.rate_item( rate_item, trans.get_user(), dataset, rating )
679
680        return self.get_ave_item_rating_data( trans.sa_session, dataset )
681        
682    @web.expose
683    def display_by_username_and_slug( self, trans, username, slug, preview=True ):
684        """ Display dataset by username and slug; because datasets do not yet have slugs, the slug is the dataset's id. """
685        dataset = self.get_dataset( trans, slug, False, True )
686        if dataset:
687            truncated, dataset_data = self.get_data( dataset, preview )
688            dataset.annotation = self.get_item_annotation_str( trans.sa_session, dataset.history.user, dataset )
689            
690            # If data is binary or an image, stream without template; otherwise, use display template.
691            # TODO: figure out a way to display images in display template.
692            if isinstance(dataset.datatype, datatypes.binary.Binary) or isinstance(dataset.datatype, datatypes.images.Image)  or isinstance(dataset.datatype, datatypes.images.Html):
693                trans.response.set_content_type( data.get_mime() )
694                return open( dataset.file_name )
695            else:
696                # Get rating data.
697                user_item_rating = 0
698                if trans.get_user():
699                    user_item_rating = self.get_user_item_rating( trans.sa_session, trans.get_user(), dataset )
700                    if user_item_rating:
701                        user_item_rating = user_item_rating.rating
702                    else:
703                        user_item_rating = 0
704                ave_item_rating, num_ratings = self.get_ave_item_rating_data( trans.sa_session, dataset )
705                
706                return trans.fill_template_mako( "/dataset/display.mako", item=dataset, item_data=dataset_data, truncated=truncated,
707                                                user_item_rating = user_item_rating, ave_item_rating=ave_item_rating, num_ratings=num_ratings )
708        else:
709            raise web.httpexceptions.HTTPNotFound()
710            
711    @web.expose
712    def get_item_content_async( self, trans, id ):
713        """ Returns item content in HTML format. """
714
715        dataset = self.get_dataset( trans, id, False, True )
716        if dataset is None:
717            raise web.httpexceptions.HTTPNotFound()
718        truncated, dataset_data = self.get_data( dataset, preview=True )
719        # Get annotation.
720        dataset.annotation = self.get_item_annotation_str( trans.sa_session, trans.user, dataset )
721        return trans.stream_template_mako( "/dataset/item_content.mako", item=dataset, item_data=dataset_data, truncated=truncated )
722        
723    @web.expose
724    def annotate_async( self, trans, id, new_annotation=None, **kwargs ):
725        dataset = self.get_dataset( trans, id, False, True )
726        if not dataset:
727            web.httpexceptions.HTTPNotFound()
728        if dataset and new_annotation:
729            # Sanitize annotation before adding it.
730            new_annotation = sanitize_html( new_annotation, 'utf-8', 'text/html' )
731            self.add_item_annotation( trans.sa_session, trans.get_user(), dataset, new_annotation )
732            trans.sa_session.flush()
733            return new_annotation
734    
735    @web.expose
736    def get_annotation_async( self, trans, id ):
737        dataset = self.get_dataset( trans, id, False, True )
738        if not dataset:
739            web.httpexceptions.HTTPNotFound()
740        return self.get_item_annotation_str( trans.sa_session, trans.user, dataset )
741
742    @web.expose
743    def display_at( self, trans, dataset_id, filename=None, **kwd ):
744        """Sets up a dataset permissions so it is viewable at an external site"""
745        site = filename
746        data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataset_id )
747        if not data:
748            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
749        if 'display_url' not in kwd or 'redirect_url' not in kwd:
750            return trans.show_error_message( 'Invalid parameters specified for "display at" link, please contact a Galaxy administrator' )
751        try:
752              redirect_url = kwd['redirect_url'] % urllib.quote_plus( kwd['display_url'] )
753        except:
754              redirect_url = kwd['redirect_url'] # not all will need custom text
755        current_user_roles = trans.get_current_user_roles()
756        if trans.app.security_agent.dataset_is_public( data.dataset ):
757            return trans.response.send_redirect( redirect_url ) # anon access already permitted by rbac
758        if trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
759            trans.app.host_security_agent.set_dataset_permissions( data, trans.user, site )
760            return trans.response.send_redirect( redirect_url )
761        else:
762            return trans.show_error_message( "You are not allowed to view this dataset at external sites.  Please contact your Galaxy administrator to acquire management permissions for this dataset." )
763
764    @web.expose
765    def display_application( self, trans, dataset_id=None, user_id=None, app_name = None, link_name = None, app_action = None, action_param = None, **kwds ):
766        """Access to external display applications"""
767        if kwds:
768            log.debug( "Unexpected Keywords passed to display_application: %s" % kwds ) #route memory?
769        #decode ids
770        data, user = decode_dataset_user( trans, dataset_id, user_id )
771        if not data:
772            raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
773        if user is None:
774            user = trans.user
775        if user:
776            user_roles = user.all_roles()
777        else:
778            user_roles = []
779        if None in [ app_name, link_name ]:
780            return trans.show_error_message( "A display application name and link name must be provided." )
781        
782        if trans.app.security_agent.can_access_dataset( user_roles, data.dataset ):
783            msg = []
784            refresh = False
785            display_app = trans.app.datatypes_registry.display_applications.get( app_name )
786            assert display_app, "Unknown display application has been requested: %s" % app_name
787            dataset_hash, user_hash = encode_dataset_user( trans, data, user )
788            display_link = display_app.get_link( link_name, data, dataset_hash, user_hash, trans )
789            assert display_link, "Unknown display link has been requested: %s" % link_name
790            if data.state == data.states.ERROR:
791                msg.append( ( 'This dataset is in an error state, you cannot view it at an external display application.', 'error' ) )
792            elif data.deleted:
793                msg.append( ( 'This dataset has been deleted, you cannot view it at an external display application.', 'error' ) )
794            elif data.state != data.states.OK:
795                msg.append( ( 'You must wait for this dataset to be created before you can view it at an external display application.', 'info' ) )
796                refresh = True
797            else:
798                #We have permissions, dataset is not deleted and is in OK state, allow access
799                if display_link.display_ready():
800                    if app_action in [ 'data', 'param' ]:
801                        assert action_param, "An action param must be provided for a data or param action"
802                        #data is used for things with filenames that could be passed off to a proxy
803                        #in case some display app wants all files to be in the same 'directory', 
804                        #data can be forced to param, but not the other way (no filename for other direction)
805                        #get param name from url param name
806                        action_param = display_link.get_param_name_by_url( action_param )
807                        value = display_link.get_param_value( action_param )
808                        assert value, "An invalid parameter name was provided: %s" % action_param
809                        assert value.parameter.viewable, "This parameter is not viewable."
810                        if value.parameter.type == 'data':
811                            content_length = os.path.getsize( value.file_name )
812                            rval = open( value.file_name )
813                        else:
814                            rval = str( value )
815                            content_length = len( rval )
816                        trans.response.set_content_type( value.mime_type() )
817                        trans.response.headers[ 'Content-Length' ] = content_length
818                        return rval
819                    elif app_action == None:
820                        #redirect user to url generated by display link
821                        return trans.response.send_redirect( display_link.display_url() )
822                    else:
823                        msg.append( ( 'Invalid action provided: %s' % app_action, 'error' ) )
824                else:
825                    if app_action == None:
826                        if trans.history != data.history:
827                            msg.append( ( 'You must import this dataset into your current history before you can view it at the desired display application.', 'error' ) )
828                        else:
829                            refresh = True
830                            msg.append( ( 'This display application is being prepared.', 'info' ) )
831                            if not display_link.preparing_display():
832                                display_link.prepare_display()
833                    else:
834                        raise Exception( 'Attempted a view action (%s) on a non-ready display application' % app_action )
835            return trans.fill_template_mako( "dataset/display_application/display.mako", msg = msg, display_app = display_app, display_link = display_link, refresh = refresh )
836        return trans.show_error_message( 'You do not have permission to view this dataset at an external display application.' )
837
838    def _delete( self, trans, dataset_id ):
839        message = None
840        status = 'done'
841        id = None
842        try:
843            id = trans.app.security.decode_id( dataset_id )
844            history = trans.get_history()
845            hda = trans.sa_session.query( self.app.model.HistoryDatasetAssociation ).get( id )
846            assert hda, 'Invalid HDA: %s' % id
847            # Walk up parent datasets to find the containing history
848            topmost_parent = hda
849            while topmost_parent.parent:
850                topmost_parent = topmost_parent.parent
851            assert topmost_parent in trans.history.datasets, "Data does not belong to current history"
852            # Mark deleted and cleanup
853            hda.mark_deleted()
854            hda.clear_associated_files()
855            trans.log_event( "Dataset id %s marked as deleted" % str(id) )
856            if hda.parent_id is None and len( hda.creating_job_associations ) > 0:
857                # Mark associated job for deletion
858                job = hda.creating_job_associations[0].job
859                if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]:
860                    # Are *all* of the job's other output datasets deleted?
861                    if job.check_if_output_datasets_deleted():
862                        job.mark_deleted( self.app.config.get_bool( 'enable_job_running', True ),
863                                       

Large files files are truncated, but you can click here to view the full file