/lib/galaxy/web/base/controller.py

https://bitbucket.org/cistrome/cistrome-harvard/ · Python · 2973 lines · 2365 code · 173 blank · 435 comment · 416 complexity · 44e622a10caafdff829845c52ffeb39f MD5 · raw file

Large files are truncated click here to view the full file

  1. """
  2. Contains functionality needed in every web interface
  3. """
  4. import logging
  5. import operator
  6. import os
  7. import re
  8. from gettext import gettext
  9. import pkg_resources
  10. pkg_resources.require("SQLAlchemy >= 0.4")
  11. from sqlalchemy import func, and_, select
  12. from paste.httpexceptions import HTTPBadRequest, HTTPInternalServerError
  13. from paste.httpexceptions import HTTPNotImplemented, HTTPRequestRangeNotSatisfiable
  14. from galaxy import exceptions
  15. from galaxy.exceptions import ItemAccessibilityException, ItemDeletionException, ItemOwnershipException
  16. from galaxy.exceptions import MessageException
  17. from galaxy import web
  18. from galaxy import model
  19. from galaxy import security
  20. from galaxy import util
  21. from galaxy import objectstore
  22. from galaxy.web import error, url_for
  23. from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField
  24. from galaxy.web.form_builder import build_select_field, HistoryField, PasswordField, WorkflowField, WorkflowMappingField
  25. from galaxy.workflow.modules import module_factory
  26. from galaxy.model.orm import eagerload, eagerload_all, desc
  27. from galaxy.security.validate_user_input import validate_publicname
  28. from galaxy.util.sanitize_html import sanitize_html
  29. from galaxy.model.item_attrs import Dictifiable, UsesAnnotations
  30. from galaxy.datatypes.interval import ChromatinInteractions
  31. from galaxy.datatypes.data import Text
  32. from galaxy.model import ExtendedMetadata, ExtendedMetadataIndex, LibraryDatasetDatasetAssociation, HistoryDatasetAssociation
  33. from galaxy.datatypes.metadata import FileParameter
  34. from galaxy.tools.parameters import RuntimeValue, visit_input_values
  35. from galaxy.tools.parameters.basic import DataToolParameter
  36. from galaxy.util.json import to_json_string
  37. from galaxy.workflow.modules import ToolModule
  38. from galaxy.workflow.steps import attach_ordered_steps
  39. log = logging.getLogger( __name__ )
  40. # States for passing messages
  41. SUCCESS, INFO, WARNING, ERROR = "done", "info", "warning", "error"
  42. def _is_valid_slug( slug ):
  43. """ Returns true if slug is valid. """
  44. VALID_SLUG_RE = re.compile( "^[a-z0-9\-]+$" )
  45. return VALID_SLUG_RE.match( slug )
  46. class BaseController( object ):
  47. """
  48. Base class for Galaxy web application controllers.
  49. """
  50. def __init__( self, app ):
  51. """Initialize an interface for application 'app'"""
  52. self.app = app
  53. self.sa_session = app.model.context
  54. def get_toolbox(self):
  55. """Returns the application toolbox"""
  56. return self.app.toolbox
  57. def get_class( self, class_name ):
  58. """ Returns the class object that a string denotes. Without this method, we'd have to do eval(<class_name>). """
  59. if class_name == 'History':
  60. item_class = self.app.model.History
  61. elif class_name == 'HistoryDatasetAssociation':
  62. item_class = self.app.model.HistoryDatasetAssociation
  63. elif class_name == 'Page':
  64. item_class = self.app.model.Page
  65. elif class_name == 'StoredWorkflow':
  66. item_class = self.app.model.StoredWorkflow
  67. elif class_name == 'Visualization':
  68. item_class = self.app.model.Visualization
  69. elif class_name == 'Tool':
  70. item_class = self.app.model.Tool
  71. elif class_name == 'Job':
  72. item_class = self.app.model.Job
  73. elif class_name == 'User':
  74. item_class = self.app.model.User
  75. elif class_name == 'Group':
  76. item_class = self.app.model.Group
  77. elif class_name == 'Role':
  78. item_class = self.app.model.Role
  79. elif class_name == 'Quota':
  80. item_class = self.app.model.Quota
  81. elif class_name == 'Library':
  82. item_class = self.app.model.Library
  83. elif class_name == 'LibraryFolder':
  84. item_class = self.app.model.LibraryFolder
  85. elif class_name == 'LibraryDatasetDatasetAssociation':
  86. item_class = self.app.model.LibraryDatasetDatasetAssociation
  87. elif class_name == 'LibraryDataset':
  88. item_class = self.app.model.LibraryDataset
  89. elif class_name == 'ToolShedRepository':
  90. item_class = self.app.install_model.ToolShedRepository
  91. else:
  92. item_class = None
  93. return item_class
  94. def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ):
  95. """
  96. Convenience method to get a model object with the specified checks.
  97. """
  98. try:
  99. decoded_id = trans.security.decode_id( id )
  100. except:
  101. raise MessageException( "Malformed %s id ( %s ) specified, unable to decode"
  102. % ( class_name, str( id ) ), type='error' )
  103. try:
  104. item_class = self.get_class( class_name )
  105. assert item_class is not None
  106. item = trans.sa_session.query( item_class ).get( decoded_id )
  107. assert item is not None
  108. except Exception, exc:
  109. log.exception( "Invalid %s id ( %s ) specified: %s" % ( class_name, id, str( exc ) ) )
  110. raise MessageException( "Invalid %s id ( %s ) specified" % ( class_name, id ), type="error" )
  111. if check_ownership or check_accessible:
  112. self.security_check( trans, item, check_ownership, check_accessible )
  113. if deleted == True and not item.deleted:
  114. raise ItemDeletionException( '%s "%s" is not deleted'
  115. % ( class_name, getattr( item, 'name', id ) ), type="warning" )
  116. elif deleted == False and item.deleted:
  117. raise ItemDeletionException( '%s "%s" is deleted'
  118. % ( class_name, getattr( item, 'name', id ) ), type="warning" )
  119. return item
  120. # this should be here - but catching errors from sharable item controllers that *should* have SharableItemMixin
  121. # but *don't* then becomes difficult
  122. #def security_check( self, trans, item, check_ownership=False, check_accessible=False ):
  123. # log.warn( 'BaseController.security_check: %s, %b, %b', str( item ), check_ownership, check_accessible )
  124. # # meant to be overridden in SharableSecurityMixin
  125. # return item
  126. def get_user( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ):
  127. return self.get_object( trans, id, 'User', check_ownership=False, check_accessible=False, deleted=deleted )
  128. def get_group( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ):
  129. return self.get_object( trans, id, 'Group', check_ownership=False, check_accessible=False, deleted=deleted )
  130. def get_role( self, trans, id, check_ownership=False, check_accessible=False, deleted=None ):
  131. return self.get_object( trans, id, 'Role', check_ownership=False, check_accessible=False, deleted=deleted )
  132. def encode_all_ids( self, trans, rval, recursive=False ):
  133. """
  134. Encodes all integer values in the dict rval whose keys are 'id' or end with '_id'
  135. It might be useful to turn this in to a decorator
  136. """
  137. if type( rval ) != dict:
  138. return rval
  139. for k, v in rval.items():
  140. if (k == 'id' or k.endswith( '_id' )) and v is not None and k not in ['tool_id']:
  141. try:
  142. rval[k] = trans.security.encode_id( v )
  143. except:
  144. pass # probably already encoded
  145. if (k.endswith("_ids") and type(v) == list):
  146. try:
  147. o = []
  148. for i in v:
  149. o.append(trans.security.encode_id( i ))
  150. rval[k] = o
  151. except:
  152. pass
  153. else:
  154. if recursive and type(v) == dict:
  155. rval[k] = self.encode_all_ids(trans, v, recursive)
  156. return rval
  157. # incoming param validation
  158. # should probably be in sep. serializer class/object _used_ by controller
  159. def validate_and_sanitize_basestring( self, key, val ):
  160. if not isinstance( val, basestring ):
  161. raise exceptions.RequestParameterInvalidException( '%s must be a string or unicode: %s'
  162. %( key, str( type( val ) ) ) )
  163. return unicode( sanitize_html( val, 'utf-8', 'text/html' ), 'utf-8' )
  164. def validate_and_sanitize_basestring_list( self, key, val ):
  165. try:
  166. assert isinstance( val, list )
  167. return [ unicode( sanitize_html( t, 'utf-8', 'text/html' ), 'utf-8' ) for t in val ]
  168. except ( AssertionError, TypeError ), err:
  169. raise exceptions.RequestParameterInvalidException( '%s must be a list of strings: %s'
  170. %( key, str( type( val ) ) ) )
  171. def validate_boolean( self, key, val ):
  172. if not isinstance( val, bool ):
  173. raise exceptions.RequestParameterInvalidException( '%s must be a boolean: %s'
  174. %( key, str( type( val ) ) ) )
  175. return val
  176. #TODO:
  177. #def validate_integer( self, key, val, min, max ):
  178. #def validate_float( self, key, val, min, max ):
  179. #def validate_number( self, key, val, min, max ):
  180. #def validate_genome_build( self, key, val ):
  181. Root = BaseController
  182. class BaseUIController( BaseController ):
  183. def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ):
  184. try:
  185. return BaseController.get_object( self, trans, id, class_name,
  186. check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
  187. except MessageException:
  188. raise # handled in the caller
  189. except:
  190. log.exception( "Execption in get_object check for %s %s:" % ( class_name, str( id ) ) )
  191. raise Exception( 'Server error retrieving %s id ( %s ).' % ( class_name, str( id ) ) )
  192. class BaseAPIController( BaseController ):
  193. def get_object( self, trans, id, class_name, check_ownership=False, check_accessible=False, deleted=None ):
  194. try:
  195. return BaseController.get_object( self, trans, id, class_name,
  196. check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
  197. except ItemDeletionException, e:
  198. raise HTTPBadRequest( detail="Invalid %s id ( %s ) specified: %s" % ( class_name, str( id ), str( e ) ) )
  199. except MessageException, e:
  200. raise HTTPBadRequest( detail=e.err_msg )
  201. except Exception, e:
  202. log.exception( "Execption in get_object check for %s %s: %s" % ( class_name, str( id ), str( e ) ) )
  203. raise HTTPInternalServerError( comment=str( e ) )
  204. def validate_in_users_and_groups( self, trans, payload ):
  205. """
  206. For convenience, in_users and in_groups can be encoded IDs or emails/group names in the API.
  207. """
  208. def get_id( item, model_class, column ):
  209. try:
  210. return trans.security.decode_id( item )
  211. except:
  212. pass # maybe an email/group name
  213. # this will raise if the item is invalid
  214. return trans.sa_session.query( model_class ).filter( column == item ).first().id
  215. new_in_users = []
  216. new_in_groups = []
  217. invalid = []
  218. for item in util.listify( payload.get( 'in_users', [] ) ):
  219. try:
  220. new_in_users.append( get_id( item, trans.app.model.User, trans.app.model.User.table.c.email ) )
  221. except:
  222. invalid.append( item )
  223. for item in util.listify( payload.get( 'in_groups', [] ) ):
  224. try:
  225. new_in_groups.append( get_id( item, trans.app.model.Group, trans.app.model.Group.table.c.name ) )
  226. except:
  227. invalid.append( item )
  228. if invalid:
  229. msg = "The following value(s) for associated users and/or groups could not be parsed: %s." % ', '.join( invalid )
  230. msg += " Valid values are email addresses of users, names of groups, or IDs of both."
  231. raise Exception( msg )
  232. payload['in_users'] = map( str, new_in_users )
  233. payload['in_groups'] = map( str, new_in_groups )
  234. def not_implemented( self, trans, **kwd ):
  235. raise HTTPNotImplemented()
  236. class Datatype( object ):
  237. """Used for storing in-memory list of datatypes currently in the datatypes registry."""
  238. def __init__( self, extension, dtype, type_extension, mimetype, display_in_upload ):
  239. self.extension = extension
  240. self.dtype = dtype
  241. self.type_extension = type_extension
  242. self.mimetype = mimetype
  243. self.display_in_upload = display_in_upload
  244. #
  245. # -- Mixins for working with Galaxy objects. --
  246. #
  247. class CreatesUsersMixin:
  248. """
  249. Mixin centralizing logic for user creation between web and API controller.
  250. Web controller handles additional features such e-mail subscription, activation,
  251. user forms, etc.... API created users are much more vanilla for the time being.
  252. """
  253. def create_user( self, trans, email, username, password ):
  254. user = trans.app.model.User( email=email )
  255. user.set_password_cleartext( password )
  256. user.username = username
  257. if trans.app.config.user_activation_on:
  258. user.active = False
  259. else:
  260. user.active = True # Activation is off, every new user is active by default.
  261. trans.sa_session.add( user )
  262. trans.sa_session.flush()
  263. trans.app.security_agent.create_private_user_role( user )
  264. if trans.webapp.name == 'galaxy':
  265. # We set default user permissions, before we log in and set the default history permissions
  266. trans.app.security_agent.user_set_default_permissions( user,
  267. default_access_private=trans.app.config.new_user_dataset_access_role_default_private )
  268. return user
  269. class CreatesApiKeysMixin:
  270. """
  271. Mixing centralizing logic for creating API keys for user objects.
  272. """
  273. def create_api_key( self, trans, user ):
  274. guid = trans.app.security.get_new_guid()
  275. new_key = trans.app.model.APIKeys()
  276. new_key.user_id = user.id
  277. new_key.key = guid
  278. trans.sa_session.add( new_key )
  279. trans.sa_session.flush()
  280. return guid
  281. class SharableItemSecurityMixin:
  282. """ Mixin for handling security for sharable items. """
  283. def security_check( self, trans, item, check_ownership=False, check_accessible=False ):
  284. """ Security checks for an item: checks if (a) user owns item or (b) item is accessible to user. """
  285. # all items are accessible to an admin
  286. if trans.user and trans.user_is_admin():
  287. return item
  288. # Verify ownership: there is a current user and that user is the same as the item's
  289. if check_ownership:
  290. if not trans.user:
  291. raise ItemOwnershipException( "Must be logged in to manage Galaxy items", type='error' )
  292. if item.user != trans.user:
  293. raise ItemOwnershipException( "%s is not owned by the current user" % item.__class__.__name__, type='error' )
  294. # Verify accessible:
  295. # if it's part of a lib - can they access via security
  296. # if it's something else (sharable) have they been added to the item's users_shared_with_dot_users
  297. if check_accessible:
  298. if type( item ) in ( trans.app.model.LibraryFolder, trans.app.model.LibraryDatasetDatasetAssociation, trans.app.model.LibraryDataset ):
  299. if not trans.app.security_agent.can_access_library_item( trans.get_current_user_roles(), item, trans.user ):
  300. raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' )
  301. else:
  302. if ( item.user != trans.user ) and ( not item.importable ) and ( trans.user not in item.users_shared_with_dot_users ):
  303. raise ItemAccessibilityException( "%s is not accessible to the current user" % item.__class__.__name__, type='error' )
  304. return item
  305. class UsesHistoryMixin( SharableItemSecurityMixin ):
  306. """ Mixin for controllers that use History objects. """
  307. def get_history( self, trans, id, check_ownership=True, check_accessible=False, deleted=None ):
  308. """
  309. Get a History from the database by id, verifying ownership.
  310. """
  311. history = self.get_object( trans, id, 'History',
  312. check_ownership=check_ownership, check_accessible=check_accessible, deleted=deleted )
  313. history = self.security_check( trans, history, check_ownership, check_accessible )
  314. return history
  315. def get_user_histories( self, trans, user=None, include_deleted=False, only_deleted=False ):
  316. """
  317. Get all the histories for a given user (defaulting to `trans.user`)
  318. ordered by update time and filtered on whether they've been deleted.
  319. """
  320. # handle default and/or anonymous user (which still may not have a history yet)
  321. user = user or trans.user
  322. if not user:
  323. current_history = trans.get_history()
  324. return [ current_history ] if current_history else []
  325. history_model = trans.model.History
  326. query = ( trans.sa_session.query( history_model )
  327. .filter( history_model.user == user )
  328. .order_by( desc( history_model.table.c.update_time ) ) )
  329. if only_deleted:
  330. query = query.filter( history_model.deleted == True )
  331. elif not include_deleted:
  332. query = query.filter( history_model.deleted == False )
  333. return query.all()
  334. def get_history_datasets( self, trans, history, show_deleted=False, show_hidden=False, show_purged=False ):
  335. """ Returns history's datasets. """
  336. query = trans.sa_session.query( trans.model.HistoryDatasetAssociation ) \
  337. .filter( trans.model.HistoryDatasetAssociation.history == history ) \
  338. .options( eagerload( "children" ) ) \
  339. .join( "dataset" ) \
  340. .options( eagerload_all( "dataset.actions" ) ) \
  341. .order_by( trans.model.HistoryDatasetAssociation.hid )
  342. if not show_deleted:
  343. query = query.filter( trans.model.HistoryDatasetAssociation.deleted == False )
  344. if not show_purged:
  345. query = query.filter( trans.model.Dataset.purged == False )
  346. return query.all()
  347. def get_hda_state_counts( self, trans, history, include_deleted=False, include_hidden=False ):
  348. """
  349. Returns a dictionary with state counts for history's HDAs. Key is a
  350. dataset state, value is the number of states in that count.
  351. """
  352. # Build query to get (state, count) pairs.
  353. cols_to_select = [ trans.app.model.Dataset.table.c.state, func.count( '*' ) ]
  354. from_obj = trans.app.model.HistoryDatasetAssociation.table.join( trans.app.model.Dataset.table )
  355. conditions = [ trans.app.model.HistoryDatasetAssociation.table.c.history_id == history.id ]
  356. if not include_deleted:
  357. # Only count datasets that have not been deleted.
  358. conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.deleted == False )
  359. if not include_hidden:
  360. # Only count datasets that are visible.
  361. conditions.append( trans.app.model.HistoryDatasetAssociation.table.c.visible == True )
  362. group_by = trans.app.model.Dataset.table.c.state
  363. query = select( columns=cols_to_select,
  364. from_obj=from_obj,
  365. whereclause=and_( *conditions ),
  366. group_by=group_by )
  367. # Initialize count dict with all states.
  368. state_count_dict = {}
  369. for k, state in trans.app.model.Dataset.states.items():
  370. state_count_dict[ state ] = 0
  371. # Process query results, adding to count dict.
  372. for row in trans.sa_session.execute( query ):
  373. state, count = row
  374. state_count_dict[ state ] = count
  375. return state_count_dict
  376. def get_hda_summary_dicts( self, trans, history ):
  377. """Returns a list of dictionaries containing summary information
  378. for each HDA in the given history.
  379. """
  380. hda_model = trans.model.HistoryDatasetAssociation
  381. # get state, name, etc.
  382. columns = ( hda_model.name, hda_model.hid, hda_model.id, hda_model.deleted,
  383. trans.model.Dataset.state )
  384. column_keys = [ "name", "hid", "id", "deleted", "state" ]
  385. query = ( trans.sa_session.query( *columns )
  386. .enable_eagerloads( False )
  387. .filter( hda_model.history == history )
  388. .join( trans.model.Dataset )
  389. .order_by( hda_model.hid ) )
  390. # build dictionaries, adding history id and encoding all ids
  391. hda_dicts = []
  392. for hda_tuple in query.all():
  393. hda_dict = dict( zip( column_keys, hda_tuple ) )
  394. hda_dict[ 'history_id' ] = history.id
  395. trans.security.encode_dict_ids( hda_dict )
  396. hda_dicts.append( hda_dict )
  397. return hda_dicts
  398. def _get_hda_state_summaries( self, trans, hda_dict_list ):
  399. """Returns two dictionaries (in a tuple): state_counts and state_ids.
  400. Each is keyed according to the possible hda states:
  401. _counts contains a sum of the datasets in each state
  402. _ids contains a list of the encoded ids for each hda in that state
  403. hda_dict_list should be a list of hda data in dictionary form.
  404. """
  405. #TODO: doc to rst
  406. # init counts, ids for each state
  407. state_counts = {}
  408. state_ids = {}
  409. for key, state in trans.app.model.Dataset.states.items():
  410. state_counts[ state ] = 0
  411. state_ids[ state ] = []
  412. for hda_dict in hda_dict_list:
  413. item_state = hda_dict['state']
  414. if not hda_dict['deleted']:
  415. state_counts[ item_state ] = state_counts[ item_state ] + 1
  416. # needs to return all ids (no deleted check)
  417. state_ids[ item_state ].append( hda_dict['id'] )
  418. return ( state_counts, state_ids )
  419. def _get_history_state_from_hdas( self, trans, history, hda_state_counts ):
  420. """Returns the history state based on the states of the HDAs it contains.
  421. """
  422. states = trans.app.model.Dataset.states
  423. num_hdas = sum( hda_state_counts.values() )
  424. # (default to ERROR)
  425. state = states.ERROR
  426. if num_hdas == 0:
  427. state = states.NEW
  428. else:
  429. if( ( hda_state_counts[ states.RUNNING ] > 0 )
  430. or ( hda_state_counts[ states.SETTING_METADATA ] > 0 )
  431. or ( hda_state_counts[ states.UPLOAD ] > 0 ) ):
  432. state = states.RUNNING
  433. elif hda_state_counts[ states.QUEUED ] > 0:
  434. state = states.QUEUED
  435. elif( ( hda_state_counts[ states.ERROR ] > 0 )
  436. or ( hda_state_counts[ states.FAILED_METADATA ] > 0 ) ):
  437. state = states.ERROR
  438. elif hda_state_counts[ states.OK ] == num_hdas:
  439. state = states.OK
  440. return state
  441. def get_history_dict( self, trans, history, hda_dictionaries=None ):
  442. """Returns history data in the form of a dictionary.
  443. """
  444. history_dict = history.to_dict( view='element', value_mapper={ 'id':trans.security.encode_id })
  445. history_dict[ 'user_id' ] = None
  446. if history.user_id:
  447. history_dict[ 'user_id' ] = trans.security.encode_id( history.user_id )
  448. history_dict[ 'nice_size' ] = history.get_disk_size( nice_size=True )
  449. history_dict[ 'annotation' ] = history.get_item_annotation_str( trans.sa_session, trans.user, history )
  450. if not history_dict[ 'annotation' ]:
  451. history_dict[ 'annotation' ] = ''
  452. #TODO: item_slug url
  453. if history_dict[ 'importable' ] and history_dict[ 'slug' ]:
  454. #TODO: this should be in History (or a superclass of)
  455. username_and_slug = ( '/' ).join(( 'u', history.user.username, 'h', history_dict[ 'slug' ] ))
  456. history_dict[ 'username_and_slug' ] = username_and_slug
  457. hda_summaries = hda_dictionaries if hda_dictionaries else self.get_hda_summary_dicts( trans, history )
  458. #TODO remove the following in v2
  459. ( state_counts, state_ids ) = self._get_hda_state_summaries( trans, hda_summaries )
  460. history_dict[ 'state_details' ] = state_counts
  461. history_dict[ 'state_ids' ] = state_ids
  462. history_dict[ 'state' ] = self._get_history_state_from_hdas( trans, history, state_counts )
  463. return history_dict
  464. def set_history_from_dict( self, trans, history, new_data ):
  465. """
  466. Changes history data using the given dictionary new_data.
  467. """
  468. #precondition: ownership of the history has already been checked
  469. #precondition: user is not None (many of these attributes require a user to set properly)
  470. user = trans.get_user()
  471. # published histories should always be importable
  472. if 'published' in new_data and new_data[ 'published' ] and not history.importable:
  473. new_data[ 'importable' ] = True
  474. # send what we can down into the model
  475. changed = history.set_from_dict( new_data )
  476. # the rest (often involving the trans) - do here
  477. #TODO: the next two could be an aspect/mixin
  478. #TODO: also need a way to check whether they've changed - assume they have for now
  479. if 'annotation' in new_data:
  480. history.add_item_annotation( trans.sa_session, user, history, new_data[ 'annotation' ] )
  481. changed[ 'annotation' ] = new_data[ 'annotation' ]
  482. if 'tags' in new_data:
  483. self.set_tags_from_list( trans, history, new_data[ 'tags' ], user=user )
  484. changed[ 'tags' ] = new_data[ 'tags' ]
  485. #TODO: sharing with user/permissions?
  486. if changed.keys():
  487. trans.sa_session.flush()
  488. # create a slug if none exists (setting importable to false should not remove the slug)
  489. if 'importable' in changed and changed[ 'importable' ] and not history.slug:
  490. self._create_history_slug( trans, history )
  491. return changed
  492. def _create_history_slug( self, trans, history ):
  493. #TODO: mixins need to die a quick, horrible death
  494. # (this is duplicate from SharableMixin which can't be added to UsesHistory without exposing various urls)
  495. cur_slug = history.slug
  496. # Setup slug base.
  497. if cur_slug is None or cur_slug == "":
  498. # Item can have either a name or a title.
  499. item_name = history.name
  500. slug_base = util.ready_name_for_url( item_name.lower() )
  501. else:
  502. slug_base = cur_slug
  503. # Using slug base, find a slug that is not taken. If slug is taken,
  504. # add integer to end.
  505. new_slug = slug_base
  506. count = 1
  507. while ( trans.sa_session.query( trans.app.model.History )
  508. .filter_by( user=history.user, slug=new_slug, importable=True )
  509. .count() != 0 ):
  510. # Slug taken; choose a new slug based on count. This approach can
  511. # handle numerous items with the same name gracefully.
  512. new_slug = '%s-%i' % ( slug_base, count )
  513. count += 1
  514. # Set slug and return.
  515. trans.sa_session.add( history )
  516. history.slug = new_slug
  517. trans.sa_session.flush()
  518. return history.slug == cur_slug
  519. class ExportsHistoryMixin:
  520. def serve_ready_history_export( self, trans, jeha ):
  521. assert jeha.ready
  522. if jeha.compressed:
  523. trans.response.set_content_type( 'application/x-gzip' )
  524. else:
  525. trans.response.set_content_type( 'application/x-tar' )
  526. disposition = 'attachment; filename="%s"' % jeha.export_name
  527. trans.response.headers["Content-Disposition"] = disposition
  528. return open( trans.app.object_store.get_filename( jeha.dataset ) )
  529. def queue_history_export( self, trans, history, gzip=True, include_hidden=False, include_deleted=False ):
  530. # Convert options to booleans.
  531. #
  532. if isinstance( gzip, basestring ):
  533. gzip = ( gzip in [ 'True', 'true', 'T', 't' ] )
  534. if isinstance( include_hidden, basestring ):
  535. include_hidden = ( include_hidden in [ 'True', 'true', 'T', 't' ] )
  536. if isinstance( include_deleted, basestring ):
  537. include_deleted = ( include_deleted in [ 'True', 'true', 'T', 't' ] )
  538. # Run job to do export.
  539. history_exp_tool = trans.app.toolbox.get_tool( '__EXPORT_HISTORY__' )
  540. params = {
  541. 'history_to_export': history,
  542. 'compress': gzip,
  543. 'include_hidden': include_hidden,
  544. 'include_deleted': include_deleted
  545. }
  546. history_exp_tool.execute( trans, incoming=params, history=history, set_output_hid=True )
  547. class ImportsHistoryMixin:
  548. def queue_history_import( self, trans, archive_type, archive_source ):
  549. # Run job to do import.
  550. history_imp_tool = trans.app.toolbox.get_tool( '__IMPORT_HISTORY__' )
  551. incoming = { '__ARCHIVE_SOURCE__' : archive_source, '__ARCHIVE_TYPE__' : archive_type }
  552. history_imp_tool.execute( trans, incoming=incoming )
  553. class UsesHistoryDatasetAssociationMixin:
  554. """
  555. Mixin for controllers that use HistoryDatasetAssociation objects.
  556. """
  557. def get_dataset( self, trans, dataset_id, check_ownership=True, check_accessible=False, check_state=True ):
  558. """
  559. Get an HDA object by id performing security checks using
  560. the current transaction.
  561. """
  562. try:
  563. dataset_id = trans.security.decode_id( dataset_id )
  564. except ( AttributeError, TypeError ):
  565. # DEPRECATION: We still support unencoded ids for backward compatibility
  566. try:
  567. dataset_id = int( dataset_id )
  568. except ValueError, v_err:
  569. raise HTTPBadRequest( "Invalid dataset id: %s." % str( dataset_id ) )
  570. try:
  571. data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( int( dataset_id ) )
  572. except:
  573. raise HTTPRequestRangeNotSatisfiable( "Invalid dataset id: %s." % str( dataset_id ) )
  574. if check_ownership:
  575. # Verify ownership.
  576. user = trans.get_user()
  577. if not user:
  578. error( "Must be logged in to manage Galaxy items" )
  579. if data.history.user != user:
  580. error( "%s is not owned by current user" % data.__class__.__name__ )
  581. if check_accessible:
  582. current_user_roles = trans.get_current_user_roles()
  583. if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
  584. error( "You are not allowed to access this dataset" )
  585. if check_state and data.state == trans.model.Dataset.states.UPLOAD:
  586. return trans.show_error_message( "Please wait until this dataset finishes uploading "
  587. + "before attempting to view it." )
  588. return data
  589. def get_history_dataset_association( self, trans, history, dataset_id,
  590. check_ownership=True, check_accessible=False, check_state=False ):
  591. """
  592. Get a HistoryDatasetAssociation from the database by id, verifying ownership.
  593. """
  594. #TODO: duplicate of above? alias to above (or vis-versa)
  595. self.security_check( trans, history, check_ownership=check_ownership, check_accessible=check_accessible )
  596. hda = self.get_object( trans, dataset_id, 'HistoryDatasetAssociation',
  597. check_ownership=False, check_accessible=False )
  598. if check_accessible:
  599. if( not trans.user_is_admin()
  600. and not trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ) ):
  601. error( "You are not allowed to access this dataset" )
  602. if check_state and hda.state == trans.model.Dataset.states.UPLOAD:
  603. error( "Please wait until this dataset finishes uploading before attempting to view it." )
  604. return hda
  605. def get_history_dataset_association_from_ids( self, trans, id, history_id ):
  606. # Just to echo other TODOs, there seems to be some overlap here, still
  607. # this block appears multiple places (dataset show, history_contents
  608. # show, upcoming history job show) so I am consolodating it here.
  609. # Someone smarter than me should determine if there is some redundancy here.
  610. # for anon users:
  611. #TODO: check login_required?
  612. #TODO: this isn't actually most_recently_used (as defined in histories)
  613. if( ( trans.user == None )
  614. and ( history_id == trans.security.encode_id( trans.history.id ) ) ):
  615. history = trans.history
  616. #TODO: dataset/hda by id (from history) OR check_ownership for anon user
  617. hda = self.get_history_dataset_association( trans, history, id,
  618. check_ownership=False, check_accessible=True )
  619. else:
  620. #TODO: do we really need the history?
  621. history = self.get_history( trans, history_id,
  622. check_ownership=False, check_accessible=True, deleted=False )
  623. hda = self.get_history_dataset_association( trans, history, id,
  624. check_ownership=False, check_accessible=True )
  625. return hda
  626. def get_hda_list( self, trans, hda_ids, check_ownership=True, check_accessible=False, check_state=True ):
  627. """
  628. Returns one or more datasets in a list.
  629. If a dataset is not found or is inaccessible to trans.user,
  630. add None in its place in the list.
  631. """
  632. # precondtion: dataset_ids is a list of encoded id strings
  633. hdas = []
  634. for id in hda_ids:
  635. hda = None
  636. try:
  637. hda = self.get_dataset( trans, id,
  638. check_ownership=check_ownership,
  639. check_accessible=check_accessible,
  640. check_state=check_state )
  641. except Exception, exception:
  642. pass
  643. hdas.append( hda )
  644. return hdas
  645. def get_data( self, dataset, preview=True ):
  646. """
  647. Gets a dataset's data.
  648. """
  649. # Get data from file, truncating if necessary.
  650. truncated = False
  651. dataset_data = None
  652. if os.path.exists( dataset.file_name ):
  653. if isinstance( dataset.datatype, Text ):
  654. max_peek_size = 1000000 # 1 MB
  655. if preview and os.stat( dataset.file_name ).st_size > max_peek_size:
  656. dataset_data = open( dataset.file_name ).read(max_peek_size)
  657. truncated = True
  658. else:
  659. dataset_data = open( dataset.file_name ).read(max_peek_size)
  660. truncated = False
  661. else:
  662. # For now, cannot get data from non-text datasets.
  663. dataset_data = None
  664. return truncated, dataset_data
  665. def check_dataset_state( self, trans, dataset ):
  666. """
  667. Returns a message if dataset is not ready to be used in visualization.
  668. """
  669. if not dataset:
  670. return dataset.conversion_messages.NO_DATA
  671. if dataset.state == trans.app.model.Job.states.ERROR:
  672. return dataset.conversion_messages.ERROR
  673. if dataset.state != trans.app.model.Job.states.OK:
  674. return dataset.conversion_messages.PENDING
  675. return None
  676. def get_hda_dict( self, trans, hda ):
  677. """Return full details of this HDA in dictionary form.
  678. """
  679. #precondition: the user's access to this hda has already been checked
  680. #TODO:?? postcondition: all ids are encoded (is this really what we want at this level?)
  681. expose_dataset_path = trans.user_is_admin() or trans.app.config.expose_dataset_path
  682. hda_dict = hda.to_dict( view='element', expose_dataset_path=expose_dataset_path )
  683. hda_dict[ 'api_type' ] = "file"
  684. # Add additional attributes that depend on trans can hence must be added here rather than at the model level.
  685. can_access_hda = trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset )
  686. can_access_hda = ( trans.user_is_admin() or can_access_hda )
  687. if not can_access_hda:
  688. return self.get_inaccessible_hda_dict( trans, hda )
  689. hda_dict[ 'accessible' ] = True
  690. #TODO: I'm unclear as to which access pattern is right
  691. hda_dict[ 'annotation' ] = hda.get_item_annotation_str( trans.sa_session, trans.user, hda )
  692. #annotation = getattr( hda, 'annotation', hda.get_item_annotation_str( trans.sa_session, trans.user, hda ) )
  693. # ---- return here if deleted AND purged OR can't access
  694. purged = ( hda.purged or hda.dataset.purged )
  695. if ( hda.deleted and purged ):
  696. #TODO: to_dict should really go AFTER this - only summary data
  697. return trans.security.encode_dict_ids( hda_dict )
  698. if expose_dataset_path:
  699. try:
  700. hda_dict[ 'file_name' ] = hda.file_name
  701. except objectstore.ObjectNotFound, onf:
  702. log.exception( 'objectstore.ObjectNotFound, HDA %s: %s', hda.id, onf )
  703. hda_dict[ 'download_url' ] = url_for( 'history_contents_display',
  704. history_id = trans.security.encode_id( hda.history.id ),
  705. history_content_id = trans.security.encode_id( hda.id ) )
  706. # indeces, assoc. metadata files, etc.
  707. meta_files = []
  708. for meta_type in hda.metadata.spec.keys():
  709. if isinstance( hda.metadata.spec[ meta_type ].param, FileParameter ):
  710. meta_files.append( dict( file_type=meta_type ) )
  711. if meta_files:
  712. hda_dict[ 'meta_files' ] = meta_files
  713. # currently, the viz reg is optional - handle on/off
  714. if trans.app.visualizations_registry:
  715. hda_dict[ 'visualizations' ] = trans.app.visualizations_registry.get_visualizations( trans, hda )
  716. else:
  717. hda_dict[ 'visualizations' ] = hda.get_visualizations()
  718. #TODO: it may also be wiser to remove from here and add as API call that loads the visualizations
  719. # when the visualizations button is clicked (instead of preloading/pre-checking)
  720. # ---- return here if deleted
  721. if hda.deleted and not purged:
  722. return trans.security.encode_dict_ids( hda_dict )
  723. return trans.security.encode_dict_ids( hda_dict )
  724. def get_inaccessible_hda_dict( self, trans, hda ):
  725. return trans.security.encode_dict_ids({
  726. 'id' : hda.id,
  727. 'history_id': hda.history.id,
  728. 'hid' : hda.hid,
  729. 'name' : hda.name,
  730. 'state' : hda.state,
  731. 'deleted' : hda.deleted,
  732. 'visible' : hda.visible,
  733. 'accessible': False
  734. })
  735. def get_hda_dict_with_error( self, trans, hda=None, history_id=None, id=None, error_msg='Error' ):
  736. return trans.security.encode_dict_ids({
  737. 'id' : hda.id if hda else id,
  738. 'history_id': hda.history.id if hda else history_id,
  739. 'hid' : hda.hid if hda else '(unknown)',
  740. 'name' : hda.name if hda else '(unknown)',
  741. 'error' : error_msg,
  742. 'state' : trans.model.Dataset.states.NEW
  743. })
  744. def get_display_apps( self, trans, hda ):
  745. display_apps = []
  746. for display_app in hda.get_display_applications( trans ).itervalues():
  747. app_links = []
  748. for link_app in display_app.links.itervalues():
  749. app_links.append({
  750. 'target': link_app.url.get( 'target_frame', '_blank' ),
  751. 'href' : link_app.get_display_url( hda, trans ),
  752. 'text' : gettext( link_app.name )
  753. })
  754. if app_links:
  755. display_apps.append( dict( label=display_app.name, links=app_links ) )
  756. return display_apps
  757. def get_old_display_applications( self, trans, hda ):
  758. display_apps = []
  759. if not trans.app.config.enable_old_display_applications:
  760. return display_apps
  761. for display_app in hda.datatype.get_display_types():
  762. target_frame, display_links = hda.datatype.get_display_links( hda,
  763. display_app, trans.app, trans.request.base )
  764. if len( display_links ) > 0:
  765. display_label = hda.datatype.get_display_label( display_app )
  766. app_links = []
  767. for display_name, display_link in display_links:
  768. app_links.append({
  769. 'target': target_frame,
  770. 'href' : display_link,
  771. 'text' : gettext( display_name )
  772. })
  773. if app_links:
  774. display_apps.append( dict( label=display_label, links=app_links ) )
  775. return display_apps
  776. def set_hda_from_dict( self, trans, hda, new_data ):
  777. """
  778. Changes HDA data using the given dictionary new_data.
  779. """
  780. # precondition: access of the hda has already been checked
  781. # send what we can down into the model
  782. changed = hda.set_from_dict( new_data )
  783. # the rest (often involving the trans) - do here
  784. if 'annotation' in new_data.keys() and trans.get_user():
  785. hda.add_item_annotation( trans.sa_session, trans.get_user(), hda, new_data[ 'annotation' ] )
  786. changed[ 'annotation' ] = new_data[ 'annotation' ]
  787. if 'tags' in new_data.keys() and trans.get_user():
  788. self.set_tags_from_list( trans, hda, new_data[ 'tags' ], user=trans.user )
  789. # sharing/permissions?
  790. # purged
  791. if changed.keys():
  792. trans.sa_session.flush()
  793. return changed
  794. def get_hda_job( self, hda ):
  795. # Get dataset's job.
  796. job = None
  797. for job_output_assoc in hda.creating_job_associations:
  798. job = job_output_assoc.job
  799. break
  800. return job
  801. def stop_hda_creating_job( self, hda ):
  802. """
  803. Stops an HDA's creating job if all the job's other outputs are deleted.
  804. """
  805. if hda.parent_id is None and len( hda.creating_job_associations ) > 0:
  806. # Mark associated job for deletion
  807. job = hda.creating_job_associations[0].job
  808. if job.state in [ self.app.model.Job.states.QUEUED, self.app.model.Job.states.RUNNING, self.app.model.Job.states.NEW ]:
  809. # Are *all* of the job's other output datasets deleted?
  810. if job.check_if_output_datasets_deleted():
  811. job.mark_deleted( self.app.config.track_jobs_in_database )
  812. self.app.job_manager.job_stop_queue.put( job.id )
  813. class UsesLibraryMixin:
  814. def get_library( self, trans, id, check_ownership=False, check_accessible=True ):
  815. l = self.get_object( trans, id, 'Library' )
  816. if check_accessible and not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( trans.get_current_user_roles(), l ) ):
  817. error( "LibraryFolder is not accessible to the current user" )
  818. return l
  819. class UsesLibraryMixinItems( SharableItemSecurityMixin ):
  820. def get_library_folder( self, trans, id, check_ownership=False, check_accessible=True ):
  821. return self.get_object( trans, id, 'LibraryFolder',
  822. check_ownership=False, check_accessible=check_accessible )
  823. def get_library_dataset_dataset_association( self, trans, id, check_ownership=False, check_accessible=True ):
  824. return self.get_object( trans, id, 'LibraryDatasetDatasetAssociation',
  825. check_ownership=False, check_accessible=check_accessible )
  826. def get_library_dataset( self, trans, id, check_ownership=False, check_accessible=True ):
  827. return self.get_object( trans, id, 'LibraryDataset',
  828. check_ownership=False, check_accessible=check_accessible )
  829. #TODO: it makes no sense that I can get roles from a user but not user.is_admin()
  830. #def can_user_add_to_library_item( self, trans, user, item ):
  831. # if not user: return False
  832. # return ( ( user.is_admin() )
  833. # or ( trans.app.security_agent.can_add_library_item( user.all_roles(), item ) ) )
  834. def can_current_user_add_to_library_item( self, trans, item ):
  835. if not trans.user: return False
  836. return ( ( trans.user_is_admin() )
  837. or ( trans.app.security_agent.can_add_library_item( trans.get_current_user_roles(), item ) ) )
  838. def copy_hda_to_library_folder( self, trans, hda, library_folder, roles=None, ldda_message='' ):
  839. #PRECONDITION: permissions for this action on hda and library_folder have been checked
  840. roles = roles or []
  841. # this code was extracted from library_common.add_history_datasets_to_library
  842. #TODO: refactor library_common.add_history_datasets_to_library to use this for each hda to copy
  843. # create the new ldda and apply the folder perms to it
  844. ldda = hda.to_library_dataset_dataset_association( trans, target_folder=library_folder,
  845. roles=roles, ldda_message=ldda_message )
  846. self._apply_library_folder_permissions_to_ldda( trans, library_folder, ldda )
  847. self._apply_hda_permissions_to_ldda( trans, hda, ldda )
  848. #TODO:?? not really clear on how permissions are being traded here
  849. # seems like hda -> ldda permissions should be set in to_library_dataset_dataset_association
  850. # then they get reset in _apply_library_folder_permissions_to_ldda
  851. # then finally, re-applies hda -> ldda for missing actions in _apply_hda_permissions_to_ldda??
  852. return ldda
  853. def _apply_library_folder_permissions_to_ldda( self, trans, library_folder, ldda ):
  854. """
  855. Copy actions/roles from library folder to an ldda (and it's library_dataset).
  856. """
  857. #PRECONDITION: permissions for this action on library_folder and ldda have been checked
  858. security_agent = trans.app.security_agent
  859. security_agent.copy_library_permissions( trans, library_folder, ldda )
  860. security_agent.copy_library_permissions( trans, library_folder, ldda.library_dataset )
  861. return security_agent.get_permissions( ldda )
  862. def _apply_hda_permissions_to_ldda( self, trans, hda, ldda ):
  863. """
  864. Copy actions/roles from hda to ldda.library_dataset (and then ldda) if ldda
  865. doesn't already have roles for the given action.
  866. """
  867. #PRECONDITION: permissions for this action on hda and ldda have been checked
  868. # Make sure to apply any defined dataset permissions, allowing the permissions inherited from the
  869. # library_dataset to over-ride the same permissions on the dataset, if they exist.
  870. security_agent = trans.app.security_agent
  871. dataset_permissions_dict = security_agent.get_permissions( hda.dataset )
  872. library_dataset = ldda.library_dataset
  873. library_dataset_actions = [ permission.action for permission in library_dataset.actions ]
  874. # except that: if DATASET_MANAGE_PERMISSIONS exists in the hda.dataset permissions,
  875. # we need to instead apply those roles to the LIBRARY_MANAGE permission to the library dataset
  876. dataset_manage_permissions_action = security_agent.get_action( 'DATASET_MANAGE_PERMISSIONS' ).action
  877. library_manage_permissions_action = security_agent.get_action( 'LIBRARY_MANAGE' ).action
  878. #TODO: test this and remove if in loop below
  879. #TODO: doesn't handle action.action
  880. #if dataset_manage_permissions_action in dataset_permissions_dict:
  881. # managing_roles = dataset_permissions_dict.pop( dataset_manage_permissions_action )
  882. # dataset_permissions_dict[ library_manage_permissions_action ] = managing_roles
  883. flush_needed = False
  884. for action, dataset_permissions_roles in dataset_permissions_dict.items():
  885. if isinstance( action, security.Action ):
  886. action = action.action
  887. # alter : DATASET_MANAGE_PERMISSIONS -> LIBRARY_MANAGE (see above)
  888. if action == dataset_manage_permissions_action:
  889. action = library_manage_permissions_action
  890. #TODO: generalize to util.update_dict_without_overwrite
  891. # add the hda actions & roles to the library_dataset
  892. #NOTE: only apply an hda perm if it's NOT set in the library_dataset perms (don't overwrite)
  893. if action not in library_dataset_actions:
  894. for role in dataset_permissions_roles:
  895. ldps = trans.model.LibraryDatasetPermissions( action, library_dataset, role )
  896. ldps = [ ldps ] if not isinstance( ldps, list ) else ldps
  897. for ldp in ldps:
  898. trans.sa_session.add( ldp )
  899. flush_needed = True
  900. if flush_needed:
  901. trans.sa_session.flush()
  902. # finally, apply the new library_dataset to it's associated ldda (must be the same)
  903. security_agent.copy_library_permissions( trans, library_dataset, ldda )
  904. return security_agent.get_permissions( ldda )
  905. class UsesVisualizationMixin( UsesHistoryDatasetAssociationMixin, UsesLibraryMixinItems ):
  906. """
  907. Mixin for controllers that use Visualization objects.
  908. """
  909. viz_types =