PageRenderTime 207ms CodeModel.GetById 24ms app.highlight 163ms RepoModel.GetById 1ms app.codeStats 0ms

/test/base/twilltestcase.py

https://bitbucket.org/cistrome/cistrome-harvard/
Python | 2600 lines | 2584 code | 9 blank | 7 comment | 39 complexity | df943a8c086a1e8b21a50a6cb0a60990 MD5 | raw file
   1import difflib
   2import filecmp
   3import logging
   4import os
   5import re
   6import pprint
   7import shutil
   8import StringIO
   9import subprocess
  10import tarfile
  11import tempfile
  12import time
  13import unittest
  14import urllib
  15import zipfile
  16
  17from galaxy.web import security
  18from galaxy.web.framework.helpers import iff
  19from galaxy.util.json import from_json_string
  20from base.asserts import verify_assertions
  21
  22from galaxy import eggs
  23eggs.require( "elementtree" )
  24eggs.require( 'twill' )
  25
  26from elementtree import ElementTree
  27
  28import twill
  29import twill.commands as tc
  30from twill.other_packages._mechanize_dist import ClientForm
  31
  32#Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose?
  33buffer = StringIO.StringIO()
  34twill.set_output( buffer )
  35tc.config( 'use_tidy', 0 )
  36
  37# Dial ClientCookie logging down (very noisy)
  38logging.getLogger( "ClientCookie.cookies" ).setLevel( logging.WARNING )
  39log = logging.getLogger( __name__ )
  40
  41
  42class TwillTestCase( unittest.TestCase ):
  43
  44    def setUp( self ):
  45        # Security helper
  46        self.security = security.SecurityHelper( id_secret='changethisinproductiontoo' )
  47        self.history_id = os.environ.get( 'GALAXY_TEST_HISTORY_ID', None )
  48        self.host = os.environ.get( 'GALAXY_TEST_HOST' )
  49        self.port = os.environ.get( 'GALAXY_TEST_PORT' )
  50        self.url = "http://%s:%s" % ( self.host, self.port )
  51        self.file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR', None )
  52        self.tool_shed_test_file = os.environ.get( 'GALAXY_TOOL_SHED_TEST_FILE', None )
  53        if self.tool_shed_test_file:
  54            f = open( self.tool_shed_test_file, 'r' )
  55            text = f.read()
  56            f.close()
  57            self.shed_tools_dict = from_json_string( text )
  58        else:
  59            self.shed_tools_dict = {}
  60        self.keepOutdir = os.environ.get( 'GALAXY_TEST_SAVE', '' )
  61        if self.keepOutdir > '':
  62            try:
  63                os.makedirs(self.keepOutdir)
  64            except:
  65                pass
  66        self.home()
  67
  68    # Functions associated with files
  69    def files_diff( self, file1, file2, attributes=None ):
  70        """Checks the contents of 2 files for differences"""
  71        def get_lines_diff( diff ):
  72            count = 0
  73            for line in diff:
  74                if ( line.startswith( '+' ) and not line.startswith( '+++' ) ) or ( line.startswith( '-' ) and not line.startswith( '---' ) ):
  75                    count += 1
  76            return count
  77        if not filecmp.cmp( file1, file2 ):
  78            files_differ = False
  79            local_file = open( file1, 'U' ).readlines()
  80            history_data = open( file2, 'U' ).readlines()
  81            if attributes is None:
  82                attributes = {}
  83            if attributes.get( 'sort', False ):
  84                history_data.sort()
  85            ##Why even bother with the check loop below, why not just use the diff output? This seems wasteful.
  86            if len( local_file ) == len( history_data ):
  87                for i in range( len( history_data ) ):
  88                    if local_file[i].rstrip( '\r\n' ) != history_data[i].rstrip( '\r\n' ):
  89                        files_differ = True
  90                        break
  91            else:
  92                files_differ = True
  93            if files_differ:
  94                allowed_diff_count = int(attributes.get( 'lines_diff', 0 ))
  95                diff = list( difflib.unified_diff( local_file, history_data, "local_file", "history_data" ) )
  96                diff_lines = get_lines_diff( diff )
  97                if diff_lines > allowed_diff_count:
  98                    if len(diff) < 60:
  99                        diff_slice = diff[0:40]
 100                    else:
 101                        diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:]
 102                    #FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff
 103                    #due to unknown desired behavior when used in conjunction with a non-zero lines_diff
 104                    #PDF forgiveness can probably be handled better by not special casing by __extension__ here
 105                    #and instead using lines_diff or a regular expression matching
 106                    #or by creating and using a specialized pdf comparison function
 107                    if file1.endswith( '.pdf' ) or file2.endswith( '.pdf' ):
 108                        # PDF files contain creation dates, modification dates, ids and descriptions that change with each
 109                        # new file, so we need to handle these differences.  As long as the rest of the PDF file does
 110                        # not differ we're ok.
 111                        valid_diff_strs = [ 'description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator' ]
 112                        valid_diff = False
 113                        invalid_diff_lines = 0
 114                        for line in diff_slice:
 115                            # Make sure to lower case strings before checking.
 116                            line = line.lower()
 117                            # Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
 118                            if ( line.startswith( '+' ) or line.startswith( '-' ) ) and line.find( 'local_file' ) < 0 and line.find( 'history_data' ) < 0:
 119                                for vdf in valid_diff_strs:
 120                                    if line.find( vdf ) < 0:
 121                                        valid_diff = False
 122                                    else:
 123                                        valid_diff = True
 124                                        # Stop checking as soon as we know we have a valid difference
 125                                        break
 126                                if not valid_diff:
 127                                    invalid_diff_lines += 1
 128                        log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines))
 129                        if invalid_diff_lines > allowed_diff_count:
 130                            # Print out diff_slice so we can see what failed
 131                            print "###### diff_slice ######"
 132                            raise AssertionError( "".join( diff_slice ) )
 133                    else:
 134                        log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines))
 135                        for line in diff_slice:
 136                            for char in line:
 137                                if ord( char ) > 128:
 138                                    raise AssertionError( "Binary data detected, not displaying diff" )
 139                        raise AssertionError( "".join( diff_slice )  )
 140
 141    def files_re_match( self, file1, file2, attributes=None ):
 142        """Checks the contents of 2 files for differences using re.match"""
 143        local_file = open( file1, 'U' ).readlines()  # regex file
 144        history_data = open( file2, 'U' ).readlines()
 145        assert len( local_file ) == len( history_data ), 'Data File and Regular Expression File contain a different number of lines (%s != %s)\nHistory Data (first 40 lines):\n%s' % ( len( local_file ), len( history_data ), ''.join( history_data[:40] ) )
 146        if attributes is None:
 147            attributes = {}
 148        if attributes.get( 'sort', False ):
 149            history_data.sort()
 150        lines_diff = int(attributes.get( 'lines_diff', 0 ))
 151        line_diff_count = 0
 152        diffs = []
 153        for i in range( len( history_data ) ):
 154            if not re.match( local_file[i].rstrip( '\r\n' ), history_data[i].rstrip( '\r\n' ) ):
 155                line_diff_count += 1
 156                diffs.append( 'Regular Expression: %s\nData file         : %s' % ( local_file[i].rstrip( '\r\n' ),  history_data[i].rstrip( '\r\n' ) ) )
 157            if line_diff_count > lines_diff:
 158                raise AssertionError( "Regular expression did not match data file (allowed variants=%i):\n%s" % ( lines_diff, "".join( diffs ) ) )
 159
 160    def files_re_match_multiline( self, file1, file2, attributes=None ):
 161        """Checks the contents of 2 files for differences using re.match in multiline mode"""
 162        local_file = open( file1, 'U' ).read()  # regex file
 163        if attributes is None:
 164            attributes = {}
 165        if attributes.get( 'sort', False ):
 166            history_data = open( file2, 'U' ).readlines()
 167            history_data.sort()
 168            history_data = ''.join( history_data )
 169        else:
 170            history_data = open( file2, 'U' ).read()
 171        #lines_diff not applicable to multiline matching
 172        assert re.match( local_file, history_data, re.MULTILINE ), "Multiline Regular expression did not match data file"
 173
 174    def files_contains( self, file1, file2, attributes=None ):
 175        """Checks the contents of file2 for substrings found in file1, on a per-line basis"""
 176        local_file = open( file1, 'U' ).readlines()  # regex file
 177        #TODO: allow forcing ordering of contains
 178        history_data = open( file2, 'U' ).read()
 179        lines_diff = int( attributes.get( 'lines_diff', 0 ) )
 180        line_diff_count = 0
 181        while local_file:
 182            contains = local_file.pop( 0 ).rstrip( '\n\r' )
 183            if contains not in history_data:
 184                line_diff_count += 1
 185            if line_diff_count > lines_diff:
 186                raise AssertionError( "Failed to find '%s' in history data. (lines_diff=%i):\n" % ( contains, lines_diff ) )
 187
 188    def get_filename( self, filename, shed_tool_id=None ):
 189        if shed_tool_id and self.shed_tools_dict:
 190            file_dir = self.shed_tools_dict[ shed_tool_id ]
 191            if not file_dir:
 192                file_dir = self.file_dir
 193        else:
 194            file_dir = self.file_dir
 195        return os.path.abspath( os.path.join( file_dir, filename ) )
 196
 197    def save_log( *path ):
 198        """Saves the log to a file"""
 199        filename = os.path.join( *path )
 200        file(filename, 'wt').write(buffer.getvalue())
 201
 202    def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab=False, metadata=None, composite_data=None, name=None, shed_tool_id=None, wait=True ):
 203        """
 204        Uploads a file.  If shed_tool_id has a value, we're testing tools migrated from the distribution to the tool shed,
 205        so the tool-data directory of test data files is contained in the installed tool shed repository.
 206        """
 207        self.visit_url( "%s/tool_runner?tool_id=upload1" % self.url )
 208        try:
 209            self.refresh_form( "file_type", ftype )  # Refresh, to support composite files
 210            tc.fv( "tool_form", "dbkey", dbkey )
 211            if metadata:
 212                for elem in metadata:
 213                    tc.fv( "tool_form", "files_metadata|%s" % elem.get( 'name' ), elem.get( 'value' ) )
 214            if composite_data:
 215                for i, composite_file in enumerate( composite_data ):
 216                    filename = self.get_filename( composite_file.get( 'value' ), shed_tool_id=shed_tool_id )
 217                    tc.formfile( "tool_form", "files_%i|file_data" % i, filename )
 218                    tc.fv( "tool_form", "files_%i|space_to_tab" % i, composite_file.get( 'space_to_tab', False ) )
 219            else:
 220                filename = self.get_filename( filename, shed_tool_id=shed_tool_id )
 221                tc.formfile( "tool_form", "file_data", filename )
 222                tc.fv( "tool_form", "space_to_tab", space_to_tab )
 223                if name:
 224                    # NAME is a hidden form element, so the following prop must
 225                    # set to use it.
 226                    tc.config("readonly_controls_writeable", 1)
 227                    tc.fv( "tool_form", "NAME", name )
 228            tc.submit( "runtool_btn" )
 229            self.home()
 230        except AssertionError, err:
 231            errmsg = "Uploading file resulted in the following exception.  Make sure the file (%s) exists.  " % filename
 232            errmsg += str( err )
 233            raise AssertionError( errmsg )
 234        if not wait:
 235            return
 236        # Make sure every history item has a valid hid
 237        hids = self.get_hids_in_history()
 238        for hid in hids:
 239            try:
 240                int( hid )
 241            except:
 242                raise AssertionError( "Invalid hid (%s) created when uploading file %s" % ( hid, filename ) )
 243        # Wait for upload processing to finish (TODO: this should be done in each test case instead)
 244        self.wait()
 245
 246    def upload_url_paste( self, url_paste, ftype='auto', dbkey='unspecified (?)' ):
 247        """Pasted data in the upload utility"""
 248        self.visit_page( "tool_runner/index?tool_id=upload1" )
 249        try: 
 250            self.refresh_form( "file_type", ftype ) #Refresh, to support composite files
 251            tc.fv( "tool_form", "dbkey", dbkey )
 252            tc.fv( "tool_form", "url_paste", url_paste )
 253            tc.submit( "runtool_btn" )
 254            self.home()
 255        except Exception, e:
 256            errmsg = "Problem executing upload utility using url_paste: %s" % str( e )
 257            raise AssertionError( errmsg )
 258        # Make sure every history item has a valid hid
 259        hids = self.get_hids_in_history()
 260        for hid in hids:
 261            try:
 262                int( hid )
 263            except:
 264                raise AssertionError( "Invalid hid (%s) created when pasting %s" % ( hid, url_paste ) )
 265        # Wait for upload processing to finish (TODO: this should be done in each test case instead)
 266        self.wait()
 267
 268    def json_from_url( self, url ):
 269        self.visit_url( url )
 270        return from_json_string( self.last_page() )
 271
 272    # Functions associated with histories
 273    def get_history_from_api( self, encoded_history_id=None ):
 274        if encoded_history_id is None:
 275            history = self.get_latest_history()
 276            encoded_history_id = history[ 'id' ]
 277        return self.json_from_url( '/api/histories/%s/contents' % encoded_history_id )
 278
 279    def get_latest_history( self ):
 280        return self.json_from_url( '/api/histories' )[ 0 ]
 281
 282    def find_hda_by_dataset_name( self, name, history=None ):
 283        if history is None:
 284            history = self.get_history_from_api()
 285        for hda in history:
 286            if hda[ 'name' ] == name:
 287                return hda
 288
 289    def check_history_for_errors( self ):
 290        """Raises an exception if there are errors in a history"""
 291        self.home()
 292        self.visit_page( "history" )
 293        page = self.last_page()
 294        if page.find( 'error' ) > -1:
 295            raise AssertionError( 'Errors in the history for user %s' % self.user )
 296
 297    def check_history_for_string( self, patt, show_deleted=False ):
 298        """Breaks patt on whitespace and searches for each element seperately in the history"""
 299        self.home()
 300        if show_deleted:
 301            self.visit_page( "history?show_deleted=True" )
 302        else:
 303            self.visit_page( "history" )
 304        for subpatt in patt.split():
 305            try:
 306                tc.find( subpatt )
 307            except:
 308                fname = self.write_temp_file( tc.browser.get_html() )
 309                errmsg = "no match to '%s'\npage content written to '%s'" % ( subpatt, fname )
 310                raise AssertionError( errmsg )
 311        self.home()
 312
 313    def check_history_for_exact_string( self, string, show_deleted=False ):
 314        """Looks for exact match to 'string' in history page"""
 315        self.home()
 316        if show_deleted:
 317            self.visit_page( "history?show_deleted=True" )
 318        else:
 319            self.visit_page( "history" )
 320        try:
 321            tc.find( string )
 322        except:
 323            fname = self.write_temp_file( tc.browser.get_html() )
 324            errmsg = "no match to '%s'\npage content written to '%s'" % ( string, fname )
 325            raise AssertionError( errmsg )
 326        self.home()
 327
 328    def check_history_json( self, pattern, check_fn, show_deleted=None, multiline=True ):
 329        """
 330        Tries to find a JSON string in the history page using the regex pattern,
 331        parse it, and assert check_fn returns True when called on that parsed
 332        data.
 333        """
 334        self.home()
 335        if show_deleted:
 336            self.visit_page( "history?show_deleted=True" )
 337        elif show_deleted == False:
 338            self.visit_page( "history?show_deleted=False" )
 339        else:
 340            self.visit_page( "history" )
 341        json_data = {}
 342        try:
 343            tc.find( pattern, flags=( 'm' if multiline else '' ) )
 344            # twill stores the regex match in a special stack variable
 345            match = twill.namespaces.get_twill_glocals()[1][ '__match__' ]
 346            json_data = from_json_string( match )
 347            assert check_fn( json_data ), 'failed check_fn: %s' % ( check_fn.func_name )
 348
 349        except Exception, exc:
 350            log.error( exc, exc_info=True )
 351            log.debug( 'json_data: %s', ( '\n' + pprint.pformat( json_data ) if json_data else '(no match)' ) )
 352            fname = self.write_temp_file( tc.browser.get_html() )
 353            errmsg = ( "json '%s' could not be found or failed check_fn" % ( pattern ) +
 354                       "\npage content written to '%s'" % ( fname ) )
 355            raise AssertionError( errmsg )
 356
 357        self.home()
 358
 359    def is_history_empty( self ):
 360        """
 361        Uses history page JSON to determine whether this history is empty
 362        (i.e. has no undeleted datasets).
 363        """
 364        return len( self.get_history_from_api() ) == 0
 365
 366    def check_hda_json_for_key_value( self, hda_id, key, value, use_string_contains=False ):
 367        """
 368        Uses the history API to determine whether the current history:
 369        (1) Has a history dataset with the required ID.
 370        (2) That dataset has the required key.
 371        (3) The contents of that key match the provided value.
 372        If use_string_contains=True, this will perform a substring match, otherwise an exact match.
 373        """
 374        #TODO: multi key, value
 375        hda = dict()
 376        for history_item in self.get_history_from_api():
 377            if history_item[ 'id' ] == hda_id:
 378                hda = self.json_from_url( history_item[ 'url' ] )
 379                break
 380        if hda:
 381            if key in hda:
 382                if use_string_contains:
 383                    return value in hda[ key ]
 384                else:
 385                    return value == hda[ key ]
 386        return False
 387
 388    def clear_history( self ):
 389        """Empties a history of all datasets"""
 390        self.visit_page( "clear_history" )
 391        self.check_history_for_string( 'Your history is empty' )
 392        self.home()
 393
 394    def delete_history( self, id ):
 395        """Deletes one or more histories"""
 396        history_list = self.get_histories_as_data_list()
 397        self.assertTrue( history_list )
 398        num_deleted = len( id.split( ',' ) )
 399        self.home()
 400        self.visit_page( "history/list?operation=delete&id=%s" % ( id ) )
 401
 402        check_str = 'Deleted %d %s' % ( num_deleted, iff( num_deleted != 1, "histories", "history" ) )
 403        self.check_page_for_string( check_str )
 404        self.home()
 405
 406    def delete_current_history( self, strings_displayed=[] ):
 407        """Deletes the current history"""
 408        self.home()
 409        self.visit_page( "history/delete_current" )
 410        for check_str in strings_displayed:
 411            self.check_page_for_string( check_str )
 412        self.home()
 413
 414    def get_histories_as_data_list( self ):
 415        """Returns the data elements of all histories"""
 416        tree = self.histories_as_xml_tree()
 417        data_list = [ elem for elem in tree.findall("data") ]
 418        return data_list
 419
 420    def get_history_as_data_list( self, show_deleted=False ):
 421        """Returns the data elements of a history"""
 422        tree = self.history_as_xml_tree( show_deleted=show_deleted )
 423        data_list = [ elem for elem in tree.findall("data") ]
 424        return data_list
 425
 426    def history_as_xml_tree( self, show_deleted=False ):
 427        """Returns a parsed xml object of a history"""
 428        self.home()
 429        self.visit_page( 'history?as_xml=True&show_deleted=%s' % show_deleted )
 430        xml = self.last_page()
 431        tree = ElementTree.fromstring(xml)
 432        return tree
 433
 434    def histories_as_xml_tree( self ):
 435        """Returns a parsed xml object of all histories"""
 436        self.home()
 437        self.visit_page( 'history/list_as_xml' )
 438        xml = self.last_page()
 439        tree = ElementTree.fromstring(xml)
 440        return tree
 441
 442    def history_options( self, user=False, active_datasets=False, activatable_datasets=False, histories_shared_by_others=False ):
 443        """Mimics user clicking on history options link"""
 444        self.home()
 445        self.visit_page( "root/history_options" )
 446        if user:
 447            self.check_page_for_string( 'Previously</a> stored histories' )
 448            if active_datasets:
 449                self.check_page_for_string( 'Create</a> a new empty history' )
 450                self.check_page_for_string( 'Construct workflow</a> from current history' )
 451                self.check_page_for_string( 'Copy</a> current history' )
 452            self.check_page_for_string( 'Share</a> current history' )
 453            self.check_page_for_string( 'Change default permissions</a> for current history' )
 454            if histories_shared_by_others:
 455                self.check_page_for_string( 'Histories</a> shared with you by others' )
 456        if activatable_datasets:
 457            self.check_page_for_string( 'Show deleted</a> datasets in current history' )
 458        self.check_page_for_string( 'Rename</a> current history' )
 459        self.check_page_for_string( 'Delete</a> current history' )
 460        self.home()
 461
 462    def new_history( self, name=None ):
 463        """Creates a new, empty history"""
 464        self.home()
 465        if name:
 466            self.visit_url( "%s/history_new?name=%s" % ( self.url, name ) )
 467        else:
 468            self.visit_url( "%s/history_new" % self.url )
 469        self.check_history_for_string('Your history is empty')
 470        self.home()
 471
 472    def rename_history( self, id, old_name, new_name ):
 473        """Rename an existing history"""
 474        self.home()
 475        self.visit_page( "history/rename?id=%s&name=%s" % ( id, new_name ) )
 476        check_str = 'History: %s renamed to: %s' % ( old_name, urllib.unquote( new_name ) )
 477        self.check_page_for_string( check_str )
 478        self.home()
 479
 480    def set_history( self ):
 481        """Sets the history (stores the cookies for this run)"""
 482        if self.history_id:
 483            self.home()
 484            self.visit_page( "history?id=%s" % self.history_id )
 485        else:
 486            self.new_history()
 487        self.home()
 488
 489    def share_current_history( self, email, strings_displayed=[], strings_displayed_after_submit=[],
 490                               action='', action_strings_displayed=[], action_strings_displayed_after_submit=[] ):
 491        """Share the current history with different users"""
 492        self.visit_url( "%s/history/share" % self.url )
 493        for check_str in strings_displayed:
 494            self.check_page_for_string( check_str )
 495        tc.fv( 'share', 'email', email )
 496        tc.submit( 'share_button' )
 497        for check_str in strings_displayed_after_submit:
 498            self.check_page_for_string( check_str )
 499        if action:
 500            # If we have an action, then we are sharing datasets with users that do not have access permissions on them
 501            for check_str in action_strings_displayed:
 502                self.check_page_for_string( check_str )
 503            tc.fv( 'share_restricted', 'action', action )
 504
 505            tc.submit( "share_restricted_button" )
 506            for check_str in action_strings_displayed_after_submit:
 507                self.check_page_for_string( check_str )
 508        self.home()
 509
 510    def share_histories_with_users( self, ids, emails, strings_displayed=[], strings_displayed_after_submit=[],
 511                                    action=None, action_strings_displayed=[] ):
 512        """Share one or more histories with one or more different users"""
 513        self.visit_url( "%s/history/list?id=%s&operation=Share" % ( self.url, ids ) )
 514        for check_str in strings_displayed:
 515            self.check_page_for_string( check_str )
 516        tc.fv( 'share', 'email', emails )
 517        tc.submit( 'share_button' )
 518        for check_str in strings_displayed_after_submit:
 519            self.check_page_for_string( check_str )
 520        if action:
 521            # If we have an action, then we are sharing datasets with users that do not have access permissions on them
 522            tc.fv( 'share_restricted', 'action', action )
 523            tc.submit( "share_restricted_button" )
 524
 525            for check_str in action_strings_displayed:
 526                self.check_page_for_string( check_str )
 527        self.home()
 528
 529    def unshare_history( self, history_id, user_id, strings_displayed=[] ):
 530        """Unshare a history that has been shared with another user"""
 531        self.visit_url( "%s/history/list?id=%s&operation=share+or+publish" % ( self.url, history_id ) )
 532        for check_str in strings_displayed:
 533            self.check_page_for_string( check_str )
 534        self.visit_url( "%s/history/sharing?unshare_user=%s&id=%s" % ( self.url, user_id, history_id ) )
 535        self.home()
 536
 537    def switch_history( self, id='', name='' ):
 538        """Switches to a history in the current list of histories"""
 539        self.visit_url( "%s/history/list?operation=switch&id=%s" % ( self.url, id ) )
 540        if name:
 541            self.check_history_for_exact_string( name )
 542        self.home()
 543
 544    def view_stored_active_histories( self, strings_displayed=[] ):
 545        self.home()
 546        self.visit_page( "history/list" )
 547        self.check_page_for_string( 'Saved Histories' )
 548        self.check_page_for_string( 'operation=Rename' )
 549        self.check_page_for_string( 'operation=Switch' )
 550        self.check_page_for_string( 'operation=Delete' )
 551        for check_str in strings_displayed:
 552            self.check_page_for_string( check_str )
 553        self.home()
 554
 555    def view_stored_deleted_histories( self, strings_displayed=[] ):
 556        self.home()
 557        self.visit_page( "history/list?f-deleted=True" )
 558        self.check_page_for_string( 'Saved Histories' )
 559        self.check_page_for_string( 'operation=Undelete' )
 560        for check_str in strings_displayed:
 561            self.check_page_for_string( check_str )
 562        self.home()
 563
 564    def view_shared_histories( self, strings_displayed=[] ):
 565        self.home()
 566        self.visit_page( "history/list_shared" )
 567        for check_str in strings_displayed:
 568            self.check_page_for_string( check_str )
 569        self.home()
 570
 571    def copy_history( self, history_id, copy_choice, strings_displayed=[], strings_displayed_after_submit=[] ):
 572        self.home()
 573        self.visit_page( "history/copy?id=%s" % history_id )
 574        for check_str in strings_displayed:
 575            self.check_page_for_string( check_str )
 576        tc.fv( '1', 'copy_choice', copy_choice )
 577        tc.submit( 'copy_choice_button' )
 578        for check_str in strings_displayed_after_submit:
 579            self.check_page_for_string( check_str )
 580        self.home()
 581
 582    def make_accessible_via_link( self, history_id, strings_displayed=[], strings_displayed_after_submit=[] ):
 583        self.home()
 584        self.visit_page( "history/list?operation=share+or+publish&id=%s" % history_id )
 585        for check_str in strings_displayed:
 586            self.check_page_for_string( check_str )
 587        # twill barfs on this form, possibly because it contains no fields, but not sure.
 588        # In any case, we have to mimic the form submission
 589        self.home()
 590        self.visit_page( 'history/sharing?id=%s&make_accessible_via_link=True' % history_id )
 591        for check_str in strings_displayed_after_submit:
 592            self.check_page_for_string( check_str )
 593        self.home()
 594
 595    def disable_access_via_link( self, history_id, strings_displayed=[], strings_displayed_after_submit=[] ):
 596        self.home()
 597        self.visit_page( "history/list?operation=share+or+publish&id=%s" % history_id )
 598        for check_str in strings_displayed:
 599            self.check_page_for_string( check_str )
 600        # twill barfs on this form, possibly because it contains no fields, but not sure.
 601        # In any case, we have to mimic the form submission
 602        self.home()
 603        self.visit_page( 'history/sharing?id=%s&disable_link_access=True' % history_id )
 604        for check_str in strings_displayed_after_submit:
 605            self.check_page_for_string( check_str )
 606        self.home()
 607
 608    def import_history_via_url( self, history_id, email, strings_displayed_after_submit=[] ):
 609        self.home()
 610        self.visit_page( "history/imp?&id=%s" % history_id )
 611        for check_str in strings_displayed_after_submit:
 612            self.check_page_for_string( check_str )
 613        self.home()
 614
 615    # Functions associated with datasets (history items) and meta data
 616    def _get_job_stream_output( self, hda_id, stream, format ):
 617        self.visit_page( "datasets/%s/%s" % ( self.security.encode_id( hda_id ), stream ) )
 618
 619        output = self.last_page()
 620        return self._format_stream( output, stream, format )
 621
 622    def _format_stream( self, output, stream, format ):
 623        if format:
 624            msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
 625            msg += output + "\n"
 626            msg += "----------------------- >> end tool %s << ------------------------\n" % stream
 627        else:
 628            msg = output
 629        return msg
 630
 631    def get_job_stdout( self, hda_id, format=False ):
 632        return self._get_job_stream_output( hda_id, 'stdout', format )
 633
 634    def get_job_stderr( self, hda_id, format=False ):
 635        return self._get_job_stream_output( hda_id, 'stderr', format )
 636
 637    def _assert_dataset_state( self, elem, state ):
 638        if elem.get( 'state' ) != state:
 639            errmsg = "Expecting dataset state '%s', but state is '%s'. Dataset blurb: %s\n\n" % ( state, elem.get('state'), elem.text.strip() )
 640            errmsg += self.get_job_stderr( elem.get( 'id' ), format=True )
 641            raise AssertionError( errmsg )
 642
 643    def check_metadata_for_string( self, patt, hid=None ):
 644        """Looks for 'patt' in the edit page when editing a dataset"""
 645        data_list = self.get_history_as_data_list()
 646        self.assertTrue( data_list )
 647        if hid is None:  # take last hid
 648            elem = data_list[-1]
 649            hid = int( elem.get('hid') )
 650        self.assertTrue( hid )
 651        self.visit_page( "dataset/edit?hid=%s" % hid )
 652        for subpatt in patt.split():
 653            tc.find(subpatt)
 654
 655    def delete_history_item( self, hda_id, strings_displayed=[] ):
 656        """Deletes an item from a history"""
 657        try:
 658            hda_id = int( hda_id )
 659        except:
 660            raise AssertionError( "Invalid hda_id '%s' - must be int" % hda_id )
 661        self.visit_url( "%s/datasets/%s/delete?show_deleted_on_refresh=False" % ( self.url, self.security.encode_id( hda_id ) ) )
 662        for check_str in strings_displayed:
 663            self.check_page_for_string( check_str )
 664
 665    def undelete_history_item( self, hda_id, strings_displayed=[] ):
 666        """Un-deletes a deleted item in a history"""
 667        try:
 668            hda_id = int( hda_id )
 669        except:
 670            raise AssertionError( "Invalid hda_id '%s' - must be int" % hda_id )
 671        self.visit_url( "%s/datasets/%s/undelete" % ( self.url, self.security.encode_id( hda_id ) ) )
 672        for check_str in strings_displayed:
 673            self.check_page_for_string( check_str )
 674
 675    def display_history_item( self, hda_id, strings_displayed=[] ):
 676        """Displays a history item - simulates eye icon click"""
 677        self.visit_url( '%s/datasets/%s/display/' % ( self.url, self.security.encode_id( hda_id ) ) )
 678        for check_str in strings_displayed:
 679            self.check_page_for_string( check_str )
 680        self.home()
 681
 682    def view_history( self, history_id, strings_displayed=[] ):
 683        """Displays a history for viewing"""
 684        self.visit_url( '%s/history/view?id=%s' % ( self.url, self.security.encode_id( history_id ) ) )
 685        for check_str in strings_displayed:
 686            self.check_page_for_string( check_str )
 687        self.home()
 688
 689    def edit_hda_attribute_info( self, hda_id, new_name='', new_info='', new_dbkey='', new_startcol='',
 690                                 strings_displayed=[], strings_not_displayed=[] ):
 691        """Edit history_dataset_association attribute information"""
 692        self.home()
 693        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
 694        submit_required = False
 695        self.check_page_for_string( 'Edit Attributes' )
 696        if new_name:
 697            tc.fv( 'edit_attributes', 'name', new_name )
 698            submit_required = True
 699        if new_info:
 700            tc.fv( 'edit_attributes', 'info', new_info )
 701            submit_required = True
 702        if new_dbkey:
 703            tc.fv( 'edit_attributes', 'dbkey', new_dbkey )
 704            submit_required = True
 705        if new_startcol:
 706            tc.fv( 'edit_attributes', 'startCol', new_startcol )
 707            submit_required = True
 708        if submit_required:
 709            tc.submit( 'save' )
 710            self.check_page_for_string( 'Attributes updated' )
 711        for check_str in strings_displayed:
 712            self.check_page_for_string( check_str )
 713        for check_str in strings_not_displayed:
 714            try:
 715                self.check_page_for_string( check_str )
 716                raise AssertionError( "String (%s) incorrectly displayed on Edit Attributes page." % check_str )
 717            except:
 718                pass
 719        self.home()
 720
 721    def check_hda_attribute_info( self, hda_id, strings_displayed=[] ):
 722        """Edit history_dataset_association attribute information"""
 723        for check_str in strings_displayed:
 724            self.check_page_for_string( check_str )
 725
 726    def auto_detect_metadata( self, hda_id ):
 727        """Auto-detect history_dataset_association metadata"""
 728        self.home()
 729        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
 730        self.check_page_for_string( 'This will inspect the dataset and attempt' )
 731        tc.fv( 'auto_detect', 'detect', 'Auto-detect' )
 732        tc.submit( 'detect' )
 733        try:
 734            self.check_page_for_string( 'Attributes have been queued to be updated' )
 735            self.wait()
 736        except AssertionError:
 737            self.check_page_for_string( 'Attributes updated' )
 738        #self.check_page_for_string( 'Attributes updated' )
 739        self.home()
 740
 741    def convert_format( self, hda_id, target_type ):
 742        """Convert format of history_dataset_association"""
 743        self.home()
 744        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
 745        self.check_page_for_string( 'This will inspect the dataset and attempt' )
 746        tc.fv( 'convert_data', 'target_type', target_type )
 747        tc.submit( 'convert_data' )
 748        self.check_page_for_string( 'The file conversion of Convert BED to GFF on data' )
 749        self.wait()  # wait for the format convert tool to finish before returning
 750        self.home()
 751
 752    def change_datatype( self, hda_id, datatype ):
 753        """Change format of history_dataset_association"""
 754        self.home()
 755        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
 756        self.check_page_for_string( 'This will change the datatype of the existing dataset but' )
 757        tc.fv( 'change_datatype', 'datatype', datatype )
 758        tc.submit( 'change' )
 759        self.check_page_for_string( 'Changed the type of dataset' )
 760        self.home()
 761
 762    def copy_history_item( self, source_dataset_id=None, target_history_id=None, all_target_history_ids=[],
 763                           deleted_history_ids=[] ):
 764        """
 765        Copy 1 history_dataset_association to 1 history (Limited by twill since it doesn't support multiple
 766        field names, such as checkboxes
 767        """
 768        self.home()
 769        self.visit_url( "%s/dataset/copy_datasets?source_dataset_ids=%s" % ( self.url, source_dataset_id ) )
 770        self.check_page_for_string( 'Source History:' )
 771        # Make sure all of users active histories are displayed
 772        for id in all_target_history_ids:
 773            self.check_page_for_string( id )
 774        # Make sure only active histories are displayed
 775        for id in deleted_history_ids:
 776            try:
 777                self.check_page_for_string( id )
 778                raise AssertionError( "deleted history id %d displayed in list of target histories" % id )
 779            except:
 780                pass
 781
 782        tc.fv( '1', 'target_history_id', target_history_id )
 783        tc.submit( 'do_copy' )
 784        check_str = '1 dataset copied to 1 history'
 785        self.check_page_for_string( check_str )
 786        self.home()
 787
 788    def get_hids_in_history( self ):
 789        """Returns the list of hid values for items in a history"""
 790        data_list = self.get_history_as_data_list()
 791        hids = []
 792        for elem in data_list:
 793            hid = elem.get('hid')
 794            hids.append(hid)
 795        return hids
 796
 797    def get_hids_in_histories( self ):
 798        """Returns the list of hids values for items in all histories"""
 799        data_list = self.get_histories_as_data_list()
 800        hids = []
 801        for elem in data_list:
 802            hid = elem.get('hid')
 803            hids.append(hid)
 804        return hids
 805
 806    def makeTfname(self, fname=None):
 807        """safe temp name - preserve the file extension for tools that interpret it"""
 808        suffix = os.path.split(fname)[-1]  # ignore full path
 809        fd, temp_prefix = tempfile.mkstemp(prefix='tmp', suffix=suffix)
 810        return temp_prefix
 811
 812    def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None, shed_tool_id=None ):
 813        """Verifies that the attributes and contents of a history item meet expectations"""
 814        if wait:
 815            self.wait( maxseconds=maxseconds )  # wait for job to finish
 816        data_list = self.get_history_as_data_list()
 817        self.assertTrue( data_list )
 818        if hid is None:  # take last hid
 819            elem = data_list[-1]
 820            hid = str( elem.get('hid') )
 821        else:
 822            hid = str( hid )
 823            elems = [ elem for elem in data_list if elem.get('hid') == hid ]
 824            self.assertTrue( len(elems) == 1 )
 825            elem = elems[0]
 826        self.assertTrue( hid )
 827        self._assert_dataset_state( elem, 'ok' )
 828        if filename is not None and self.is_zipped( filename ):
 829            errmsg = 'History item %s is a zip archive which includes invalid files:\n' % hid
 830            zip_file = zipfile.ZipFile( filename, "r" )
 831            name = zip_file.namelist()[0]
 832            test_ext = name.split( "." )[1].strip().lower()
 833            if not ( test_ext == 'scf' or test_ext == 'ab1' or test_ext == 'txt' ):
 834                raise AssertionError( errmsg )
 835            for name in zip_file.namelist():
 836                ext = name.split( "." )[1].strip().lower()
 837                if ext != test_ext:
 838                    raise AssertionError( errmsg )
 839        else:
 840            # See not in controllers/root.py about encoded_id.
 841            hda_id = self.security.encode_id( elem.get( 'id' ) )
 842            self.verify_hid( filename, hid=hid, hda_id=hda_id, attributes=attributes, shed_tool_id=shed_tool_id)
 843
 844    def verify_hid( self, filename, hda_id, attributes, shed_tool_id, hid="", dataset_fetcher=None):
 845        dataset_fetcher = dataset_fetcher or self.__default_dataset_fetcher()
 846        data = dataset_fetcher( hda_id )
 847        if attributes is not None and attributes.get( "assert_list", None ) is not None:
 848            try:
 849                verify_assertions(data, attributes["assert_list"])
 850            except AssertionError, err:
 851                errmsg = 'History item %s different than expected\n' % (hid)
 852                errmsg += str( err )
 853                raise AssertionError( errmsg )
 854        if filename is not None:
 855            local_name = self.get_filename( filename, shed_tool_id=shed_tool_id )
 856            temp_name = self.makeTfname(fname=filename)
 857            file( temp_name, 'wb' ).write( data )
 858
 859            # if the server's env has GALAXY_TEST_SAVE, save the output file to that dir
 860            if self.keepOutdir:
 861                ofn = os.path.join( self.keepOutdir, os.path.basename( local_name ) )
 862                log.debug( 'keepoutdir: %s, ofn: %s', self.keepOutdir, ofn )
 863                try:
 864                    shutil.copy( temp_name, ofn )
 865                except Exception, exc:
 866                    error_log_msg = ( 'TwillTestCase could not save output file %s to %s: ' % ( temp_name, ofn ) )
 867                    error_log_msg += str( exc )
 868                    log.error( error_log_msg, exc_info=True )
 869                else:
 870                    log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % ( self.keepOutdir, ofn ) )
 871            try:
 872                if attributes is None:
 873                    attributes = {}
 874                compare = attributes.get( 'compare', 'diff' )
 875                if attributes.get( 'ftype', None ) == 'bam':
 876                    local_fh, temp_name = self._bam_to_sam( local_name, temp_name )
 877                    local_name = local_fh.name
 878                extra_files = attributes.get( 'extra_files', None )
 879                if compare == 'diff':
 880                    self.files_diff( local_name, temp_name, attributes=attributes )
 881                elif compare == 're_match':
 882                    self.files_re_match( local_name, temp_name, attributes=attributes )
 883                elif compare == 're_match_multiline':
 884                    self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
 885                elif compare == 'sim_size':
 886                    delta = attributes.get('delta', '100')
 887                    s1 = len(data)
 888                    s2 = os.path.getsize(local_name)
 889                    if abs(s1 - s2) > int(delta):
 890                        raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) )
 891                elif compare == "contains":
 892                    self.files_contains( local_name, temp_name, attributes=attributes )
 893                else:
 894                    raise Exception( 'Unimplemented Compare type: %s' % compare )
 895                if extra_files:
 896                    self.verify_extra_files_content( extra_files, hda_id, shed_tool_id=shed_tool_id, dataset_fetcher=dataset_fetcher )
 897            except AssertionError, err:
 898                errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare )
 899                errmsg += "( %s v. %s )\n" % ( local_name, temp_name )
 900                errmsg += str( err )
 901                raise AssertionError( errmsg )
 902            finally:
 903                if 'GALAXY_TEST_NO_CLEANUP' not in os.environ:
 904                    os.remove( temp_name )
 905
 906    def __default_dataset_fetcher( self ):
 907        def fetcher( hda_id, filename=None ):
 908            if filename is None:
 909                page_url = "display?encoded_id=%s" % hda_id
 910                self.home()  # I assume this is not needed.
 911            else:
 912                page_url = "datasets/%s/display/%s" % ( hda_id, filename )
 913            self.visit_page( page_url )
 914            data = self.last_page()
 915            return data
 916
 917        return fetcher
 918
 919    def _bam_to_sam( self, local_name, temp_name ):
 920        temp_local = tempfile.NamedTemporaryFile( suffix='.sam', prefix='local_bam_converted_to_sam_' )
 921        fd, temp_temp = tempfile.mkstemp( suffix='.sam', prefix='history_bam_converted_to_sam_' )
 922        os.close( fd )
 923        p = subprocess.Popen( args='samtools view -h -o "%s" "%s"' % ( temp_local.name, local_name  ), shell=True )
 924        assert not p.wait(), 'Converting local (test-data) bam to sam failed'
 925        p = subprocess.Popen( args='samtools view -h -o "%s" "%s"' % ( temp_temp, temp_name  ), shell=True )
 926        assert not p.wait(), 'Converting history bam to sam failed'
 927        os.remove( temp_name )
 928        return temp_local, temp_temp
 929
 930    def verify_extra_files_content( self, extra_files, hda_id, dataset_fetcher, shed_tool_id=None ):
 931        files_list = []
 932        for extra_type, extra_value, extra_name, extra_attributes in extra_files:
 933            if extra_type == 'file':
 934                files_list.append( ( extra_name, extra_value, extra_attributes ) )
 935            elif extra_type == 'directory':
 936                for filename in os.listdir( self.get_filename( extra_value, shed_tool_id=shed_tool_id ) ):
 937                    files_list.append( ( filename, os.path.join( extra_value, filename ), extra_attributes ) )
 938            else:
 939                raise ValueError( 'unknown extra_files type: %s' % extra_type )
 940        for filename, filepath, attributes in files_list:
 941            self.verify_composite_datatype_file_content( filepath, hda_id, base_name=filename, attributes=attributes, dataset_fetcher=dataset_fetcher, shed_tool_id=shed_tool_id )
 942
 943    def verify_composite_datatype_file_content( self, file_name, hda_id, base_name=None, attributes=None, dataset_fetcher=None, shed_tool_id=None ):
 944        dataset_fetcher = dataset_fetcher or self.__default_dataset_fetcher()
 945        local_name = self.get_filename( file_name, shed_tool_id=shed_tool_id )
 946        if base_name is None:
 947            base_name = os.path.split(file_name)[-1]
 948        temp_name = self.makeTfname(fname=base_name)
 949        data = dataset_fetcher( hda_id, base_name )
 950        file( temp_name, 'wb' ).write( data )
 951        if self.keepOutdir > '':
 952            ofn = os.path.join(self.keepOutdir, base_name)
 953            shutil.copy(temp_name, ofn)
 954            log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % (self.keepOutdir, ofn))
 955        try:
 956            if attributes is None:
 957                attributes = {}
 958            compare = attributes.get( 'compare', 'diff' )
 959            if compare == 'diff':
 960                self.files_diff( local_name, temp_name, attributes=attributes )
 961            elif compare == 're_match':
 962                self.files_re_match( local_name, temp_name, attributes=attributes )
 963            elif compare == 're_match_multiline':
 964                self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
 965            elif compare == 'sim_size':
 966                delta = attributes.get('delta', '100')
 967                s1 = len(data)
 968                s2 = os.path.getsize(local_name)
 969                if abs(s1 - s2) > int(delta):
 970                    raise Exception( 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name, s1, local_name, s2, delta) )
 971            else:
 972                raise Exception( 'Unimplemented Compare type: %s' % compare )
 973        except AssertionError, err:
 974            errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare )
 975            errmsg += str( err )
 976            raise AssertionError( errmsg )
 977        finally:
 978            if 'GALAXY_TEST_NO_CLEANUP' not in os.environ:
 979                os.remove( temp_name )
 980
 981    def is_zipped( self, filename ):
 982        if not zipfile.is_zipfile( filename ):
 983            return False
 984        return True
 985
 986    def is_binary( self, filename ):
 987        temp = open( filename, "U" )  # why is this not filename? Where did temp_name come from
 988        lineno = 0
 989        for line in temp:
 990            lineno += 1
 991            line = line.strip()
 992            if line:
 993                for char in line:
 994                    if ord( char ) > 128:
 995                        return True
 996            if lineno > 10:
 997                break
 998        return False
 999
1000    def verify_genome_build( self, dbkey='hg17' ):
1001        """Verifies that the last used genome_build at history id 'hid' is as expected"""
1002        data_list = self.get_history_as_data_list()
1003        self.assertTrue( data_list )
1004        elems = [ elem for elem in data_list ]
1005        elem = elems[-1]
1006        genome_build = elem.get('dbkey')
1007        self.assertTrue( genome_build == dbkey )
1008
1009    # Functions associated with user accounts
1010    def create( self, cntrller='user', email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ):
1011        # HACK: don't use panels because late_javascripts() messes up the twill browser and it
1012        # can't find form fields (and hence user can't be logged in).
1013        self.visit_url( "%s/user/create?cntrller=%s&use_panels=False" % ( self.url, cntrller ) )
1014        tc.fv( 'registration', 'email', email )
1015        tc.fv( 'registration', 'redirect', redirect )
1016        tc.fv( 'registration', 'password', password )
1017        tc.fv( 'registration', 'confirm', password )
1018        tc.fv( 'registration', 'username', username )
1019        tc.submit( 'create_user_button' )
1020        previously_created = False
1021        username_taken = False
1022        invalid_username = False
1023        try:
1024            self.check_page_for_string( "Created new user account" )
1025        except:
1026            try:
1027                # May have created the account in a previous test run...
1028                self.check_page_for_string( "User with that email already exists" )
1029                previously_created = True
1030            except:
1031                try:
1032                    self.check_page_for_string( 'Public name is taken; please choose another' )
1033                    username_taken = True
1034                except:
1035                    try:
1036                        # Note that we're only checking if the usr name is >< 4 chars here...
1037                        self.check_page_for_string( 'Public name must be at least 4 characters in length' )
1038                        invalid_username = True
1039                    except:
1040                        pass
1041        return previously_created, username_taken, invalid_username
1042
1043    def create_user_with_info( self, email, password, username, user_info_values, user_type_fd_id='', cntrller='user',
1044                               strings_displayed=[], strings_displayed_after_submit=[] ):
1045        # This method creates a new user with associated info
1046        self.visit_url( "%s/user/create?cntrller=%s&use_panels=False" % ( self.url, cntrller ) )
1047        tc.fv( "1", "email", email )
1048        tc.fv( "1", "password", password )
1049        tc.fv( "1", "confirm", password )
1050        tc.fv( "1", "username", username )
1051        if user_type_fd_id:
1052            # The user_type_fd_id SelectField requires a refresh_on_change
1053            self.refresh_form( 'user_type_fd_id', user_type_fd_id )
1054            tc.fv( "1", "password", password )
1055            tc.fv( "1", "confirm", password )
1056            for index, ( field_name, info_value ) in enumerate( user_info_values ):
1057                tc.fv( "1", field_name, info_value )
1058        for check_str in strings_displayed:
1059            self.check_page_for_string( check_str)
1060        tc.submit( "create_user_button" )
1061
1062    def edit_user_info( self, cntrller='user', id='', new_email='', new_username='', password='', new_password='',
1063                        info_values=[], strings_displayed=[], strings_displayed_after_submit=[] ):
1064        if cntrller == 'admin':
1065            url = "%s/admin/users?id=%s&operation=information" % ( self.url, id )
1066        else:  # cntrller == 'user:
1067            # The user is editing his own info, so the user id is gotten from trans.user.
1068            url = "%s/user/manage_user_info?cntrller=user" % self.url
1069        self.visit_url( url )
1070        for check_str in strings_displayed:
1071            self.check_page_for_string( check_str )
1072        if new_email or new_username:
1073            if new_email:
1074                tc.fv( "login_info", "email", new_email )
1075            if new_username:
1076                tc.fv( "login_info", "username", new_username )
1077            tc.submit( "login_info_button" )
1078        if password and new_password:
1079            tc.fv( "change_password", "current", password )
1080            tc.fv( "change_password", "password", new_password )
1081            tc.fv( "change_password", "confirm", new_password )
1082            tc.submit( "change_password_button" )
1083        if info_values:
1084            for index, ( field_name, info_value ) in enumerate( info_values ):
1085                tc.fv( "user_info", field_name, info_value )
1086            tc.submit( "edit_user_info_button" )
1087        for check_str in strings_displayed_after_submit:
1088            self.check_page_for_string( check_str )
1089        self.home()
1090
1091    def user_set_default_permissions( self, cntrller='user', permissions_out=[], permissions_in=[], role_id='2' ):
1092        # role.id = 2 is Private Role for test2@bx.psu.edu
1093        # NOTE: Twill has a bug that requires the ~/user/permissions page to contain at least 1 option value
1094        # in each select list or twill throws an exception, which is: ParseError: OPTION outside of SELECT
1095        # Due to this bug, we'll bypass visiting the page, and simply pass the permissions on to the
1096        # /user/set_default_permissions method.
1097        url = "user/set_default_permissions?cntrller=%s&update_roles_button=Save&id=None" % cntrller
1098        for po in permissions_out:
1099            key = '%s_out' % po
1100            url = "%s&%s=%s" % ( url, key, str( role_id ) )
1101        for pi in permissions_in:
1102            key = '%s_in' % pi
1103            url = "%s&%s=%s" % ( url, key, str( role_id ) )
1104        self.visit_url( "%s/%s" % ( self.url, url ) )
1105        self.check_page_for_string( 'Default new history permissions have been changed.' )
1106        self.home()
1107
1108    def history_set_default_permissions( self, permissions_out=[], permissions_in=[], role_id=3 ):  # role.id = 3 is Private Role for test3@bx.psu.edu
1109        # NOTE: Twill has a bug that requires the ~/user/permissions page to contain at least 1 option value
1110        # in each select list or twill throws an exception, which is: ParseError: OPTION outside of SELECT
1111        # Due to this bug, we'll bypass visiting the page, and simply pass the permissions on to the
1112        # /user/set_default_permissions method.
1113        url = "root/history_set_default_permissions?update_roles_button=Save&id=None&dataset=True"
1114        for po in permissions_out:
1115            key = '%s_out' % po
1116            url = "%s&%s=%s" % ( url, key, str( role_id ) )
1117        for pi in permissions_in:
1118            key = '%s_in' % pi
1119            url = "%s&%s=%s" % ( url, key, str( role_id ) )
1120        self.home()
1121        self.visit_url( "%s/%s" % ( self.url, url ) )
1122        self.check_page_for_string( 'Default history permissions have been changed.' )
1123        self.home()
1124
1125    def login( self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ):
1126        # test@bx.psu.edu is configured as an admin user
1127        previously_created, username_taken, invalid_username = \
1128            self.create( email=email, password=password, username=username, redirect=redirect )
1129        if previously_created:
1130            # The acount has previously been created, so just login.
1131            # HACK: don't use panels because late_javascripts() messes up the twill browser and it
1132            # can't find form fields (and hence user can't be logged in).
1133            self.visit_url( "%s/user/login?use_panels=False" % self.url )
1134            self.submit_form( 1, 'login_button', email=email, redirect=redirect, password=password )
1135
1136    def logout( self ):
1137        self.home()
1138        self.visit_page( "user/logout" )
1139        self.check_page_for_string( "You have been logged out" )
1140        self.home()
1141
1142    # Functions associated with browsers, cookies, HTML forms and page visits
1143
1144    def check_for_strings( self, strings_displayed=[], strings_not_displayed=[] ):
1145        if strings_displayed:
1146            for string in strings_displayed:
1147                self.check_page_for_string( string )
1148        if strings_not_displayed:
1149            for string in strings_not_displayed:
1150                self.check_string_not_in_page( string )
1151
1152    def check_page_for_string( self, patt ):
1153        """Looks for 'patt' in the current browser page"""
1154        page = self.last_page()
1155        if page.find( patt ) == -1:
1156            fname = self.write_temp_file( page )
1157            errmsg = "no match to '%s'\npage content written to '%s'" % ( patt, fname )
1158            raise AssertionError( errmsg )
1159
1160    def check_string_count_in_page( self, patt, min_count ):
1161        """Checks the number of 'patt' occurrences in the current browser page"""
1162        page = self.last_page()
1163        patt_count = page.count( patt )
1164        # The number of occurrences of patt in the page should be at least min_count
1165        # so show error if patt_count is less than min_count
1166        if patt_count < min_count:
1167            fname = self.write_temp_file( page )
1168            errmsg = "%i occurrences of '%s' found instead of %i.\npage content written to '%s' " % ( min_count, patt, patt_count, fname )
1169            raise AssertionError( errmsg )
1170
1171    def check_string_not_in_page( self, patt ):
1172        """Checks to make sure 'patt' is NOT in the page."""
1173        page = self.last_page()
1174        if page.find( patt ) != -1:
1175            fname = self.write_temp_file( page )
1176            errmsg = "string (%s) incorrectly displayed in page.\npage content written to '%s'" % ( patt, fname )
1177            raise AssertionError( errmsg )
1178
1179    def check_page(self, strings_displayed, strings_displayed_count, strings_not_displayed):
1180        """Checks a page for strings displayed, not displayed and number of occurrences of a string"""
1181        for check_str in strings_displayed:
1182            self.check_page_for_string( check_str )
1183        for check_str, count in strings_displayed_count:
1184            self.check_string_count_in_page( check_str, count )
1185        for check_str in strings_not_displayed:
1186            self.check_string_not_in_page( check_str )
1187
1188    def write_temp_file( self, content, suffix='.html' ):
1189        fd, fname = tempfile.mkstemp( suffix=suffix, prefix='twilltestcase-' )
1190        f = os.fdopen( fd, "w" )
1191        f.write( content )
1192        f.close()
1193        return fname
1194
1195    def clear_cookies( self ):
1196        tc.clear_cookies()
1197
1198    def clear_form( self, form=0 ):
1199        """Clears a form"""
1200        tc.formclear(str(form))
1201
1202    def home( self ):
1203        self.visit_url( self.url )
1204
1205    def last_page( self ):
1206        return tc.browser.get_html()
1207
1208    def load_cookies( self, file, shed_tool_id=None ):
1209        filename = self.get_filename( file, shed_tool_id=shed_tool_id )
1210        tc.load_cookies(filename)
1211
1212    def reload_page( self ):
1213        tc.reload()
1214        tc.code(200)
1215
1216    def show_cookies( self ):
1217        return tc.show_cookies()
1218
1219    def showforms( self ):
1220        """Shows form, helpful for debugging new tests"""
1221        return tc.showforms()
1222
1223    def submit_form( self, form_no=0, button="runtool_btn", **kwd ):
1224        """Populates and submits a form from the keyword arguments."""
1225        # An HTMLForm contains a sequence of Controls.  Supported control classes are:
1226        # TextControl, FileControl, ListControl, RadioControl, CheckboxControl, SelectControl,
1227        # SubmitControl, ImageControl
1228        for i, f in enumerate( self.showforms() ):
1229            if i == form_no:
1230                break
1231        # To help with debugging a tool, print out the form controls when the test fails
1232        print "form '%s' contains the following controls ( note the values )" % f.name
1233        controls = {}
1234        formcontrols = []
1235        hc_prefix = '<HiddenControl('
1236        for i, control in enumerate( f.controls ):
1237            formcontrols.append( "control %d: %s" % ( i, str( control ) ) )
1238        for i, control in enumerate( f.controls ):
1239            if not hc_prefix in str( control ):
1240                try:
1241                    #check if a repeat element needs to be added
1242                    if control.name is not None:
1243                        if control.name not in kwd and control.name.endswith( '_add' ):
1244                            #control name doesn't exist, could be repeat
1245                            repeat_startswith = control.name[0:-4]
1246                            if repeat_startswith and not [ c_name for c_name in controls.keys() if c_name.startswith( repeat_startswith ) ] and [ c_name for c_name in kwd.keys() if c_name.startswith( repeat_startswith ) ]:
1247                                tc.browser.clicked( f, control )
1248                                tc.submit( control.name )
1249                                return self.submit_form( form_no=form_no, button=button, **kwd )
1250                    # Check for refresh_on_change attribute, submit a change if required
1251                    if hasattr( control, 'attrs' ) and 'refresh_on_change' in control.attrs.keys():
1252                        changed = False
1253                        # For DataToolParameter, control.value is the HDA id, but kwd contains the filename.
1254                        # This loop gets the filename/label for the selected values.
1255                        item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ]
1256                        for value in kwd[ control.name ]:
1257                            if value not in control.value and True not in [ value in item_label for item_label in item_labels ]:
1258                                changed = True
1259                                break
1260                        if changed:
1261                            # Clear Control and set to proper value
1262                            control.clear()
1263                            # kwd[control.name] should be a singlelist
1264                            for elem in kwd[ control.name ]:
1265                                tc.fv( f.name, control.name, str( elem ) )
1266                            # Create a new submit control, allows form to refresh, instead of going to next page
1267                            control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name': 'refresh_grouping'} )
1268                            control.add_to_form( f )
1269                            control.fixup()
1270                            # Submit for refresh
1271                            tc.submit( '___refresh_grouping___' )
1272                            return self.submit_form( form_no=form_no, button=button, **kwd )
1273                except Exception:
1274                    log.exception( "In submit_form, continuing, but caught exception." )
1275                    for formcontrol in formcontrols:
1276                        log.debug( formcontrol )
1277                    continue
1278                controls[ control.name ] = control
1279        # No refresh_on_change attribute found in current form, so process as usual
1280        for control_name, control_value in kwd.items():
1281            if control_name not in controls:
1282                continue  # these cannot be handled safely - cause the test to barf out
1283            if not isinstance( control_value, list ):
1284                control_value = [ control_value ]
1285            control = controls[ control_name ]
1286            control.clear()
1287            if control.is_of_kind( "text" ):
1288                tc.fv( f.name, control.name, ",".join( control_value ) )
1289            elif control.is_of_kind( "list" ):
1290                try:
1291                    if control.is_of_kind( "multilist" ):
1292                        if control.type == "checkbox":
1293                            def is_checked( value ):
1294                                # Copied from form_builder.CheckboxField
1295                                if value == True:
1296                                    return True
1297                                if isinstance( value, list ):
1298                                    value = value[0]
1299                                return isinstance( value, basestring ) and value.lower() in ( "yes", "true", "on" )
1300                            try:
1301                                checkbox = control.get()
1302                                checkbox.selected = is_checked( control_value )
1303                            except Exception, e1:
1304                                print "Attempting to set checkbox selected value threw exception: ", e1
1305                                # if there's more than one checkbox, probably should use the behaviour for
1306                                # ClientForm.ListControl ( see twill code ), but this works for now...
1307                                for elem in control_value:
1308                                    control.get( name=elem ).selected = True
1309                        else:
1310                            for elem in control_value:
1311                                try:
1312                                    # Doubt this case would ever work, but want
1313                                    # to preserve backward compat.
1314                                    control.get( name=elem ).selected = True
1315                                except Exception:
1316                                    # ... anyway this is really what we want to
1317                                    # do, probably even want to try the len(
1318                                    # elem ) > 30 check below.
1319                                    control.get( label=elem ).selected = True
1320                    else:  # control.is_of_kind( "singlelist" )
1321                        for elem in control_value:
1322                            try:
1323                                tc.fv( f.name, control.name, str( elem ) )
1324                            except Exception:
1325                                try:
1326                                    # Galaxy truncates long file names in the dataset_collector in galaxy/tools/parameters/basic.py
1327                                    if len( elem ) > 30:
1328                                        elem_name = '%s..%s' % ( elem[:17], elem[-11:] )
1329                                        tc.fv( f.name, control.name, str( elem_name ) )
1330                                        pass
1331                                    else:
1332                                        raise
1333                                except Exception:
1334                                    raise
1335                            except Exception:
1336                                for formcontrol in formcontrols:
1337                                    log.debug( formcontrol )
1338                                log.exception( "Attempting to set control '%s' to value '%s' (also tried '%s') threw exception.", control.name, elem, elem_name )
1339                                pass
1340                except Exception, exc:
1341                    for formcontrol in formcontrols:
1342                        log.debug( formcontrol )
1343                    errmsg = "Attempting to set field '%s' to value '%s' in form '%s' threw exception: %s\n" % ( control_name, str( control_value ), f.name, str( exc ) )
1344                    errmsg += "control: %s\n" % str( control )
1345                    errmsg += "If the above control is a DataToolparameter whose data type class does not include a sniff() method,\n"
1346                    errmsg += "make sure to include a proper 'ftype' attribute to the tag for the control within the <test> tag set.\n"
1347                    raise AssertionError( errmsg )
1348            else:
1349                # Add conditions for other control types here when necessary.
1350                pass
1351        tc.submit( button )
1352
1353    def refresh_form( self, control_name, value, form_no=0, **kwd ):
1354        """Handles Galaxy's refresh_on_change for forms without ultimately submitting the form"""
1355        # control_name is the name of the form field that requires refresh_on_change, and value is
1356        # the value to which that field is being set.
1357        for i, f in enumerate( self.showforms() ):
1358            if i == form_no:
1359                break
1360        try:
1361            control = f.find_control( name=control_name )
1362        except:
1363            # This assumes we always want the first control of the given name, which may not be ideal...
1364            control = f.find_control( name=control_name, nr=0 )
1365        # Check for refresh_on_change attribute, submit a change if required
1366        if 'refresh_on_change' in control.attrs.keys():
1367            # Clear Control and set to proper value
1368            control.clear()
1369            tc.fv( f.name, control.name, value )
1370            # Create a new submit control, allows form to refresh, instead of going to next page
1371            control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name': 'refresh_grouping'} )
1372            control.add_to_form( f )
1373            control.fixup()
1374            # Submit for refresh
1375            tc.submit( '___refresh_grouping___' )
1376
1377    def visit_page( self, page ):
1378        # tc.go("./%s" % page)
1379        if not page.startswith( "/" ):
1380            page = "/" + page
1381        tc.go( self.url + page )
1382        tc.code( 200 )
1383
1384    def visit_url( self, url ):
1385        tc.go("%s" % url)
1386        tc.code( 200 )
1387
1388    """Functions associated with Galaxy tools"""
1389    def run_tool( self, tool_id, repeat_name=None, **kwd ):
1390        tool_id = tool_id.replace(" ", "+")
1391        """Runs the tool 'tool_id' and passes it the key/values from the *kwd"""
1392        self.visit_url( "%s/tool_runner/index?tool_id=%s" % (self.url, tool_id) )
1393        # Must click somewhere in tool_form, to disambiguate what form
1394        # is being targetted.
1395        tc.browser.clicked( tc.browser.get_form( 'tool_form' ), None )
1396        if repeat_name is not None:
1397            repeat_button = '%s_add' % repeat_name
1398            # Submit the "repeat" form button to add an input)
1399            tc.submit( repeat_button )
1400        tc.find( 'runtool_btn' )
1401        self.submit_form( **kwd )
1402
1403    def run_ucsc_main( self, track_params, output_params ):
1404        """Gets Data From UCSC"""
1405        tool_id = "ucsc_table_direct1"
1406        track_string = urllib.urlencode( track_params )
1407        galaxy_url = urllib.quote_plus( "%s/tool_runner/index?" % self.url )
1408        self.visit_url( "http://genome.ucsc.edu/cgi-bin/hgTables?GALAXY_URL=%s&hgta_compressType=none&tool_id=%s&%s" % ( galaxy_url, tool_id, track_string ) )
1409        tc.fv( "mainForm", "hgta_doTopSubmit", "get output" )
1410        self.submit_form( button="get output" )
1411        tc.fv( 2, "hgta_doGalaxyQuery", "Send query to Galaxy" )
1412        self.submit_form( button="Send query to Galaxy" )
1413
1414    def get_running_datasets( self ):
1415        self.visit_url( '/api/histories' )
1416        history_id = from_json_string( self.last_page() )[0][ 'id' ]
1417        self.visit_url( '/api/histories/%s/contents' % history_id )
1418        jsondata = from_json_string( self.last_page() )
1419        for history_item in jsondata:
1420            self.visit_url( history_item[ 'url' ] )
1421            item_json = from_json_string( self.last_page() )
1422            if item_json[ 'state' ] in [ 'queued', 'running', 'paused' ]:
1423                return True
1424        return False
1425
1426    def wait( self, **kwds ):
1427        """Waits for the tools to finish"""
1428        return self.wait_for(lambda: self.get_running_datasets(), **kwds)
1429
1430    def wait_for( self, func, **kwd ):
1431        sleep_amount = 0.1
1432        slept = 0
1433        walltime_exceeded = 86400
1434        while slept <= walltime_exceeded:
1435            result = func()
1436            if result:
1437                time.sleep( sleep_amount )
1438                slept += sleep_amount
1439                sleep_amount *= 2
1440                if slept + sleep_amount > walltime_exceeded:
1441                    sleep_amount = walltime_exceeded - slept  # don't overshoot maxseconds
1442            else:
1443                break
1444        assert slept < walltime_exceeded, 'Tool run exceeded reasonable walltime of 24 hours, terminating.'
1445
1446    # Dataset Security stuff
1447    # Tests associated with users
1448    def create_new_account_as_admin( self, email='test4@bx.psu.edu', password='testuser',
1449                                     username='regular-user4', redirect='' ):
1450        """Create a new account for another user"""
1451        # HACK: don't use panels because late_javascripts() messes up the twill browser and it
1452        # can't find form fields (and hence user can't be logged in).
1453        self.visit_url( "%s/user/create?cntrller=admin" % self.url )
1454        self.submit_form( 1, 'create_user_button', email=email, redirect=redirect, password=password, confirm=password, username=username )
1455        previously_created = False
1456        username_taken = False
1457        invalid_username = False
1458        try:
1459            self.check_page_for_string( "Created new user account" )
1460        except:
1461            try:
1462                # May have created the account in a previous test run...
1463                self.check_page_for_string( "User with that email already exists" )
1464                previously_created = True
1465            except:
1466                try:
1467                    self.check_page_for_string( 'Public name is taken; please choose another' )
1468                    username_taken = True
1469                except:
1470                    try:
1471                        # Note that we're only checking if the usr name is >< 4 chars here...
1472                        self.check_page_for_string( 'Public name must be at least 4 characters in length' )
1473                        invalid_username = True
1474                    except:
1475                        pass
1476        return previously_created, username_taken, invalid_username
1477
1478    def reset_password_as_admin( self, user_id, password='testreset' ):
1479        """Reset a user password"""
1480        self.home()
1481        self.visit_url( "%s/admin/reset_user_password?id=%s" % ( self.url, user_id ) )
1482        tc.fv( "1", "password", password )
1483        tc.fv( "1", "confirm", password )
1484        tc.submit( "reset_user_password_button" )
1485        self.check_page_for_string( "Passwords reset for 1 user." )
1486        self.home()
1487
1488    def mark_user_deleted( self, user_id, email='' ):
1489        """Mark a user as deleted"""
1490        self.home()
1491        self.visit_url( "%s/admin/users?operation=delete&id=%s" % ( self.url, user_id ) )
1492        check_str = "Deleted 1 users"
1493        self.check_page_for_string( check_str )
1494        self.home()
1495
1496    def undelete_user( self, user_id, email='' ):
1497        """Undelete a user"""
1498        self.home()
1499        self.visit_url( "%s/admin/users?operation=undelete&id=%s" % ( self.url, user_id ) )
1500        check_str = "Undeleted 1 users"
1501        self.check_page_for_string( check_str )
1502        self.home()
1503
1504    def purge_user( self, user_id, email ):
1505        """Purge a user account"""
1506        self.home()
1507        self.visit_url( "%s/admin/users?operation=purge&id=%s" % ( self.url, user_id ) )
1508        check_str = "Purged 1 users"
1509        self.check_page_for_string( check_str )
1510        self.home()
1511
1512    def manage_roles_and_groups_for_user( self, user_id, in_role_ids=[], out_role_ids=[],
1513                                          in_group_ids=[], out_group_ids=[], strings_displayed=[] ):
1514        self.home()
1515        url = "%s/admin/manage_roles_and_groups_for_user?id=%s" % ( self.url, user_id )
1516        if in_role_ids:
1517            url += "&in_roles=%s" % ','.join( in_role_ids )
1518        if out_role_ids:
1519            url += "&out_roles=%s" % ','.join( out_role_ids )
1520        if in_group_ids:
1521            url += "&in_groups=%s" % ','.join( in_group_ids )
1522        if out_group_ids:
1523            url += "&out_groups=%s" % ','.join( out_group_ids )
1524        if in_role_ids or out_role_ids or in_group_ids or out_group_ids:
1525            url += "&user_roles_groups_edit_button=Save"
1526        self.visit_url( url )
1527        for check_str in strings_displayed:
1528            self.check_page_for_string( check_str )
1529        self.home()
1530    # Tests associated with roles
1531
1532    def browse_roles( self, strings_displayed=[] ):
1533        self.visit_url( '%s/admin/roles' % self.url )
1534        for check_str in strings_displayed:
1535            self.check_page_for_string( check_str )
1536
1537    def create_role( self,
1538                     name='Role One',
1539                     description="This is Role One",
1540                     in_user_ids=[],
1541                     in_group_ids=[],
1542                     create_group_for_role='',
1543                     private_role='',
1544                     strings_displayed=[] ):
1545        """Create a new role"""
1546        url = "%s/admin/roles?operation=create&create_role_button=Save&name=%s&description=%s" % \
1547            ( self.url, name.replace( ' ', '+' ), description.replace( ' ', '+' ) )
1548        if in_user_ids:
1549            url += "&in_users=%s" % ','.join( in_user_ids )
1550        if in_group_ids:
1551            url += "&in_groups=%s" % ','.join( in_group_ids )
1552        if create_group_for_role == 'yes':
1553            url += '&create_group_for_role=yes&create_group_for_role=yes'
1554        self.home()
1555        self.visit_url( url )
1556        for check_str in strings_displayed:
1557            self.check_page_for_string( check_str )
1558        if private_role:
1559            # Make sure no private roles are displayed
1560            try:
1561                self.check_page_for_string( private_role )
1562                errmsg = 'Private role %s displayed on Roles page' % private_role
1563                raise AssertionError( errmsg )
1564            except AssertionError:
1565                # Reaching here is the behavior we want since no private roles should be displayed
1566                pass
1567        self.home()
1568        self.visit_url( "%s/admin/roles" % self.url )
1569        self.check_page_for_string( name )
1570        self.home()
1571
1572    def rename_role( self, role_id, name='Role One Renamed', description='This is Role One Re-described' ):
1573        """Rename a role"""
1574        self.home()
1575        self.visit_url( "%s/admin/roles?operation=rename&id=%s" % ( self.url, role_id ) )
1576        self.check_page_for_string( 'Change role name and description' )
1577        tc.fv( "1", "name", name )
1578        tc.fv( "1", "description", description )
1579        tc.submit( "rename_role_button" )
1580        self.home()
1581
1582    def mark_role_deleted( self, role_id, role_name ):
1583        """Mark a role as deleted"""
1584        self.home()
1585        self.visit_url( "%s/admin/roles?operation=delete&id=%s" % ( self.url, role_id ) )
1586        check_str = "Deleted 1 roles:  %s" % role_name
1587        self.check_page_for_string( check_str )
1588        self.home()
1589
1590    def undelete_role( self, role_id, role_name ):
1591        """Undelete an existing role"""
1592        self.home()
1593        self.visit_url( "%s/admin/roles?operation=undelete&id=%s" % ( self.url, role_id ) )
1594        check_str = "Undeleted 1 roles:  %s" % role_name
1595        self.check_page_for_string( check_str )
1596        self.home()
1597
1598    def purge_role( self, role_id, role_name ):
1599        """Purge an existing role"""
1600        self.home()
1601        self.visit_url( "%s/admin/roles?operation=purge&id=%s" % ( self.url, role_id ) )
1602        check_str = "Purged 1 roles:  %s" % role_name
1603        self.check_page_for_string( check_str )
1604        self.home()
1605
1606    def associate_users_and_groups_with_role( self, role_id, role_name, user_ids=[], group_ids=[] ):
1607        self.home()
1608        url = "%s/admin/role?id=%s&role_members_edit_button=Save" % ( self.url, role_id )
1609        if user_ids:
1610            url += "&in_users=%s" % ','.join( user_ids )
1611        if group_ids:
1612            url += "&in_groups=%s" % ','.join( group_ids )
1613        self.visit_url( url )
1614        check_str = "Role '%s' has been updated with %d associated users and %d associated groups" % ( role_name, len( user_ids ), len( group_ids ) )
1615        self.check_page_for_string( check_str )
1616        self.home()
1617
1618    # Tests associated with groups
1619    def create_group( self, name='Group One', in_user_ids=[], in_role_ids=[], create_role_for_group='', strings_displayed=[] ):
1620        """Create a new group"""
1621        url = "%s/admin/groups?operation=create&create_group_button=Save&name=%s" % ( self.url, name.replace( ' ', '+' ) )
1622        if in_user_ids:
1623            url += "&in_users=%s" % ','.join( in_user_ids )
1624        if in_role_ids:
1625            url += "&in_roles=%s" % ','.join( in_role_ids )
1626        if create_role_for_group == 'yes':
1627            url += '&create_role_for_group=yes&create_role_for_group=yes'
1628        self.home()
1629        self.visit_url( url )
1630        for check_str in strings_displayed:
1631            self.check_page_for_string( check_str )
1632        self.home()
1633        self.visit_url( "%s/admin/groups" % self.url )
1634        self.check_page_for_string( name )
1635        self.home()
1636
1637    def browse_groups( self, strings_displayed=[] ):
1638        self.visit_url( '%s/admin/groups' % self.url )
1639        for check_str in strings_displayed:
1640            self.check_page_for_string( check_str )
1641
1642    def rename_group( self, group_id, name='Group One Renamed' ):
1643        """Rename a group"""
1644        self.home()
1645        self.visit_url( "%s/admin/groups?operation=rename&id=%s" % ( self.url, group_id ) )
1646        self.check_page_for_string( 'Change group name' )
1647        tc.fv( "1", "name", name )
1648        tc.submit( "rename_group_button" )
1649        self.home()
1650
1651    def associate_users_and_roles_with_group( self, group_id, group_name, user_ids=[], role_ids=[] ):
1652        self.home()
1653        url = "%s/admin/manage_users_and_roles_for_group?id=%s&group_roles_users_edit_button=Save" % ( self.url, group_id )
1654        if user_ids:
1655            url += "&in_users=%s" % ','.join( user_ids )
1656        if role_ids:
1657            url += "&in_roles=%s" % ','.join( role_ids )
1658        self.visit_url( url )
1659        check_str = "Group '%s' has been updated with %d associated roles and %d associated users" % ( group_name, len( role_ids ), len( user_ids ) )
1660        self.check_page_for_string( check_str )
1661        self.home()
1662
1663    def mark_group_deleted( self, group_id, group_name ):
1664        """Mark a group as deleted"""
1665        self.home()
1666        self.visit_url( "%s/admin/groups?operation=delete&id=%s" % ( self.url, group_id ) )
1667        check_str = "Deleted 1 groups:  %s" % group_name
1668        self.check_page_for_string( check_str )
1669        self.home()
1670
1671    def undelete_group( self, group_id, group_name ):
1672        """Undelete an existing group"""
1673        self.home()
1674        self.visit_url( "%s/admin/groups?operation=undelete&id=%s" % ( self.url, group_id ) )
1675        check_str = "Undeleted 1 groups:  %s" % group_name
1676        self.check_page_for_string( check_str )
1677        self.home()
1678
1679    def purge_group( self, group_id, group_name ):
1680        """Purge an existing group"""
1681        self.home()
1682        self.visit_url( "%s/admin/groups?operation=purge&id=%s" % ( self.url, group_id ) )
1683        check_str = "Purged 1 groups:  %s" % group_name
1684        self.check_page_for_string( check_str )
1685        self.home()
1686
1687    # Form stuff
1688    def create_form( self, name, description, form_type, field_type='TextField', form_layout_name='',
1689                     num_fields=1, num_options=0, field_name='1_field_name', strings_displayed=[],
1690                     strings_displayed_after_submit=[] ):
1691        """Create a new form definition."""
1692        self.visit_url( "%s/forms/create_form_definition" % self.url )
1693        for check_str in strings_displayed:
1694            self.check_page_for_string( check_str )
1695        tc.fv( "1", "name", name )
1696        tc.fv( "1", "description", description )
1697        tc.fv( "1", "form_type_select_field", form_type )
1698        tc.submit( "create_form_button" )
1699        if form_type == "Sequencing Sample Form":
1700            tc.submit( "add_layout_grid" )
1701            tc.fv( "1", "grid_layout0", form_layout_name )
1702        # if not adding any fields at this time, remove the default empty field
1703        if num_fields == 0:
1704            tc.submit( "remove_button" )
1705        # Add fields to the new form definition
1706        for index1 in range( num_fields ):
1707            field_label = 'field_label_%i' % index1
1708            field_contents = field_type
1709            field_help_name = 'field_helptext_%i' % index1
1710            field_help_contents = 'Field %i help' % index1
1711            field_default = 'field_default_0'
1712            field_default_contents = '%s default contents' % form_type
1713            tc.fv( "1", field_label, field_contents )
1714            tc.fv( "1", field_help_name, field_help_contents )
1715            if field_type == 'SelectField':
1716                # SelectField field_type requires a refresh_on_change
1717                self.refresh_form( 'field_type_0', field_type )
1718                # Add options so our select list is functional
1719                if num_options == 0:
1720                    # Default to 2 options
1721                    num_options = 2
1722                for index2 in range( 1, num_options + 1 ):
1723                    tc.submit( "addoption_0" )
1724                # Add contents to the new options fields
1725                for index2 in range( num_options ):
1726                    option_field_name = 'field_0_option_%i' % index2
1727                    option_field_value = 'Option%i' % index2
1728                    tc.fv( "1", option_field_name, option_field_value )
1729            else:
1730                tc.fv( "1", "field_type_0", field_type )
1731            tc.fv( "1", 'field_name_0', field_name )
1732            tc.fv( "1", field_default, field_default_contents )
1733        # All done... now save
1734        tc.submit( "save_changes_button" )
1735        for check_str in strings_displayed_after_submit:
1736            self.check_page_for_string( check_str )
1737        self.home()
1738
1739    def edit_form( self, id, form_type='', new_form_name='', new_form_desc='', field_dicts=[], field_index=0,
1740                   strings_displayed=[], strings_not_displayed=[], strings_displayed_after_submit=[] ):
1741        """Edit form details; name and description"""
1742        self.home()
1743        self.visit_url( "%s/forms/edit_form_definition?id=%s" % ( self.url, id ) )
1744        for check_str in strings_displayed:
1745            self.check_page_for_string( check_str )
1746        if new_form_name:
1747            tc.fv( "1", "name", new_form_name )
1748        if new_form_desc:
1749            tc.fv( "1", "description", new_form_desc )
1750        for i, field_dict in enumerate( field_dicts ):
1751            index = i + field_index
1752            tc.submit( "add_field_button" )
1753            field_label = "field_label_%i" % index
1754            field_label_value = field_dict[ 'label' ]
1755            field_help = "field_helptext_%i" % index
1756            field_help_value = field_dict[ 'desc' ]
1757            field_type = "field_type_%i" % index
1758            field_type_value = field_dict[ 'type' ]
1759            field_required = "field_required_%i" % index
1760            field_required_value = field_dict[ 'required' ]
1761            field_name = "field_name_%i" % index
1762            field_name_value = field_dict.get( 'name', '%i_field_name' % index )
1763            tc.fv( "1", field_label, field_label_value )
1764            tc.fv( "1", field_help, field_help_value )
1765            tc.fv( "1", field_required, field_required_value )
1766            tc.fv( "1", field_name, field_name_value )
1767            if field_type_value.lower() == 'selectfield':
1768                # SelectFields require a refresh_on_change
1769                self.refresh_form( field_type, field_type_value )
1770                for option_index, option in enumerate( field_dict[ 'selectlist' ] ):
1771                    tc.submit( "addoption_%i" % index )
1772                    tc.fv( "1", "field_%i_option_%i" % ( index, option_index ), option )
1773            else:
1774                tc.fv( "1", field_type, field_type_value )
1775        tc.submit( "save_changes_button" )
1776        for check_str in strings_displayed_after_submit:
1777            self.check_page_for_string( check_str )
1778        self.home()
1779    def view_form( self, id, form_type='', form_name='', form_desc='', form_layout_name='', field_dicts=[] ):
1780        '''View form details'''
1781        self.home()
1782        self.visit_url( "%s/forms/view_latest_form_definition?id=%s" % ( self.url, id ) )
1783        #self.check_page_for_string( form_type )
1784        self.check_page_for_string( form_name )
1785        #self.check_page_for_string( form_desc )
1786        self.check_page_for_string( form_layout_name )
1787        for i, field_dict in enumerate( field_dicts ):
1788            self.check_page_for_string( field_dict[ 'label' ] )
1789            self.check_page_for_string( field_dict[ 'desc' ] )
1790            self.check_page_for_string( field_dict[ 'type' ] )
1791            if field_dict[ 'type' ].lower() == 'selectfield':
1792                for option_index, option in enumerate( field_dict[ 'selectlist' ] ):
1793                    self.check_page_for_string( option )
1794        self.home()
1795    def mark_form_deleted( self, form_id ):
1796        """Mark a form_definition as deleted"""
1797        self.home()
1798        url = "%s/forms/delete_form_definition?id=%s" % ( self.url, form_id )
1799        self.visit_url( url )
1800        check_str = "1 forms have been deleted."
1801        self.check_page_for_string( check_str )
1802        self.home()
1803    
1804    # External services stuff
1805    def reload_external_service( self, external_service_type_id, strings_displayed=[], strings_displayed_after_submit=[] ):
1806        self.visit_url( '%s/external_service/reload_external_service_types' % self.url )
1807        for check_str in strings_displayed:
1808            self.check_page_for_string( check_str )
1809        tc.fv( "1", "external_service_type_id", external_service_type_id )
1810        tc.submit( "reload_external_service_type_button" )
1811        for check_str in strings_displayed_after_submit:
1812            self.check_page_for_string( check_str )
1813    def create_external_service( self, name, description, version, external_service_type_id, field_values={}, strings_displayed=[], strings_displayed_after_submit=[] ):
1814        self.visit_url( '%s/external_service/create_external_service' % self.url )
1815        for check_str in strings_displayed:
1816            self.check_page_for_string( check_str )
1817        tc.fv( "1", "name", name )
1818        tc.fv( "1", "description", description )
1819        tc.fv( "1", "version", version )
1820        self.refresh_form( "external_service_type_id", external_service_type_id )
1821        for field, value in field_values.items():
1822            tc.fv( "1", field, value )
1823        tc.submit( "create_external_service_button" )
1824        for check_str in strings_displayed_after_submit:
1825            self.check_page_for_string( check_str )
1826    def view_external_service( self, external_service_id, strings_displayed=[] ):
1827        self.visit_url( '%s/external_service/view_external_service?id=%s' % ( self.url, external_service_id ) )
1828        for check_str in strings_displayed:
1829            self.check_page_for_string( check_str )
1830    def edit_external_service( self, external_service_id, field_values={}, strings_displayed=[], strings_displayed_after_submit=[] ):
1831        self.visit_url( '%s/external_service/edit_external_service?id=%s' % ( self.url, external_service_id ) )
1832        for check_str in strings_displayed:
1833            self.check_page_for_string( check_str )
1834        for field, value in field_values.items():
1835            tc.fv( "1", field, value )
1836        tc.submit( "edit_external_service_button" )
1837        for check_str in strings_displayed_after_submit:
1838            self.check_page_for_string( check_str )
1839    # Sample tracking stuff
1840    def check_request_grid( self, cntrller, state, deleted=False, strings_displayed=[] ):
1841        self.visit_url( '%s/%s/browse_requests?sort=create_time&f-state=%s&f-deleted=%s' % \
1842                        ( self.url, cntrller, state.replace( ' ', '+' ), str( deleted ) ) )
1843        for check_str in strings_displayed:
1844            self.check_page_for_string( check_str )
1845    def create_request_type( self, name, desc, request_form_id, sample_form_id, states, strings_displayed=[], strings_displayed_after_submit=[] ):
1846        self.home()
1847        self.visit_url( "%s/request_type/create_request_type" % self.url )
1848        for check_str in strings_displayed:
1849            self.check_page_for_string( check_str )
1850        tc.fv( "1", "name", name )
1851        tc.fv( "1", "desc", desc )
1852        tc.fv( "1", "request_form_id", request_form_id )
1853        tc.fv( "1", "sample_form_id", sample_form_id )
1854        for index, state in enumerate(states):
1855            tc.fv("1", "state_name_%i" % index, state[0])
1856            tc.fv("1", "state_desc_%i" % index, state[1])
1857            tc.submit( "add_state_button" )
1858        tc.submit( "create_request_type_button" )
1859        for check_str in strings_displayed_after_submit:
1860            self.check_page_for_string( check_str )
1861    def request_type_permissions( self, request_type_id, request_type_name, role_ids_str, permissions_in, permissions_out ):
1862        # role_ids_str must be a comma-separated string of role ids
1863        url = "request_type/request_type_permissions?id=%s&update_roles_button=Save" % ( request_type_id )
1864        for po in permissions_out:
1865            key = '%s_out' % po
1866            url = "%s&%s=%s" % ( url, key, role_ids_str )
1867        for pi in permissions_in:
1868            key = '%s_in' % pi
1869            url = "%s&%s=%s" % ( url, key, role_ids_str )
1870        self.home()
1871        self.visit_url( "%s/%s" % ( self.url, url ) )
1872        check_str = "Permissions updated for request type '%s'" % request_type_name
1873        self.check_page_for_string( check_str )
1874        self.home()
1875    def view_request_type( self, request_type_id, request_type_name, sample_states, strings_displayed=[] ):
1876        '''View request_type details'''
1877        self.home()
1878        self.visit_url( "%s/request_type/view_request_type?id=%s" % ( self.url, request_type_id ) )
1879        self.check_page_for_string( '"%s" request type' % request_type_name )
1880        for name, desc in sample_states:
1881            self.check_page_for_string( name )
1882            self.check_page_for_string( desc )
1883        for check_str in strings_displayed:
1884            self.check_page_for_string( check_str )
1885    def create_request( self, cntrller, request_type_id, name, desc, field_value_tuples, other_users_id='',
1886                        strings_displayed=[], strings_displayed_after_submit=[] ):
1887        self.visit_url( "%s/requests_common/create_request?cntrller=%s" % ( self.url, cntrller ) )
1888        # The request_type SelectList requires a refresh_on_change
1889        self.refresh_form( 'request_type_id', request_type_id )
1890        if cntrller == 'requests_admin' and other_users_id:
1891            # The admin is creating a request on behalf of another user
1892            # The user_id SelectField requires a refresh_on_change so that the selected
1893            # user's addresses will be populated in the AddressField widget
1894            self.refresh_form( "user_id", other_users_id )
1895        for check_str in strings_displayed:
1896            self.check_page_for_string( check_str )
1897        tc.fv( "1", "name", name )
1898        tc.fv( "1", "desc", desc )
1899        for index, field_value_tuple in enumerate( field_value_tuples ):
1900            field_index = index + 1
1901            field_name, field_value, refresh_on_change = field_value_tuple
1902            if refresh_on_change:
1903                # Only the AddressField type has a refresh on change setup on selecting an option
1904                address_option = field_value[0]
1905                address_value = field_value[1]
1906                self.refresh_form( field_name, address_option )
1907                if address_option == 'new':
1908                    # handle new address
1909                    self.check_page_for_string( 'Short address description' )
1910                    for address_field, value in address_value.items():
1911                        tc.fv( "1", field_name+'_'+address_field, value )
1912                else:
1913                    # existing address
1914                    tc.fv( "1", field_name, address_value )
1915            else:
1916                tc.fv( "1", field_name, field_value )
1917        tc.submit( "create_request_button" )
1918        for check_str in strings_displayed_after_submit:
1919            self.check_page_for_string( check_str )
1920        self.home()
1921    def view_request( self, cntrller, request_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
1922        self.visit_url( "%s/%s/browse_requests?operation=view_request&id=%s" % ( self.url, cntrller, request_id ) )
1923        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
1924    def view_request_history( self, cntrller, request_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
1925        self.visit_url( "%s/requests_common/view_request_history?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
1926        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
1927    def view_sample_history( self, cntrller, sample_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
1928        self.visit_url( "%s/requests_common/view_sample_history?cntrller=%s&sample_id=%s" % ( self.url, cntrller, sample_id ) )
1929        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
1930    def view_sample_dataset( self, sample_dataset_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
1931        self.visit_url( "%s/requests_admin/manage_datasets?operation=view&id=%s" % ( self.url, sample_dataset_id ) )
1932        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
1933    def edit_basic_request_info( self, cntrller, request_id, name, new_name='', new_desc='', new_fields=[],
1934                                 strings_displayed=[], strings_displayed_after_submit=[] ):
1935        self.visit_url( "%s/requests_common/edit_basic_request_info?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
1936        for check_str in strings_displayed:
1937            self.check_page_for_string( check_str )
1938        if new_name:
1939            tc.fv( "1", "name", new_name )
1940        if new_desc:
1941            tc.fv( "1", "desc", new_desc )
1942        for index, ( field_name, field_value ) in enumerate( new_fields ):
1943            field_name_index = index + 1
1944            tc.fv( "1", field_name, field_value )
1945        tc.submit( "edit_basic_request_info_button" )
1946        for check_str in strings_displayed_after_submit:
1947            self.check_page_for_string( check_str )
1948    def edit_request_email_settings( self, cntrller, request_id, check_request_owner=True, additional_emails='', 
1949                                     check_sample_states=[], strings_displayed=[], strings_displayed_after_submit=[] ):
1950        self.visit_url( "%s/requests_common/edit_basic_request_info?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
1951        for check_str in strings_displayed:
1952            self.check_page_for_string( check_str )
1953        tc.fv( "2", "email_address", check_request_owner )
1954        tc.fv( "2", "additional_email_addresses", additional_emails )
1955        for state_name, state_id, is_checked in check_sample_states:
1956            tc.fv( "2", "sample_state_%i" % state_id, is_checked )
1957        tc.submit( "edit_email_settings_button" )
1958        for check_str in strings_displayed_after_submit:
1959            self.check_page_for_string( check_str )
1960    def add_samples( self, cntrller, request_id, sample_value_tuples, folder_options=[], strings_displayed=[], strings_displayed_after_submit=[] ):
1961        url = "%s/requests_common/add_sample?cntrller=%s&request_id=%s&add_sample_button=Add+sample" % ( self.url, cntrller, request_id )
1962        self.visit_url( url )
1963        for check_str in strings_displayed:
1964            self.check_page_for_string( check_str )
1965        for sample_index, ( sample_name, target_library_info, sample_field_values ) in enumerate( sample_value_tuples ):
1966            tc.fv( "1", "sample_%i_name" % sample_index, sample_name )
1967            tc.fv( "1", "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
1968            self.refresh_form( "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
1969            # check if the folder selectfield has been correctly populated
1970            for check_str in folder_options:
1971                self.check_page_for_string( check_str )
1972            tc.fv( "1", "sample_%i_folder_id" % sample_index, target_library_info[ 'folder' ] )
1973            for field_index, field_value in enumerate( sample_field_values ):
1974                tc.fv( "1", "sample_%i_field_%i" % ( sample_index, field_index ), field_value )
1975            # Do not click on Add sample button when all the sample have been added 
1976            if sample_index < len( sample_value_tuples ) - 1:
1977                tc.submit( "add_sample_button" )
1978        # select the correct form before submitting it
1979        tc.fv( "1", "copy_sample_index", "-1" )
1980        tc.submit( "save_samples_button" )
1981        for check_str in strings_displayed_after_submit:
1982            self.check_page_for_string( check_str )
1983    def edit_samples( self, cntrller, request_id, sample_value_tuples, strings_displayed=[], strings_displayed_after_submit=[] ):
1984        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
1985        self.visit_url( url )
1986        for check_str in strings_displayed:
1987            self.check_page_for_string( check_str )
1988        for sample_index, ( sample_name, target_library_info, sample_field_values ) in enumerate( sample_value_tuples ):
1989            tc.fv( "1", "sample_%i_name" % sample_index, sample_name )
1990            tc.fv( "1", "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
1991            self.refresh_form( "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
1992            tc.fv( "1", "sample_%i_folder_id" % sample_index, target_library_info[ 'folder' ] )
1993            for field_index, field_value in enumerate( sample_field_values ):
1994                tc.fv( "1", "sample_%i_field_%i" % ( sample_index, field_index ), field_value )
1995        tc.submit( "save_samples_button" )
1996        for check_str in strings_displayed_after_submit:
1997            self.check_page_for_string( check_str )
1998    def add_bar_codes( self, cntrller, request_id, bar_codes, strings_displayed=[], strings_displayed_after_submit=[] ):
1999        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
2000        self.visit_url( url )
2001        for check_str in strings_displayed:
2002            self.check_page_for_string( check_str )
2003        for sample_index, bar_code in enumerate( bar_codes ):
2004            tc.fv( "1", "sample_%i_bar_code" % sample_index, bar_code )
2005        tc.submit( "save_samples_button" )
2006        for check_str in strings_displayed_after_submit:
2007            self.check_page_for_string( check_str )
2008    def submit_request( self, cntrller, request_id, request_name, strings_displayed_after_submit=[] ):
2009        self.visit_url( "%s/requests_common/submit_request?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
2010        for check_str in strings_displayed_after_submit:
2011            self.check_page_for_string( check_str )
2012    def reject_request( self, request_id, request_name, comment, strings_displayed=[], strings_displayed_after_submit=[] ):
2013        self.visit_url( "%s/requests_admin/reject_request?id=%s" % ( self.url, request_id ) )
2014        for check_str in strings_displayed:
2015            self.check_page_for_string( check_str )
2016        tc.fv( "1", "comment", comment )
2017        tc.submit( "reject_button" )
2018        for check_str in strings_displayed_after_submit:
2019            self.check_page_for_string( check_str )
2020    def change_sample_state( self, request_id, sample_ids, new_sample_state_id, comment='', strings_displayed=[], strings_displayed_after_submit=[] ):
2021        url = "%s/requests_common/edit_samples?cntrller=requests_admin&id=%s" % ( self.url, request_id )
2022        self.visit_url( url )
2023        for check_str in strings_displayed:
2024            self.check_page_for_string( check_str )
2025        for sample_id in sample_ids:
2026            tc.fv( "1", "select_sample_%i" % sample_id, True )
2027        tc.fv( "1", "sample_operation", 'Change state' )
2028        # refresh on change to show the sample states selectfield
2029        self.refresh_form( "sample_operation", 'Change state' )
2030        self.check_page_for_string( "Change current state" )
2031        tc.fv( "1", "sample_state_id", new_sample_state_id )
2032        tc.fv( "1", "sample_event_comment", comment )
2033        tc.submit( "save_samples_button" )
2034        for check_str in strings_displayed_after_submit:
2035            self.check_page_for_string( check_str )
2036    def change_sample_target_data_library( self, cntrller, request_id, sample_ids, new_library_id, new_folder_id, folder_options=[], comment='', strings_displayed=[], strings_displayed_after_submit=[] ):
2037        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
2038        self.visit_url( url )
2039        for check_str in strings_displayed:
2040            self.check_page_for_string( check_str )
2041        for sample_id in sample_ids:
2042            tc.fv( "1", "select_sample_%i" % sample_id, True )
2043        tc.fv( "1", "sample_operation", 'Select data library and folder' )
2044        # refresh on change to show the data libraries selectfield
2045        self.refresh_form( "sample_operation", 'Select data library and folder' )
2046        self.check_page_for_string( "Select data library:" )
2047        tc.fv( "1", "sample_operation_library_id", new_library_id )
2048        # refresh on change to show the selectfield with the list of 
2049        # folders in the selected data library above
2050        self.refresh_form( "sample_operation_library_id", new_library_id )
2051        self.check_page_for_string( "Select folder:" )
2052        # check if the folder selectfield has been correctly populated
2053        for check_str in folder_options:
2054            self.check_page_for_string( check_str )
2055        tc.fv( "1", "sample_operation_folder_id", new_folder_id )
2056        tc.submit( "save_samples_button" )
2057        for check_str in strings_displayed_after_submit:
2058            self.check_page_for_string( check_str )
2059    def add_datasets_to_sample( self, request_id, sample_id, external_service_id, sample_datasets, strings_displayed=[], strings_displayed_after_submit=[] ):
2060        # visit the dataset selection page
2061        url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s&external_service_id=%s" % \
2062            ( self.url, sample_id, request_id, external_service_id )
2063        self.visit_url( url )
2064        for check_str in strings_displayed:
2065            self.check_page_for_string( check_str )
2066        # Datasets are associated with the given by the building the appropriate url
2067        # and calling it as the dataset selection UI is a javascript dynatree
2068        url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s" % ( self.url, sample_id, request_id )
2069        url += '&select_datasets_to_transfer_button=Select%20datasets'
2070        url += '&selected_datasets_to_transfer=%s' % ','.join( sample_datasets )
2071        self.visit_url( url )
2072        for check_str in strings_displayed_after_submit:
2073            self.check_page_for_string( check_str )
2074    def rename_sample_datasets( self, sample_id, sample_dataset_ids, new_sample_dataset_names, strings_displayed=[], strings_displayed_after_submit=[] ):
2075        sample_dataset_ids_string = ','.join( sample_dataset_ids )
2076        url = "%s/requests_admin/manage_datasets?operation=rename&id=%s" % ( self.url, sample_dataset_ids_string )
2077        self.visit_url( url )
2078        for check_str in strings_displayed:
2079            self.check_page_for_string( check_str )
2080        for sample_dataset_id, ( prefix, new_name ) in zip( sample_dataset_ids, new_sample_dataset_names ):
2081            tc.fv( "1", 'rename_datasets_for_sample_%s' % sample_dataset_id, prefix )
2082            tc.fv( "1", 'new_name_%s' % sample_dataset_id, new_name )
2083        tc.submit( "rename_datasets_button" )
2084        for check_str in strings_displayed_after_submit:
2085            self.check_page_for_string( check_str )
2086    def delete_sample_datasets( self, sample_id, sample_dataset_ids, strings_displayed=[], strings_displayed_after_submit=[], strings_not_displayed=[] ):
2087        url = '%s/requests_admin/manage_datasets?cntrller=requests_admin&sample_id=%s' % ( self.url, sample_id )
2088        self.visit_url( url )
2089        for check_str in strings_displayed:
2090            self.check_page_for_string( check_str )
2091        # simulate selecting datasets and clicking the delete button on the sample datasets grid
2092        sample_dataset_ids_string = ','.join( sample_dataset_ids )
2093        url = "%s/requests_admin/manage_datasets?operation=delete&id=%s" % ( self.url, sample_dataset_ids_string )
2094        self.visit_url( url )
2095        for check_str in strings_displayed_after_submit:
2096            self.check_page_for_string( check_str )
2097        for check_str in strings_not_displayed:
2098            self.check_string_not_in_page( check_str )
2099    def start_sample_datasets_transfer( self, sample_id, sample_dataset_ids, strings_displayed=[], strings_displayed_after_submit=[], strings_displayed_count=[], strings_not_displayed=[] ):
2100        url = '%s/requests_admin/manage_datasets?cntrller=requests_admin&sample_id=%s' % ( self.url, sample_id )
2101        self.visit_url( url )
2102        for check_str in strings_displayed:
2103            self.check_page_for_string( check_str )
2104        # simulate selecting datasets and clicking the transfer button on the sample datasets grid
2105        sample_dataset_ids_string = ','.join( sample_dataset_ids )
2106        url = "%s/requests_admin/manage_datasets?operation=transfer&id=%s" % ( self.url, sample_dataset_ids_string )
2107        self.visit_url( url )
2108        for check_str in strings_displayed_after_submit:
2109            self.check_page_for_string( check_str )
2110        for check_str in strings_not_displayed:
2111            self.check_string_not_in_page( check_str )
2112        for check_str, count in strings_displayed_count:
2113            self.check_string_count_in_page( check_str, count )
2114    def add_user_address( self, user_id, address_dict ):
2115        self.home()
2116        self.visit_url( "%s/user/new_address?cntrller=user&user_id=%s" % ( self.url, user_id ) )
2117        self.check_page_for_string( 'Add new address' )
2118        for field_name, value in address_dict.items():
2119            tc.fv( "1", field_name, value )
2120        tc.submit( "new_address_button" )
2121        self.check_page_for_string( 'Address (%s) has been added' % address_dict[ 'short_desc' ] )
2122        
2123    # Library stuff
2124    def add_template( self, cntrller, item_type, form_type, form_id, form_name,
2125                      library_id=None, folder_id=None, ldda_id=None, request_type_id=None, sample_id=None ):
2126        """
2127        Add a new template to an item - for library items, the template will ALWAYS BE SET TO INHERITABLE here.  If you want to
2128        dis-inherit your template, call the manage_library_template_inheritance() below immediately after you call this
2129        method in your test code.  Templates added to Requesttype objects are always inherited to samples.
2130        """
2131        self.home()
2132        if item_type == 'library':
2133            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s" % \
2134            ( self.url, cntrller, item_type, form_type, library_id )
2135        elif item_type == 'folder':
2136            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s&folder_id=%s" % \
2137            ( self.url, cntrller, item_type, form_type, library_id, folder_id )
2138        elif item_type == 'ldda':
2139            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s&folder_id=%s&ldda_id=%s" % \
2140            ( self.url, cntrller, item_type, form_type, library_id, folder_id, ldda_id )
2141        self.visit_url( url )
2142        self.check_page_for_string ( "Select a template for the" )
2143        self.refresh_form( "form_id", form_id )
2144        # For some unknown reason, twill barfs if the form number ( 1 ) is used in the following
2145        # rather than the form anme ( select_template ), so we have to use the form name.
2146        tc.fv( "select_template", "inheritable", '1' )
2147        tc.submit( "add_template_button" )
2148        self.check_page_for_string = 'A template based on the form "%s" has been added to this' % form_name
2149        self.home()
2150    def manage_library_template_inheritance( self, cntrller, item_type, library_id, folder_id=None, ldda_id=None, inheritable=True ):
2151        # If inheritable is True, the item is currently inheritable.
2152        self.home()
2153        if item_type == 'library':
2154            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s" % \
2155            ( self.url, cntrller, item_type, library_id )
2156        elif item_type == 'folder':
2157            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s" % \
2158            ( self.url, cntrller, item_type, library_id, folder_id )
2159        elif item_type == 'ldda':
2160            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s&ldda_id=%s" % \
2161            ( self.url, cntrller, item_type, library_id, folder_id, ldda_id )
2162        self.visit_url( url )
2163        if inheritable:
2164            self.check_page_for_string = 'will no longer be inherited to contained folders and datasets'
2165        else:
2166            self.check_page_for_string = 'will now be inherited to contained folders and datasets'
2167        self.home()
2168    def browse_libraries_admin( self, deleted=False, strings_displayed=[], strings_not_displayed=[] ):
2169        self.visit_url( '%s/library_admin/browse_libraries?sort=name&f-description=All&f-name=All&f-deleted=%s' % ( self.url, str( deleted ) ) )
2170        for check_str in strings_displayed:
2171            self.check_page_for_string( check_str )
2172        for check_str in strings_not_displayed:
2173            try:
2174                self.check_page_for_string( check_str )
2175                raise AssertionError( "String (%s) incorrectly displayed when browing library." % check_str )
2176            except:
2177                pass
2178    def browse_libraries_regular_user( self, strings_displayed=[], strings_not_displayed=[] ):
2179        self.visit_url( '%s/library/browse_libraries' % self.url )
2180        for check_str in strings_displayed:
2181            self.check_page_for_string( check_str )
2182        for check_str in strings_not_displayed:
2183            try:
2184                self.check_page_for_string( check_str )
2185                raise AssertionError( "String (%s) incorrectly displayed when browing library." % check_str )
2186            except:
2187                pass
2188    def browse_library( self, cntrller, library_id, show_deleted=False, strings_displayed=[], strings_not_displayed=[] ):
2189        self.visit_url( '%s/library_common/browse_library?cntrller=%s&id=%s&show_deleted=%s' % ( self.url, cntrller, library_id, str( show_deleted ) ) )
2190        for check_str in strings_displayed:
2191            self.check_page_for_string( check_str )
2192        for check_str in strings_not_displayed:
2193            try:
2194                self.check_page_for_string( check_str )
2195                raise AssertionError( "String (%s) incorrectly displayed when browing library." % check_str )
2196            except:
2197                pass
2198    def create_library( self, name='Library One', description='This is Library One', synopsis='Synopsis for Library One' ):
2199        """Create a new library"""
2200        self.visit_url( "%s/library_admin/create_library" % self.url )
2201        self.check_page_for_string( 'Create a new data library' )
2202        tc.fv( "1", "name", name )
2203        tc.fv( "1", "description", description )
2204        tc.fv( "1", "synopsis", synopsis )
2205        tc.submit( "create_library_button" )
2206        check_str = "The new library named '%s' has been created" % name
2207        self.check_page_for_string( check_str )
2208        self.home()
2209    def edit_template( self, cntrller, item_type, form_type, library_id, field_type, field_label_1, field_helptext_1, field_default_1,
2210                       folder_id='', ldda_id='', action='add_field'  ):
2211        """Edit the form fields defining a library template"""
2212        self.visit_url( "%s/library_common/edit_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s" % \
2213                        ( self.url, cntrller, item_type, form_type, library_id ) )
2214        self.check_page_for_string( "Edit form definition" )
2215        if action == 'add_field':
2216            tc.submit( "add_field_button" )
2217            tc.fv( "edit_form", "field_label_1", field_label_1 )
2218            tc.fv( "edit_form", "field_helptext_1", field_helptext_1 )
2219            if field_type == 'SelectField':
2220                # Performs a refresh_on_change in this case
2221                self.refresh_form( "field_type_1", field_type )
2222            else:
2223                tc.fv( "edit_form", "field_type_1", field_type )
2224            tc.fv( "edit_form", "field_default_1", field_default_1 )
2225        tc.submit( 'save_changes_button' )
2226        self.check_page_for_string( "The template for this data library has been updated with your changes." )
2227    def library_info( self, cntrller, library_id, library_name='', new_name='', new_description='', new_synopsis='', 
2228                      template_fields=[], strings_displayed=[] ):
2229        """Edit information about a library, optionally using an existing template with up to 2 elements"""
2230        self.visit_url( "%s/library_common/library_info?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
2231        for check_str in strings_displayed:
2232            self.check_page_for_string( check_str )
2233        if new_name and new_description and new_synopsis:
2234            tc.fv( '1', 'name', new_name )
2235            tc.fv( '1', 'description', new_description )
2236            tc.fv( '1', 'synopsis', new_synopsis )
2237            tc.submit( 'library_info_button' )
2238            self.check_page_for_string( "Information updated for library" )
2239        if template_fields:
2240            for field_name, field_value in template_fields:
2241                # The 2nd form on the page contains the template, and the form is named edit_info.
2242                # Set the template field value
2243                tc.fv( "edit_info", field_name, field_value )
2244            tc.submit( 'edit_info_button' )
2245        self.home()
2246    def library_permissions( self, library_id, library_name, role_ids_str, permissions_in, permissions_out, cntrller='library_admin' ):
2247        # role_ids_str must be a comma-separated string of role ids
2248        url = "library_common/library_permissions?id=%s&cntrller=%s&update_roles_button=Save" % ( library_id, cntrller )
2249        for po in permissions_out:
2250            key = '%s_out' % po
2251            url = "%s&%s=%s" % ( url, key, role_ids_str )
2252        for pi in permissions_in:
2253            key = '%s_in' % pi
2254            url = "%s&%s=%s" % ( url, key, role_ids_str )
2255        self.home()
2256        self.visit_url( "%s/%s" % ( self.url, url ) )
2257        check_str = "Permissions updated for library '%s'." % library_name
2258        self.check_page_for_string( check_str )
2259        self.home()
2260    def make_library_item_public( self, library_id, id, cntrller='library_admin', item_type='library',
2261                                  contents=False, library_name='', folder_name='', ldda_name='' ):
2262        url = "%s/library_common/make_library_item_public?cntrller=%s&library_id=%s&item_type=%s&id=%s&contents=%s" % \
2263            ( self.url, cntrller, library_id, item_type, id, str( contents ) )
2264        self.visit_url( url )
2265        if item_type == 'library':
2266            if contents:
2267                check_str = "The data library (%s) and all it's contents have been made publicly accessible." % library_name
2268            else:
2269                check_str = "The data library (%s) has been made publicly accessible, but access to it's contents has been left unchanged." % library_name
2270        elif item_type == 'folder':
2271            check_str = "All of the contents of folder (%s) have been made publicly accessible." % folder_name
2272        elif item_type == 'ldda':
2273            check_str = "The libary dataset (%s) has been made publicly accessible." % ldda_name
2274        self.check_page_for_string( check_str )
2275
2276    # Library folder stuff
2277    def add_folder( self, cntrller, library_id, folder_id, name='Folder One', description='This is Folder One' ):
2278        """Create a new folder"""
2279        url = "%s/library_common/create_folder?cntrller=%s&library_id=%s&parent_id=%s" % ( self.url, cntrller, library_id, folder_id )
2280        self.visit_url( url )
2281        self.check_page_for_string( 'Create a new folder' )
2282        tc.fv( "1", "name", name )
2283        tc.fv( "1", "description", description )
2284        tc.submit( "new_folder_button" )
2285        check_str = "The new folder named '%s' has been added to the data library." % name
2286        self.check_page_for_string( check_str )
2287        self.home()
2288    def folder_info( self, cntrller, folder_id, library_id, name='', new_name='', description='', template_refresh_field_name='1_field_name',
2289                     template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[],
2290                     strings_displayed_after_submit=[], strings_not_displayed_after_submit=[] ):
2291        """Add information to a library using an existing template with 2 elements"""
2292        self.visit_url( "%s/library_common/folder_info?cntrller=%s&id=%s&library_id=%s" % \
2293                        ( self.url, cntrller, folder_id, library_id ) )
2294        if name and new_name and description:
2295            tc.fv( '1', "name", new_name )
2296            tc.fv( '1', "description", description )
2297            tc.submit( 'rename_folder_button' )
2298        for check_str in strings_displayed:
2299            self.check_page_for_string( check_str )
2300        for check_str in strings_not_displayed:
2301            try:
2302                self.check_page_for_string( check_str )
2303                raise AssertionError( "String (%s) incorrectly displayed." % check_str )
2304            except:
2305                pass
2306        if template_refresh_field_contents:
2307            # A template containing an AddressField is displayed on the form, so we need to refresh the form 
2308            # with the received template_refresh_field_contents.  There are 2 forms on the folder_info page
2309            # when in edit mode, and the 2nd one is the one we want.
2310            self.refresh_form( template_refresh_field_name, template_refresh_field_contents, form_no=2 )
2311        if template_fields:
2312            # We have an information template associated with the folder, so
2313            # there are 2 forms on this page and the template is the 2nd form
2314            for field_name, field_value in template_fields:
2315                tc.fv( "edit_info", field_name, field_value )
2316            tc.submit( 'edit_info_button' )
2317        for check_str in strings_displayed_after_submit:
2318            self.check_page_for_string( check_str )
2319        for check_str in strings_not_displayed_after_submit:
2320            try:
2321                self.check_page_for_string( check_str )
2322                raise AssertionError( "String (%s) incorrectly displayed." % check_str )
2323            except:
2324                pass
2325        self.home()
2326
2327    # Library dataset stuff
2328    def upload_library_dataset( self, cntrller, library_id, folder_id, filename='', server_dir='', replace_id='',
2329                                upload_option='upload_file', file_type='auto', dbkey='hg18', space_to_tab='',
2330                                link_data_only='copy_files', preserve_dirs='Yes', filesystem_paths='', roles=[],
2331                                ldda_message='', hda_ids='', template_refresh_field_name='1_field_name',
2332                                template_refresh_field_contents='', template_fields=[], show_deleted='False', strings_displayed=[] ):
2333        """Add datasets to library using any upload_option"""
2334        # NOTE: due to the library_wait() method call at the end of this method, no tests should be done
2335        # for strings_displayed_after_submit.
2336        url = "%s/library_common/upload_library_dataset?cntrller=%s&library_id=%s&folder_id=%s" % \
2337            ( self.url, cntrller, library_id, folder_id )
2338        if replace_id:
2339            # If we're uploading a new version of a library dataset, we have to include the replace_id param in the
2340            # request because the form field named replace_id will not be displayed on the upload form if we dont.
2341            url += "&replace_id=%s" % replace_id
2342        self.visit_url( url )
2343        if template_refresh_field_contents:
2344            # A template containing an AddressField is displayed on the upload form, so we need to refresh the form 
2345            # with the received template_refresh_field_contents.
2346            self.refresh_form( template_refresh_field_name, template_refresh_field_contents )
2347        for tup in template_fields:
2348            tc.fv( "1", tup[0], tup[1] )
2349        tc.fv( "1", "library_id", library_id )
2350        tc.fv( "1", "folder_id", folder_id )
2351        tc.fv( "1", "show_deleted", show_deleted )
2352        tc.fv( "1", "ldda_message", ldda_message )
2353        tc.fv( "1", "file_type", file_type )
2354        tc.fv( "1", "dbkey", dbkey )
2355        if space_to_tab:
2356            tc.fv( "1", "space_to_tab", space_to_tab )
2357        for role_id in roles:
2358            tc.fv( "1", "roles", role_id )
2359        # Refresh the form by selecting the upload_option - we do this here to ensure
2360        # all previously entered form contents are retained.
2361        self.refresh_form( 'upload_option', upload_option )
2362        if upload_option == 'import_from_history':
2363            for check_str in strings_displayed:
2364                self.check_page_for_string( check_str )
2365            if hda_ids:
2366                # Twill cannot handle multi-checkboxes, so the form can only have 1 hda_ids checkbox
2367                try:
2368                    tc.fv( "add_history_datasets_to_library", "hda_ids", hda_ids )
2369                except:
2370                    tc.fv( "add_history_datasets_to_library", "hda_ids", '1' )
2371            tc.submit( 'add_history_datasets_to_library_button' )
2372        else:
2373            if upload_option in [ 'upload_paths', 'upload_directory' ]:
2374                tc.fv( "1", "link_data_only", link_data_only )
2375            if upload_option == 'upload_paths':
2376                tc.fv( "1", "filesystem_paths", filesystem_paths )
2377            if upload_option == 'upload_directory' and server_dir:
2378                tc.fv( "1", "server_dir", server_dir )
2379            if upload_option == 'upload_file':
2380                if filename:
2381                    filename = self.get_filename( filename )
2382                    tc.formfile( "1", "files_0|file_data", filename )
2383            for check_str in strings_displayed:
2384                self.check_page_for_string( check_str )
2385            tc.submit( "runtool_btn" )
2386        # Give the files some time to finish uploading
2387        self.library_wait( library_id )
2388        self.home()
2389    def ldda_permissions( self, cntrller, library_id, folder_id, id, role_ids_str,
2390                          permissions_in=[], permissions_out=[], strings_displayed=[], ldda_name='' ):
2391        # role_ids_str must be a comma-separated string of role ids
2392        url = "%s/library_common/ldda_permissions?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
2393            ( self.url, cntrller, library_id, folder_id, id )
2394        for po in permissions_out:
2395            key = '%s_out' % po
2396            url = "%s&%s=%s" % ( url, key, role_ids_str )
2397        for pi in permissions_in:
2398            key = '%s_in' % pi
2399            url = "%s&%s=%s" % ( url, key, role_ids_str )
2400        if permissions_in or permissions_out:
2401            url += "&update_roles_button=Save"
2402            self.visit_url( url )
2403        if not strings_displayed:
2404            strings_displayed = [ "Permissions updated for dataset '%s'." % ldda_name ]
2405        for check_str in strings_displayed:
2406            self.check_page_for_string( check_str )
2407        self.home()
2408    def ldda_info( self, cntrller, library_id, folder_id, ldda_id, strings_displayed=[], strings_not_displayed=[] ):
2409        """View library_dataset_dataset_association information"""
2410        self.visit_url( "%s/library_common/ldda_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
2411                        ( self.url, cntrller, library_id, folder_id, ldda_id ) )
2412        for check_str in strings_displayed:
2413            self.check_page_for_string( check_str )
2414        for check_str in strings_not_displayed:
2415            try:
2416                self.check_page_for_string( check_str )
2417                raise AssertionError( "String (%s) should not have been displayed on ldda info page." % check_str )
2418            except:
2419                pass
2420        self.home()
2421    def ldda_edit_info( self, cntrller, library_id, folder_id, ldda_id, ldda_name, new_ldda_name='', template_refresh_field_name='1_field_name',
2422                        template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[] ):
2423        """Edit library_dataset_dataset_association information, optionally template element information"""
2424        self.visit_url( "%s/library_common/ldda_edit_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
2425                        ( self.url, cntrller, library_id, folder_id, ldda_id ) )        
2426        check_str = 'Edit attributes of %s' % ldda_name
2427        self.check_page_for_string( check_str )
2428        if new_ldda_name:
2429            tc.fv( '1', 'name', new_ldda_name )
2430            tc.submit( 'save' )
2431            check_str = "Attributes updated for library dataset '%s'." % new_ldda_name
2432            self.check_page_for_string( check_str )
2433        if template_refresh_field_contents:
2434            # A template containing an AddressField is displayed on the upload form, so we need to refresh the form 
2435            # with the received template_refresh_field_contents.  There are 4 forms on this page, and the template is
2436            # contained in the 4th form named "edit_info".
2437            self.refresh_form( template_refresh_field_name, template_refresh_field_contents, form_no=4 )
2438        if template_fields:
2439            # We have an information template associated with the folder, so
2440            # there are 2 forms on this page and the template is the 2nd form
2441            for field_name, field_value in template_fields:
2442                tc.fv( "edit_info", field_name, field_value )
2443            tc.submit( 'edit_info_button' )
2444        for check_str in strings_displayed:
2445            self.check_page_for_string( check_str )
2446        for check_str in strings_not_displayed:
2447            try:
2448                self.check_page_for_string( check_str )
2449                raise AssertionError( "String (%s) should not have been displayed on ldda Edit Attributes page." % check_str )
2450            except:
2451                pass
2452        self.home()
2453    def act_on_multiple_datasets( self, cntrller, library_id, do_action, ldda_ids='', strings_displayed=[] ):
2454        # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission
2455        # by going directly to the form action
2456        self.visit_url( '%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&ldda_ids=%s&do_action=%s' \
2457                        % ( self.url, cntrller, library_id, ldda_ids, do_action ) )
2458        for check_str in strings_displayed:
2459            self.check_page_for_string( check_str )
2460    def import_datasets_to_histories( self, cntrller, library_id, ldda_ids='', new_history_name='Unnamed history', strings_displayed=[] ):
2461        # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission
2462        # by going directly to the form action
2463        self.visit_url( '%s/library_common/import_datasets_to_histories?cntrller=%s&library_id=%s&ldda_ids=%s&new_history_name=%s&import_datasets_to_histories_button=Import+library+datasets' \
2464                        % ( self.url, cntrller, library_id, ldda_ids, new_history_name ) )
2465        for check_str in strings_displayed:
2466            self.check_page_for_string( check_str )
2467    def download_archive_of_library_files( self, cntrller, library_id, ldda_ids, format ):
2468        self.home()
2469        # Here it would be ideal to have twill set form values and submit the form, but
2470        # twill barfs on that due to the recently introduced page wrappers around the contents
2471        # of the browse_library.mako template which enable panel layout when visiting the
2472        # page from an external URL.  By "barfs", I mean that twill somehow loses hod on the 
2473        # cntrller param.  We'll just simulate the form submission by building the URL manually.
2474        # Here's the old, better approach...
2475        #self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
2476        #for ldda_id in ldda_ids:
2477        #    tc.fv( "1", "ldda_ids", ldda_id )
2478        #tc.fv( "1", "do_action", format )
2479        #tc.submit( "action_on_datasets_button" )
2480        # Here's the new approach...
2481        url = "%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&do_action=%s" \
2482            % ( self.url, cntrller, library_id, format )
2483        for ldda_id in ldda_ids:
2484            url += "&ldda_ids=%s" % ldda_id
2485        self.visit_url( url )
2486        tc.code( 200 )
2487        archive = self.write_temp_file( self.last_page(), suffix='.' + format )
2488        self.home()
2489        return archive
2490    def check_archive_contents( self, archive, lddas ):
2491        def get_ldda_path( ldda ):
2492            path = ""
2493            parent_folder = ldda.library_dataset.folder
2494            while parent_folder is not None:
2495                if parent_folder.parent is None:
2496                    path = os.path.join( parent_folder.library_root[0].name, path )
2497                    break
2498                path = os.path.join( parent_folder.name, path )
2499                parent_folder = parent_folder.parent
2500            path += ldda.name
2501            return path
2502        def mkdir( file ):
2503            dir = os.path.join( tmpd, os.path.dirname( file ) )
2504            if not os.path.exists( dir ):
2505                os.makedirs( dir )
2506        tmpd = tempfile.mkdtemp()
2507        if tarfile.is_tarfile( archive ):
2508            t = tarfile.open( archive )
2509            for n in t.getnames():
2510                mkdir( n )
2511                t.extract( n, tmpd )
2512            t.close()
2513        elif zipfile.is_zipfile( archive ):
2514            z = zipfile.ZipFile( archive, 'r' )
2515            for n in z.namelist():
2516                mkdir( n )
2517                open( os.path.join( tmpd, n ), 'wb' ).write( z.read( n ) )
2518            z.close()
2519        else:
2520            raise Exception( 'Unable to read archive: %s' % archive )
2521        for ldda in lddas:
2522            orig_file = self.get_filename( ldda.name )
2523            downloaded_file = os.path.join( tmpd, get_ldda_path( ldda ) )
2524            assert os.path.exists( downloaded_file )
2525            try:
2526                self.files_diff( orig_file, downloaded_file )
2527            except AssertionError, err:
2528                errmsg = 'Library item %s different than expected, difference:\n' % ldda.name
2529                errmsg += str( err )
2530                errmsg += 'Unpacked archive remains in: %s\n' % tmpd
2531                raise AssertionError( errmsg )
2532        shutil.rmtree( tmpd )
2533    def move_library_item( self, cntrller, item_type, item_id, source_library_id, make_target_current,
2534                           target_library_id='', target_folder_id='', strings_displayed=[], strings_displayed_after_submit=[] ):
2535        self.home()
2536        self.visit_url( "%s/library_common/move_library_item?cntrller=%s&item_type=%s&item_id=%s&source_library_id=%s&make_target_current=%s" \
2537                        % ( self.url, cntrller, item_type, item_id, source_library_id, make_target_current ) )
2538        if target_library_id:
2539            self.refresh_form( 'target_library_id', target_library_id )
2540        if target_folder_id:
2541            tc.fv( '1', 'target_folder_id', target_folder_id )
2542        for check_str in strings_displayed:
2543            self.check_page_for_string( check_str )
2544        tc.submit( 'move_library_item_button' )
2545        for check_str in strings_displayed_after_submit:
2546            self.check_page_for_string( check_str )
2547        self.home()
2548    def delete_library_item( self, cntrller, library_id, item_id, item_name, item_type='library_dataset' ):
2549        """Mark a library item as deleted"""
2550        self.home()
2551        self.visit_url( "%s/library_common/delete_library_item?cntrller=%s&library_id=%s&item_id=%s&item_type=%s" \
2552                        % ( self.url, cntrller, library_id, item_id, item_type ) )
2553        if item_type == 'library_dataset':
2554            item_desc = 'Dataset'
2555        else:
2556            item_desc = item_type.capitalize()
2557        check_str = "marked deleted"
2558        self.check_page_for_string( check_str )
2559        self.home()
2560    def undelete_library_item( self, cntrller, library_id, item_id, item_name, item_type='library_dataset' ):
2561        """Mark a library item as deleted"""
2562        self.home()
2563        self.visit_url( "%s/library_common/undelete_library_item?cntrller=%s&library_id=%s&item_id=%s&item_type=%s" \
2564                        % ( self.url, cntrller, library_id, item_id, item_type ) )
2565        if item_type == 'library_dataset':
2566            item_desc = 'Dataset'
2567        else:
2568            item_desc = item_type.capitalize()
2569        check_str = "marked undeleted"
2570        self.check_page_for_string( check_str )
2571        self.home()
2572    def purge_library( self, library_id, library_name ):
2573        """Purge a library"""
2574        self.home()
2575        self.visit_url( "%s/library_admin/purge_library?id=%s" % ( self.url, library_id ) )
2576        check_str = "Library '%s' and all of its contents have been purged" % library_name
2577        self.check_page_for_string( check_str )
2578        self.home()
2579    def library_wait( self, library_id, cntrller='library_admin', maxiter=90 ):
2580        """Waits for the tools to finish"""
2581        count = 0
2582        sleep_amount = 1
2583        while count < maxiter:
2584            count += 1
2585            self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
2586            page = tc.browser.get_html()
2587            if page.find( '<!-- running: do not change this comment, used by TwillTestCase.library_wait -->' ) > -1:
2588                time.sleep( sleep_amount )
2589                sleep_amount += 1
2590            else:
2591                break
2592        self.assertNotEqual(count, maxiter)
2593
2594    # Tests associated with tags
2595    def add_tag( self, item_id, item_class, context, new_tag ):
2596        self.visit_url( "%s/tag/add_tag_async?item_id=%s&item_class=%s&context=%s&new_tag=%s" % \
2597                        ( self.url, item_id, item_class, context, new_tag ) )
2598    def get_tags( self, item_id, item_class ):
2599        self.visit_url( "%s/tag/get_tagging_elt_async?item_id=%s&item_class=%s" % \
2600                        ( self.url, item_id, item_class ) )