/lib/galaxy/model/migrate/check.py

https://bitbucket.org/cistrome/cistrome-harvard/ · Python · 135 lines · 91 code · 10 blank · 34 comment · 23 complexity · 78b172252efa934aec005cb091c4f1de MD5 · raw file

  1. import sys
  2. import os.path
  3. import logging
  4. from galaxy import eggs
  5. eggs.require( "SQLAlchemy" )
  6. eggs.require( "decorator" ) #Required by sqlalchemy-migrate
  7. eggs.require( "Tempita " ) #Required by sqlalchemy-migrate
  8. eggs.require( "sqlalchemy-migrate" )
  9. from sqlalchemy import *
  10. from sqlalchemy.exc import NoSuchTableError
  11. from migrate.versioning import repository, schema
  12. from galaxy.model.orm import dialect_to_egg
  13. log = logging.getLogger( __name__ )
  14. # path relative to galaxy
  15. migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 )
  16. migrate_repository = repository.Repository( migrate_repository_directory )
  17. def create_or_verify_database( url, galaxy_config_file, engine_options={}, app=None ):
  18. """
  19. Check that the database is use-able, possibly creating it if empty (this is
  20. the only time we automatically create tables, otherwise we force the
  21. user to do it using the management script so they can create backups).
  22. 1) Empty database --> initialize with latest version and return
  23. 2) Database older than migration support --> fail and require manual update
  24. 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
  25. 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
  26. """
  27. dialect = ( url.split( ':', 1 ) )[0]
  28. try:
  29. egg = dialect_to_egg[dialect]
  30. try:
  31. eggs.require( egg )
  32. log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
  33. except:
  34. # If the module is in the path elsewhere (i.e. non-egg), it'll still load.
  35. log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
  36. except KeyError:
  37. # Let this go, it could possibly work with db's we don't support
  38. log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
  39. # Create engine and metadata
  40. engine = create_engine( url, **engine_options )
  41. def migrate():
  42. try:
  43. # Declare the database to be under a repository's version control
  44. db_schema = schema.ControlledSchema.create( engine, migrate_repository )
  45. except:
  46. # The database is already under version control
  47. db_schema = schema.ControlledSchema( engine, migrate_repository )
  48. # Apply all scripts to get to current version
  49. migrate_to_current_version( engine, db_schema )
  50. meta = MetaData( bind=engine )
  51. if app and getattr( app.config, 'database_auto_migrate', False ):
  52. migrate()
  53. return
  54. # Try to load dataset table
  55. try:
  56. dataset_table = Table( "dataset", meta, autoload=True )
  57. except NoSuchTableError:
  58. # No 'dataset' table means a completely uninitialized database. If we have an app, we'll
  59. # set it's new_installation setting to True so the tool migration process will be skipped.
  60. if app:
  61. app.new_installation = True
  62. log.info( "No database, initializing" )
  63. migrate()
  64. return
  65. try:
  66. hda_table = Table( "history_dataset_association", meta, autoload=True )
  67. except NoSuchTableError:
  68. raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
  69. # There is a 'history_dataset_association' table, so we (hopefully) have
  70. # version 1 of the database, but without the migrate_version table. This
  71. # happens if the user has a build from right before migration was added.
  72. # Verify that this is true, if it is any older they'll have to update
  73. # manually
  74. if 'copied_from_history_dataset_association_id' not in hda_table.c:
  75. # The 'copied_from_history_dataset_association_id' column was added in
  76. # rev 1464:c7acaa1bb88f. This is the oldest revision we currently do
  77. # automated versioning for, so stop here
  78. raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
  79. # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure
  80. # that the db has version information. This is the trickiest case -- we
  81. # have a database but no version control, and are assuming it is a certain
  82. # version. If the user has postion version 1 changes this could cause
  83. # problems
  84. try:
  85. version_table = Table( "migrate_version", meta, autoload=True )
  86. except NoSuchTableError:
  87. # The database exists but is not yet under migrate version control, so init with version 1
  88. log.info( "Adding version control to existing database" )
  89. try:
  90. metadata_file_table = Table( "metadata_file", meta, autoload=True )
  91. schema.ControlledSchema.create( engine, migrate_repository, version=2 )
  92. except NoSuchTableError:
  93. schema.ControlledSchema.create( engine, migrate_repository, version=1 )
  94. # Verify that the code and the DB are in sync
  95. db_schema = schema.ControlledSchema( engine, migrate_repository )
  96. if migrate_repository.versions.latest != db_schema.version:
  97. config_arg = ''
  98. if os.path.abspath( os.path.join( os.getcwd(), 'universe_wsgi.ini' ) ) != galaxy_config_file:
  99. config_arg = ' -c %s' % galaxy_config_file.replace( os.path.abspath( os.getcwd() ), '.' )
  100. raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh%s upgrade'."
  101. % ( db_schema.version, migrate_repository.versions.latest, config_arg ) )
  102. else:
  103. log.info( "At database version %d" % db_schema.version )
  104. def migrate_to_current_version( engine, schema ):
  105. # Changes to get to current version
  106. changeset = schema.changeset( None )
  107. for ver, change in changeset:
  108. nextver = ver + changeset.step
  109. log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) )
  110. old_stdout = sys.stdout
  111. class FakeStdout( object ):
  112. def __init__( self ):
  113. self.buffer = []
  114. def write( self, s ):
  115. self.buffer.append( s )
  116. def flush( self ):
  117. pass
  118. sys.stdout = FakeStdout()
  119. try:
  120. schema.runchange( ver, change, changeset.step )
  121. finally:
  122. for message in "".join( sys.stdout.buffer ).split( "\n" ):
  123. log.info( message )
  124. sys.stdout = old_stdout