/scripts/galaxy_messaging/server/amqp_consumer.py

https://bitbucket.org/cistrome/cistrome-harvard/ · Python · 164 lines · 128 code · 18 blank · 18 comment · 11 complexity · 6b91ec8bfa4738f8b0112040fcf26df0 MD5 · raw file

  1. '''
  2. Galaxy Messaging with AMQP (RabbitMQ)
  3. Galaxy uses AMQ protocol to receive messages from external sources like
  4. bar code scanners. Galaxy has been tested against RabbitMQ AMQP implementation.
  5. For Galaxy to receive messages from a message queue the RabbitMQ server has
  6. to be set up with a user account and other parameters listed in the [galaxy_amqp]
  7. section in the universe_wsgi.ini config file
  8. Once the RabbitMQ server has been setup and started with the given parameters,
  9. this script can be run to receive messages and update the Galaxy database accordingly
  10. '''
  11. import ConfigParser
  12. import sys, os
  13. import optparse
  14. import xml.dom.minidom
  15. import subprocess
  16. import urllib2
  17. from xml_helper import get_value, get_value_index
  18. from galaxydb_interface import GalaxyDbInterface
  19. api_path = [ os.path.join( os.getcwd(), "scripts/api" ) ]
  20. sys.path.extend( api_path )
  21. import common as api
  22. assert sys.version_info[:2] >= ( 2, 4 )
  23. new_path = [ os.path.join( os.getcwd(), "lib" ) ]
  24. new_path.extend( sys.path[1:] ) # remove scripts/ from the path
  25. sys.path = new_path
  26. from galaxy import eggs
  27. from galaxy.web.api.requests import RequestsAPIController
  28. import pkg_resources
  29. pkg_resources.require( "amqp" )
  30. import amqp
  31. import logging
  32. log = logging.getLogger("GalaxyAMQP")
  33. log.setLevel(logging.DEBUG)
  34. fh = logging.FileHandler("galaxy_listener.log")
  35. fh.setLevel(logging.DEBUG)
  36. formatter = logging.Formatter("%(asctime)s - %(name)s - %(message)s")
  37. fh.setFormatter(formatter)
  38. log.addHandler(fh)
  39. # data transfer script
  40. data_transfer_script = os.path.join( os.getcwd(),
  41. "scripts/galaxy_messaging/server/data_transfer.py" )
  42. global config
  43. global config_file_name
  44. global http_server_section
  45. def start_data_transfer( message ):
  46. # fork a new process to transfer datasets
  47. cmd = '%s "%s" "%s" "%s"' % ( "python",
  48. data_transfer_script,
  49. message.body,
  50. config_file_name ) # Galaxy config file name
  51. pid = subprocess.Popen(cmd, shell=True).pid
  52. log.debug('Started process (%i): %s' % (pid, str(cmd)))
  53. def update_sample_state( message ):
  54. dom = xml.dom.minidom.parseString(message.body)
  55. barcode = get_value(dom, 'barcode')
  56. state = get_value(dom, 'state')
  57. api_key = get_value(dom, 'api_key')
  58. log.debug('Barcode: ' + barcode)
  59. log.debug('State: ' + state)
  60. log.debug('API Key: ' + api_key)
  61. # validate
  62. if not barcode or not state or not api_key:
  63. log.debug( 'Incomplete sample_state_update message received. Sample barcode, desired state and user API key is required.' )
  64. return
  65. # update the sample state in Galaxy db
  66. dbconnstr = config.get("app:main", "database_connection")
  67. galaxydb = GalaxyDbInterface( dbconnstr )
  68. sample_id = galaxydb.get_sample_id( field_name='bar_code', value=barcode )
  69. if sample_id == -1:
  70. log.debug( 'Invalid barcode.' )
  71. return
  72. galaxydb.change_state(sample_id, state)
  73. # after updating the sample state, update request status
  74. request_id = galaxydb.get_request_id(sample_id)
  75. update_request( api_key, request_id )
  76. def update_request( api_key, request_id ):
  77. encoded_request_id = api.encode_id( config.get( "app:main", "id_secret" ), request_id )
  78. data = dict( update_type=RequestsAPIController.update_types.REQUEST )
  79. url = "http://%s:%s/api/requests/%s" % ( config.get(http_server_section, "host"),
  80. config.get(http_server_section, "port"),
  81. encoded_request_id )
  82. log.debug( 'Updating request %i' % request_id )
  83. try:
  84. retval = api.update( api_key, url, data, return_formatted=False )
  85. log.debug( str( retval ) )
  86. except Exception, e:
  87. log.debug( 'ERROR(update_request (%s)): %s' % ( str((self.api_key, url, data)), str(e) ) )
  88. def recv_callback( message ):
  89. # check the meesage type.
  90. msg_type = message.properties['application_headers'].get('msg_type')
  91. log.debug( 'MESSAGE RECVD: ' + str( msg_type ) )
  92. if msg_type == 'data_transfer':
  93. log.debug( 'DATA TRANSFER' )
  94. start_data_transfer( message )
  95. elif msg_type == 'sample_state_update':
  96. log.debug( 'SAMPLE STATE UPDATE' )
  97. update_sample_state( message )
  98. def main():
  99. parser = optparse.OptionParser()
  100. parser.add_option('-c', '--config-file', help='Galaxy configuration file',
  101. dest='config_file', action='store')
  102. parser.add_option('-s', '--http-server-section', help='Name of the HTTP server section in the Galaxy configuration file',
  103. dest='http_server_section', action='store')
  104. (opts, args) = parser.parse_args()
  105. log.debug( "GALAXY LISTENER PID: " + str(os.getpid()) + " - " + str( opts ) )
  106. # read the Galaxy config file
  107. global config_file_name
  108. config_file_name = opts.config_file
  109. global config
  110. config = ConfigParser.ConfigParser()
  111. config.read( opts.config_file )
  112. global http_server_section
  113. http_server_section = opts.http_server_section
  114. amqp_config = {}
  115. for option in config.options("galaxy_amqp"):
  116. amqp_config[option] = config.get("galaxy_amqp", option)
  117. log.debug( str( amqp_config ) )
  118. # connect
  119. conn = amqp.Connection(host=amqp_config['host']+":"+amqp_config['port'],
  120. userid=amqp_config['userid'],
  121. password=amqp_config['password'],
  122. virtual_host=amqp_config['virtual_host'])
  123. chan = conn.channel()
  124. chan.queue_declare( queue=amqp_config['queue'],
  125. durable=True,
  126. exclusive=True,
  127. auto_delete=False)
  128. chan.exchange_declare( exchange=amqp_config['exchange'],
  129. type="direct",
  130. durable=True,
  131. auto_delete=False,)
  132. chan.queue_bind( queue=amqp_config['queue'],
  133. exchange=amqp_config['exchange'],
  134. routing_key=amqp_config['routing_key'])
  135. chan.basic_consume( queue=amqp_config['queue'],
  136. no_ack=True,
  137. callback=recv_callback,
  138. consumer_tag="testtag")
  139. log.debug('Connected to rabbitmq server - '+amqp_config['host']+":"+amqp_config['port'])
  140. while True:
  141. chan.wait()
  142. chan.basic_cancel("testtag")
  143. chan.close()
  144. conn.close()
  145. if __name__ == '__main__':
  146. main()