/tools/data_source/data_source.py

  1#!/usr/bin/env python
  2# Retrieves data from external data source applications and stores in a dataset file.
  3# Data source application parameters are temporarily stored in the dataset file.
  4import socket, urllib, sys, os
  5from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg...
  6from galaxy.util.json import from_json_string, to_json_string
  7from galaxy.util import get_charset_from_http_headers
  8import galaxy.model # need to import model before sniff to resolve a circular import dependency
  9from galaxy.datatypes import sniff
 10from galaxy.datatypes.registry import Registry
 11from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE
 12
 13assert sys.version_info[:2] >= ( 2, 4 )
 14
 15def stop_err( msg ):
 16    sys.stderr.write( msg )
 17    sys.exit()
 18
 19GALAXY_PARAM_PREFIX = 'GALAXY'
 20GALAXY_ROOT_DIR = os.path.realpath( os.path.join( os.path.split( os.path.realpath( __file__ ) )[0], '..', '..' ) )
 21GALAXY_DATATYPES_CONF_FILE = os.path.join( GALAXY_ROOT_DIR, 'datatypes_conf.xml' )
 22
 23def load_input_parameters( filename, erase_file = True ):
 24    datasource_params = {}
 25    try:
 26        json_params = from_json_string( open( filename, 'r' ).read() )
 27        datasource_params = json_params.get( 'param_dict' )
 28    except:
 29        json_params = None
 30        for line in open( filename, 'r' ):
 31            try:
 32                line = line.strip()
 33                fields = line.split( '\t' )
 34                datasource_params[ fields[0] ] = fields[1]
 35            except:
 36                continue
 37    if erase_file:
 38        open( filename, 'w' ).close() #open file for writing, then close, removes params from file
 39    return json_params, datasource_params
 40
 41def __main__():
 42    filename = sys.argv[1]
 43    import os
 44    try:
 45        max_file_size = int( sys.argv[2] )
 46    except:
 47        max_file_size = 0
 48    
 49    job_params, params = load_input_parameters( filename )
 50    if job_params is None: #using an older tabular file
 51        enhanced_handling = False
 52        job_params = dict( param_dict = params )
 53        job_params[ 'output_data' ] =  [ dict( out_data_name = 'output',
 54                                               ext = 'data',
 55                                               file_name = filename,
 56                                               extra_files_path = None ) ]
 57        job_params[ 'job_config' ] = dict( GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, TOOL_PROVIDED_JOB_METADATA_FILE = TOOL_PROVIDED_JOB_METADATA_FILE )
 58    else:
 59        enhanced_handling = True
 60        json_file = open( job_params[ 'job_config' ][ 'TOOL_PROVIDED_JOB_METADATA_FILE' ], 'w' ) #specially named file for output junk to pass onto set metadata
 61    
 62    datatypes_registry = Registry()
 63    datatypes_registry.load_datatypes( root_dir = job_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = job_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] )
 64    
 65    URL = params.get( 'URL', None ) #using exactly URL indicates that only one dataset is being downloaded
 66    URL_method = params.get( 'URL_method', None )
 67    
 68    # The Python support for fetching resources from the web is layered. urllib uses the httplib
 69    # library, which in turn uses the socket library.  As of Python 2.3 you can specify how long
 70    # a socket should wait for a response before timing out. By default the socket module has no
 71    # timeout and can hang. Currently, the socket timeout is not exposed at the httplib or urllib2
 72    # levels. However, you can set the default timeout ( in seconds ) globally for all sockets by
 73    # doing the following.
 74    socket.setdefaulttimeout( 600 )
 75    
 76    for data_dict in job_params[ 'output_data' ]:
 77        cur_filename =  data_dict.get( 'file_name', filename )
 78        cur_URL =  params.get( '%s|%s|URL' % ( GALAXY_PARAM_PREFIX, data_dict[ 'out_data_name' ] ), URL )
 79        if not cur_URL:
 80            open( cur_filename, 'w' ).write( "" )
 81            stop_err( 'The remote data source application has not sent back a URL parameter in the request.' )
 82        
 83        # The following calls to urllib.urlopen() will use the above default timeout
 84        try:
 85            if not URL_method or URL_method == 'get':
 86                page = urllib.urlopen( cur_URL )
 87            elif URL_method == 'post':
 88                page = urllib.urlopen( cur_URL, urllib.urlencode( params ) )
 89        except Exception, e:
 90            stop_err( 'The remote data source application may be off line, please try again later. Error: %s' % str( e ) )
 91        if max_file_size:
 92            file_size = int( page.info().get( 'Content-Length', 0 ) )
 93            if file_size > max_file_size:
 94                stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) )
 95        #do sniff stream for multi_byte
 96        try:
 97            cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) )
 98        except Exception, e:
 99            stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) )
100
101
102        #here import checks that upload tool performs
103        if enhanced_handling:
104            try:
105                ext = sniff.handle_uploaded_dataset_file( filename, datatypes_registry, ext = data_dict[ 'ext' ], is_multi_byte = is_multi_byte )
106            except Exception, e:
107                stop_err( str( e ) )
108            info = dict( type = 'dataset',
109                         dataset_id = data_dict[ 'dataset_id' ],
110                         ext = ext)
111            
112            json_file.write( "%s\n" % to_json_string( info ) )
113
114    
115if __name__ == "__main__": __main__()