PageRenderTime 15ms CodeModel.GetById 2ms app.highlight 9ms RepoModel.GetById 1ms app.codeStats 0ms

/historical/hive_odbc.py

https://bitbucket.org/lindenlab/apiary/
Python | 251 lines | 221 code | 6 blank | 24 comment | 2 complexity | bf874214602301adc690d5dc34709583 MD5 | raw file
  1#
  2# $LicenseInfo:firstyear=2010&license=mit$
  3# 
  4# Copyright (c) 2010, Linden Research, Inc.
  5# 
  6# Permission is hereby granted, free of charge, to any person obtaining a copy
  7# of this software and associated documentation files (the "Software"), to deal
  8# in the Software without restriction, including without limitation the rights
  9# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10# copies of the Software, and to permit persons to whom the Software is
 11# furnished to do so, subject to the following conditions:
 12# 
 13# The above copyright notice and this permission notice shall be included in
 14# all copies or substantial portions of the Software.
 15# 
 16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 22# THE SOFTWARE.
 23# $/LicenseInfo$
 24#
 25
 26from optparse import OptionParser
 27
 28import mx.ODBC.iODBC
 29import re
 30import time
 31
 32import hive
 33import sqllog
 34import timestamp
 35
 36def now(future=0.0):
 37    return timestamp.TimeStamp(time.time() + future)
 38
 39def wait_until(ts):
 40    tn = now()
 41    if ts > tn:
 42        delta = float(ts - tn)
 43        if delta > 65.0:
 44            raise Exception("trying to wait longer than 65s. now = %s, until = %s"
 45                % (str(tn), str(ts)))
 46        time.sleep(delta)
 47
 48default_dsn = 'Local MySQL'
 49default_db_host = 'localhost'
 50default_db_port = 3306
 51default_db_user = 'guest'
 52default_db_passwd = ''
 53default_db_database = 'test'
 54
 55class ODBCWorker(hive.Worker):
 56    def __init__(self, options, arguments):
 57        hive.Worker.__init__(self, options, arguments)
 58        self._asap = options.asap
 59        self._error = False
 60        self._errormsg = ''
 61        self._connect_options = {}
 62        self._connect_options['dsn'] = options.dsn
 63        self._connect_options['host'] = options.db_host
 64        self._connect_options['port'] = options.db_port
 65        self._connect_options['user'] = options.db_user
 66        self._connect_options['passwd'] = options.db_passwd
 67        self._connect_options['db'] = options.db_database
 68        self._connection = None
 69    
 70    def start(self):
 71#        self.log("starting")
 72        self._error = False
 73        self._errormsg = ''
 74        try:
 75            if self._connection is None:
 76#                self.log("making ODBC connection")
 77                connection = mx.ODBC.iODBC.Connect(self._connect_options['dsn'], self._connect_options['user'], self._connect_options['passwd'])
 78                self._connection = connection
 79            pass
 80        except Exception, e: # more restrictive error catching?
 81            self._error = True
 82            self._errormsg = "500 " + str(e)
 83        
 84    def event(self, data):
 85#        self.log("event")
 86        if self._error:
 87            return
 88        try:
 89            (tstr, sql) = data.split("\t", 1)
 90            if not self._asap:
 91#                self.log("waiting")
 92                wait_until(timestamp.TimeStamp(tstr))
 93            sql = sql.strip()
 94            if sql:
 95                #self.log("executing SQL: " + sql)
 96                self.execute_sql(sql)  
 97#                self.log("finished SQL")
 98        except Exception, e: # more restrictive error catching?
 99            self._error = True
100            self._errormsg = "501 " + str(e)
101        
102    def end(self):
103#        try:
104#            self.log("committing")
105#            self._connection.commit() # unclear if this should be here
106#        except Exception, e:
107#            pass
108        try:
109            if True:
110#                self.log("closing")
111                self._connection.close()
112                self._connection = None
113        except Exception, e:
114            if not self._error:
115                self._error = True
116                self._errormsg = "502 " + str(e)
117#        self.log("ended")
118        if self._error:
119            return self._errormsg
120        return '200 OK'
121
122    def execute_sql(self, sql):
123        """Execute an SQL statement.
124        
125        Subclasses may override this to alter the SQL before being executed.
126        If so, they should call this inherited version to actually execute
127        the statement. It is acceptable for a sub-class to call this version
128        multiple times.
129        
130        Exceptions are caught in the outer calling function (event())
131        
132        """
133        cursor = self._connection.cursor()
134        cursor.execute(sql)
135        try:
136            cursor.fetchall()
137        except:
138            pass # not all SQL has data to fetch
139        cursor.close()
140        
141
142class ODBCCentral(hive.Central):
143    def __init__(self, options, arguments):
144        hive.Central.__init__(self, options, arguments)
145        self._events = sqllog.input_events(arguments)
146            # this builds a sequence of events from the log streams in the
147            # arguments, which come here from the command line
148        self._connections = {}
149        self._tally = {}
150        self._time_offset = None
151        self._tally_time = time.time() + 15.0
152        
153    def tally(self, msg):
154        if "Duplicate entry" in msg:
155            msg = '501 (1062, "Duplicate entry for key")'
156        if "You have an error in your SQL syntax" in msg:
157            msg = '501 (1064, "You have an error in your SQL syntax")'
158        self._tally[msg] = self._tally.get(msg, 0) + 1
159        if time.time() > self._tally_time:
160            self.print_tally()
161
162    
163    def print_tally(self):
164        keys = self._tally.keys()
165        keys.sort()
166        print
167        print "       count - message"
168        print "------------   -------------------------------------------"
169        for k in keys:
170            print ("%12d - %s" % (self._tally[k], k))
171        self._tally_time = time.time() + 15.0
172
173        
174    def next(self):
175        try:
176            while True:
177                e = self._events.next() # e is a sqllog.Event object
178                if e.state != sqllog.Event.Response:
179                    break
180        except StopIteration:
181            return False
182        
183        if self._time_offset is None:
184            self._time_offset = now(1.0) - e.time
185        t = self._time_offset + e.time
186        
187        id = e.id
188        if e.state == sqllog.Event.End:
189            if id in self._connections:
190                self.tally("102 End connection")
191                del self._connections[id]
192                self.end(id)
193            else:
194                self.tally("103 Duplicate end")
195        else:
196            if id not in self._connections:
197                self.tally("100 Start connection")
198                s = self._connections[id] = True
199                self.start(id)
200            self.tally("101 Event")
201            self.event(id, str(t) + "\t" + e.body)
202
203        return True
204    
205    def result(self, seq, d):
206        self.tally(d)
207    
208        
209    def main(self):
210        t = - time.time()
211        c = - time.clock()
212        hive.Central.main(self)
213        c += time.clock()
214        t += time.time()
215
216        # This never actually happens because hive.Central.main(self) never completes.  
217        # Figure out how to force it to say it exited neatly?
218        print ("Timing: %f process clock, %f wall clock" % (c, t))
219        self.print_tally()
220
221
222class ODBCHive(hive.Hive):
223    def __init__(self, central_cls=ODBCCentral, worker_cls=ODBCWorker):
224        hive.Hive.__init__(self, central_cls, worker_cls)
225    
226    def add_options(self, parser):
227        hive.Hive.add_options(self, parser)
228        parser.add_option('--asap',
229                            action='store_true', default=False,
230                            help='run SQL connections as fast as possible (default: off)')
231        parser.add_option('--dsn',
232                            default=default_dsn, metavar='DSN',
233                            help='ODBC DSN to connect to (default: %default)')
234        parser.add_option('--db-host',
235                            default=default_db_host, metavar='HOST',
236                            help='ODBC server to connect to (default: %default)')
237        parser.add_option('--db-port',
238                            default=default_db_port, metavar='PORT',
239                            help='ODBC port to connect on (default: %default)')
240        parser.add_option('--db-user',
241                            default=default_db_user, metavar='USER',
242                            help='ODBC user to connect as (default: %default)')
243        parser.add_option('--db-passwd',
244                            default=default_db_passwd, metavar='PW',
245                            help='ODBC password to connect with (default: %default)')
246        parser.add_option('--db-database',
247                            default=default_db_database, metavar='DB',
248                            help='ODBC database to connect to (default: %default)')
249
250if __name__ == '__main__':
251    ODBCHive().main()