PageRenderTime 52ms CodeModel.GetById 7ms RepoModel.GetById 0ms app.codeStats 0ms

/historical/hive_odbc.py

https://bitbucket.org/lindenlab/apiary/
Python | 251 lines | 221 code | 6 blank | 24 comment | 3 complexity | bf874214602301adc690d5dc34709583 MD5 | raw file
  1. #
  2. # $LicenseInfo:firstyear=2010&license=mit$
  3. #
  4. # Copyright (c) 2010, Linden Research, Inc.
  5. #
  6. # Permission is hereby granted, free of charge, to any person obtaining a copy
  7. # of this software and associated documentation files (the "Software"), to deal
  8. # in the Software without restriction, including without limitation the rights
  9. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10. # copies of the Software, and to permit persons to whom the Software is
  11. # furnished to do so, subject to the following conditions:
  12. #
  13. # The above copyright notice and this permission notice shall be included in
  14. # all copies or substantial portions of the Software.
  15. #
  16. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22. # THE SOFTWARE.
  23. # $/LicenseInfo$
  24. #
  25. from optparse import OptionParser
  26. import mx.ODBC.iODBC
  27. import re
  28. import time
  29. import hive
  30. import sqllog
  31. import timestamp
  32. def now(future=0.0):
  33. return timestamp.TimeStamp(time.time() + future)
  34. def wait_until(ts):
  35. tn = now()
  36. if ts > tn:
  37. delta = float(ts - tn)
  38. if delta > 65.0:
  39. raise Exception("trying to wait longer than 65s. now = %s, until = %s"
  40. % (str(tn), str(ts)))
  41. time.sleep(delta)
  42. default_dsn = 'Local MySQL'
  43. default_db_host = 'localhost'
  44. default_db_port = 3306
  45. default_db_user = 'guest'
  46. default_db_passwd = ''
  47. default_db_database = 'test'
  48. class ODBCWorker(hive.Worker):
  49. def __init__(self, options, arguments):
  50. hive.Worker.__init__(self, options, arguments)
  51. self._asap = options.asap
  52. self._error = False
  53. self._errormsg = ''
  54. self._connect_options = {}
  55. self._connect_options['dsn'] = options.dsn
  56. self._connect_options['host'] = options.db_host
  57. self._connect_options['port'] = options.db_port
  58. self._connect_options['user'] = options.db_user
  59. self._connect_options['passwd'] = options.db_passwd
  60. self._connect_options['db'] = options.db_database
  61. self._connection = None
  62. def start(self):
  63. # self.log("starting")
  64. self._error = False
  65. self._errormsg = ''
  66. try:
  67. if self._connection is None:
  68. # self.log("making ODBC connection")
  69. connection = mx.ODBC.iODBC.Connect(self._connect_options['dsn'], self._connect_options['user'], self._connect_options['passwd'])
  70. self._connection = connection
  71. pass
  72. except Exception, e: # more restrictive error catching?
  73. self._error = True
  74. self._errormsg = "500 " + str(e)
  75. def event(self, data):
  76. # self.log("event")
  77. if self._error:
  78. return
  79. try:
  80. (tstr, sql) = data.split("\t", 1)
  81. if not self._asap:
  82. # self.log("waiting")
  83. wait_until(timestamp.TimeStamp(tstr))
  84. sql = sql.strip()
  85. if sql:
  86. #self.log("executing SQL: " + sql)
  87. self.execute_sql(sql)
  88. # self.log("finished SQL")
  89. except Exception, e: # more restrictive error catching?
  90. self._error = True
  91. self._errormsg = "501 " + str(e)
  92. def end(self):
  93. # try:
  94. # self.log("committing")
  95. # self._connection.commit() # unclear if this should be here
  96. # except Exception, e:
  97. # pass
  98. try:
  99. if True:
  100. # self.log("closing")
  101. self._connection.close()
  102. self._connection = None
  103. except Exception, e:
  104. if not self._error:
  105. self._error = True
  106. self._errormsg = "502 " + str(e)
  107. # self.log("ended")
  108. if self._error:
  109. return self._errormsg
  110. return '200 OK'
  111. def execute_sql(self, sql):
  112. """Execute an SQL statement.
  113. Subclasses may override this to alter the SQL before being executed.
  114. If so, they should call this inherited version to actually execute
  115. the statement. It is acceptable for a sub-class to call this version
  116. multiple times.
  117. Exceptions are caught in the outer calling function (event())
  118. """
  119. cursor = self._connection.cursor()
  120. cursor.execute(sql)
  121. try:
  122. cursor.fetchall()
  123. except:
  124. pass # not all SQL has data to fetch
  125. cursor.close()
  126. class ODBCCentral(hive.Central):
  127. def __init__(self, options, arguments):
  128. hive.Central.__init__(self, options, arguments)
  129. self._events = sqllog.input_events(arguments)
  130. # this builds a sequence of events from the log streams in the
  131. # arguments, which come here from the command line
  132. self._connections = {}
  133. self._tally = {}
  134. self._time_offset = None
  135. self._tally_time = time.time() + 15.0
  136. def tally(self, msg):
  137. if "Duplicate entry" in msg:
  138. msg = '501 (1062, "Duplicate entry for key")'
  139. if "You have an error in your SQL syntax" in msg:
  140. msg = '501 (1064, "You have an error in your SQL syntax")'
  141. self._tally[msg] = self._tally.get(msg, 0) + 1
  142. if time.time() > self._tally_time:
  143. self.print_tally()
  144. def print_tally(self):
  145. keys = self._tally.keys()
  146. keys.sort()
  147. print
  148. print " count - message"
  149. print "------------ -------------------------------------------"
  150. for k in keys:
  151. print ("%12d - %s" % (self._tally[k], k))
  152. self._tally_time = time.time() + 15.0
  153. def next(self):
  154. try:
  155. while True:
  156. e = self._events.next() # e is a sqllog.Event object
  157. if e.state != sqllog.Event.Response:
  158. break
  159. except StopIteration:
  160. return False
  161. if self._time_offset is None:
  162. self._time_offset = now(1.0) - e.time
  163. t = self._time_offset + e.time
  164. id = e.id
  165. if e.state == sqllog.Event.End:
  166. if id in self._connections:
  167. self.tally("102 End connection")
  168. del self._connections[id]
  169. self.end(id)
  170. else:
  171. self.tally("103 Duplicate end")
  172. else:
  173. if id not in self._connections:
  174. self.tally("100 Start connection")
  175. s = self._connections[id] = True
  176. self.start(id)
  177. self.tally("101 Event")
  178. self.event(id, str(t) + "\t" + e.body)
  179. return True
  180. def result(self, seq, d):
  181. self.tally(d)
  182. def main(self):
  183. t = - time.time()
  184. c = - time.clock()
  185. hive.Central.main(self)
  186. c += time.clock()
  187. t += time.time()
  188. # This never actually happens because hive.Central.main(self) never completes.
  189. # Figure out how to force it to say it exited neatly?
  190. print ("Timing: %f process clock, %f wall clock" % (c, t))
  191. self.print_tally()
  192. class ODBCHive(hive.Hive):
  193. def __init__(self, central_cls=ODBCCentral, worker_cls=ODBCWorker):
  194. hive.Hive.__init__(self, central_cls, worker_cls)
  195. def add_options(self, parser):
  196. hive.Hive.add_options(self, parser)
  197. parser.add_option('--asap',
  198. action='store_true', default=False,
  199. help='run SQL connections as fast as possible (default: off)')
  200. parser.add_option('--dsn',
  201. default=default_dsn, metavar='DSN',
  202. help='ODBC DSN to connect to (default: %default)')
  203. parser.add_option('--db-host',
  204. default=default_db_host, metavar='HOST',
  205. help='ODBC server to connect to (default: %default)')
  206. parser.add_option('--db-port',
  207. default=default_db_port, metavar='PORT',
  208. help='ODBC port to connect on (default: %default)')
  209. parser.add_option('--db-user',
  210. default=default_db_user, metavar='USER',
  211. help='ODBC user to connect as (default: %default)')
  212. parser.add_option('--db-passwd',
  213. default=default_db_passwd, metavar='PW',
  214. help='ODBC password to connect with (default: %default)')
  215. parser.add_option('--db-database',
  216. default=default_db_database, metavar='DB',
  217. help='ODBC database to connect to (default: %default)')
  218. if __name__ == '__main__':
  219. ODBCHive().main()