/scripts/check_galaxy.py

https://bitbucket.org/cistrome/cistrome-harvard/ · Python · 401 lines · 392 code · 0 blank · 9 comment · 5 complexity · a2dbbfc279f5d876091d73f3f38c6188 MD5 · raw file

  1. #!/usr/bin/env python
  2. """
  3. check_galaxy can be run by hand, although it is meant to run from cron
  4. via the check_galaxy.sh script in Galaxy's cron/ directory.
  5. """
  6. import socket, sys, os, time, tempfile, filecmp, htmllib, formatter, getopt
  7. from user import home
  8. # options
  9. if os.environ.has_key( "DEBUG" ):
  10. debug = os.environ["DEBUG"]
  11. else:
  12. debug = False
  13. scripts_dir = os.path.abspath( os.path.dirname( sys.argv[0] ) )
  14. test_data_dir = os.path.join( scripts_dir, "..", "test-data" )
  15. # what tools to run - not so pretty
  16. tools = {
  17. "gops_intersect_1" :
  18. [
  19. {
  20. "inputs" :
  21. (
  22. os.path.join( test_data_dir, "1.bed" ),
  23. os.path.join( test_data_dir, "2.bed" )
  24. )
  25. },
  26. { "check_file" : os.path.join( test_data_dir, "gops_intersect_out.bed" ) },
  27. {
  28. "tool_run_options" :
  29. {
  30. "input1" : "1.bed",
  31. "input2" : "2.bed",
  32. "min" : "1",
  33. "returntype" : ""
  34. }
  35. }
  36. ]
  37. }
  38. # handle arg(s)
  39. def usage():
  40. print "usage: check_galaxy.py <server>"
  41. sys.exit(1)
  42. try:
  43. opts, args = getopt.getopt( sys.argv[1:], 'n' )
  44. except getopt.GetoptError, e:
  45. print str(e)
  46. usage()
  47. if len( args ) < 1:
  48. usage()
  49. server = args[0]
  50. if server.endswith(".g2.bx.psu.edu"):
  51. if debug:
  52. print "Checking a PSU Galaxy server, using maint file"
  53. maint = "/errordocument/502/%s/maint" % args[0].split('.', 1)[0]
  54. else:
  55. maint = None
  56. new_history = False
  57. for o, a in opts:
  58. if o == "-n":
  59. if debug:
  60. print "Specified -n, will create a new history"
  61. new_history = True
  62. else:
  63. usage()
  64. # state information
  65. var_dir = os.path.join( home, ".check_galaxy", server )
  66. if not os.access( var_dir, os.F_OK ):
  67. os.makedirs( var_dir, 0700 )
  68. # get user/pass
  69. login_file = os.path.join( var_dir, "login" )
  70. try:
  71. f = open( login_file, 'r' )
  72. except:
  73. print "Please create the file:"
  74. print " ", login_file
  75. print "This should contain a username and password to log in to"
  76. print "Galaxy with, on one line, separated by whitespace, e.g.:"
  77. print ""
  78. print "check_galaxy@example.com password"
  79. print ""
  80. print "If the user does not exist, check_galaxy will create it"
  81. print "for you."
  82. sys.exit(1)
  83. ( username, password ) = f.readline().split()
  84. # find/import twill
  85. lib_dir = os.path.join( scripts_dir, "..", "lib" )
  86. sys.path.append( lib_dir )
  87. from galaxy import eggs
  88. import pkg_resources
  89. pkg_resources.require( "twill" )
  90. import twill
  91. import twill.commands as tc
  92. # default timeout for twill browser is never
  93. socket.setdefaulttimeout(300)
  94. # user-agent
  95. tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.1)")
  96. tc.config('use_tidy', 0)
  97. class Browser:
  98. def __init__(self):
  99. self.server = server
  100. self.maint = maint
  101. self.tool = None
  102. self.tool_opts = None
  103. self.id = None
  104. self.status = None
  105. self.check_file = None
  106. self.hid = None
  107. self.cookie_jar = os.path.join( var_dir, "cookie_jar" )
  108. dprint("cookie jar path: %s" % self.cookie_jar)
  109. if not os.access(self.cookie_jar, os.R_OK):
  110. dprint("no cookie jar at above path, creating")
  111. tc.save_cookies(self.cookie_jar)
  112. tc.load_cookies(self.cookie_jar)
  113. def get(self, path):
  114. tc.go("http://%s%s" % (self.server, path))
  115. tc.code(200)
  116. def reset(self):
  117. self.tool = None
  118. self.tool_opts = None
  119. self.id = None
  120. self.status = None
  121. self.check_file = None
  122. self.delete_datasets()
  123. self.get("/root/history")
  124. p = didParser()
  125. p.feed(tc.browser.get_html())
  126. if len(p.dids) > 0:
  127. print "Remaining datasets ids:", " ".join( p.dids )
  128. raise Exception, "History still contains datasets after attempting to delete them"
  129. if new_history:
  130. self.get("/history/delete_current")
  131. tc.save_cookies(self.cookie_jar)
  132. def check_redir(self, url):
  133. try:
  134. tc.get_browser()._browser.set_handle_redirect(False)
  135. tc.go(url)
  136. tc.code(302)
  137. tc.get_browser()._browser.set_handle_redirect(True)
  138. dprint( "%s is returning redirect (302)" % url )
  139. return(True)
  140. except twill.errors.TwillAssertionError, e:
  141. tc.get_browser()._browser.set_handle_redirect(True)
  142. dprint( "%s is not returning redirect (302): %s" % (url, e) )
  143. code = tc.browser.get_code()
  144. if code == 502:
  145. is_maint = self.check_maint()
  146. if is_maint:
  147. dprint( "Galaxy is down, but a maint file was found, so not sending alert" )
  148. sys.exit(0)
  149. else:
  150. print "Galaxy is down (code 502)"
  151. sys.exit(1)
  152. return(False)
  153. # checks for a maint file
  154. def check_maint(self):
  155. if self.maint is None:
  156. #dprint( "Warning: unable to check maint file for %s" % self.server )
  157. return(False)
  158. try:
  159. self.get(self.maint)
  160. return(True)
  161. except twill.errors.TwillAssertionError, e:
  162. return(False)
  163. def login(self, user, pw):
  164. self.get("/user/login")
  165. tc.fv("1", "email", user)
  166. tc.fv("1", "password", pw)
  167. tc.submit("Login")
  168. tc.code(200)
  169. if len(tc.get_browser().get_all_forms()) > 0:
  170. # uh ohs, fail
  171. p = userParser()
  172. p.feed(tc.browser.get_html())
  173. if p.no_user:
  174. dprint("user does not exist, will try creating")
  175. self.create_user(user, pw)
  176. elif p.bad_pw:
  177. raise Exception, "Password is incorrect"
  178. else:
  179. raise Exception, "Unknown error logging in"
  180. tc.save_cookies(self.cookie_jar)
  181. def create_user(self, user, pw):
  182. self.get("/user/create")
  183. tc.fv("1", "email", user)
  184. tc.fv("1", "password", pw)
  185. tc.fv("1", "confirm", pw)
  186. tc.submit("Submit")
  187. tc.code(200)
  188. if len(tc.get_browser().get_all_forms()) > 0:
  189. p = userParser()
  190. p.feed(tc.browser.get_html())
  191. if p.already_exists:
  192. raise Exception, 'The user you were trying to create already exists'
  193. def upload(self, file):
  194. self.get("/tool_runner/index?tool_id=upload1")
  195. tc.fv("1","file_type", "bed")
  196. tc.formfile("1","file_data", file)
  197. tc.submit("runtool_btn")
  198. tc.code(200)
  199. def runtool(self):
  200. self.get("/tool_runner/index?tool_id=%s" % self.tool)
  201. for k, v in self.tool_opts.items():
  202. tc.fv("1", k, v)
  203. tc.submit("runtool_btn")
  204. tc.code(200)
  205. def wait(self):
  206. sleep_amount = 1
  207. count = 0
  208. maxiter = 16
  209. while count < maxiter:
  210. count += 1
  211. self.get("/root/history")
  212. page = tc.browser.get_html()
  213. if page.find( '<!-- running: do not change this comment, used by TwillTestCase.wait -->' ) > -1:
  214. time.sleep( sleep_amount )
  215. sleep_amount += 1
  216. else:
  217. break
  218. if count == maxiter:
  219. raise Exception, "Tool never finished"
  220. def check_status(self):
  221. self.get("/root/history")
  222. p = historyParser()
  223. p.feed(tc.browser.get_html())
  224. if p.status != "ok":
  225. raise Exception, "JOB %s NOT OK: %s" % (p.id, p.status)
  226. self.id = p.id
  227. self.status = p.status
  228. #return((p.id, p.status))
  229. def diff(self):
  230. self.get("/datasets/%s/display/display?to_ext=bed" % self.id)
  231. data = tc.browser.get_html()
  232. tmp = tempfile.mkstemp()
  233. dprint("tmp file: %s" % tmp[1])
  234. tmpfh = os.fdopen(tmp[0], 'w')
  235. tmpfh.write(data)
  236. tmpfh.close()
  237. if filecmp.cmp(tmp[1], self.check_file):
  238. dprint("Tool output is as expected")
  239. else:
  240. if not debug:
  241. os.remove(tmp[1])
  242. raise Exception, "Tool output differs from expected"
  243. if not debug:
  244. os.remove(tmp[1])
  245. def delete_datasets(self):
  246. self.get("/root/history")
  247. p = didParser()
  248. p.feed(tc.browser.get_html())
  249. dids = p.dids
  250. for did in dids:
  251. self.get("/datasets/%s/delete" % did)
  252. def check_if_logged_in(self):
  253. self.get("/user?cntrller=user")
  254. p = loggedinParser()
  255. p.feed(tc.browser.get_html())
  256. return p.logged_in
  257. class userParser(htmllib.HTMLParser):
  258. def __init__(self):
  259. htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
  260. self.in_span = False
  261. self.in_div = False
  262. self.no_user = False
  263. self.bad_pw = False
  264. self.already_exists = False
  265. def start_span(self, attrs):
  266. self.in_span = True
  267. def start_div(self, attrs):
  268. self.in_div = True
  269. def end_span(self):
  270. self.in_span = False
  271. def end_div(self):
  272. self.in_div = False
  273. def handle_data(self, data):
  274. if self.in_span or self.in_div:
  275. if data == "No such user (please note that login is case sensitive)":
  276. self.no_user = True
  277. elif data == "Invalid password":
  278. self.bad_pw = True
  279. elif data == "User with that email already exists":
  280. self.already_exists = True
  281. class historyParser(htmllib.HTMLParser):
  282. def __init__(self):
  283. htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
  284. self.status = None
  285. self.id = None
  286. def start_div(self, attrs):
  287. # find the top history item
  288. for i in attrs:
  289. if i[0] == "class" and i[1].startswith("historyItemWrapper historyItem historyItem-"):
  290. self.status = i[1].rsplit("historyItemWrapper historyItem historyItem-", 1)[1]
  291. dprint("status: %s" % self.status)
  292. if i[0] == "id" and i[1].startswith("historyItem-"):
  293. self.id = i[1].rsplit("historyItem-", 1)[1]
  294. dprint("id: %s" % self.id)
  295. if self.status is not None:
  296. self.reset()
  297. class didParser(htmllib.HTMLParser):
  298. def __init__(self):
  299. htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
  300. self.dids = []
  301. def start_div(self, attrs):
  302. for i in attrs:
  303. if i[0] == "id" and i[1].startswith("historyItemContainer-"):
  304. self.dids.append( i[1].rsplit("historyItemContainer-", 1)[1] )
  305. dprint("got a dataset id: %s" % self.dids[-1])
  306. class loggedinParser(htmllib.HTMLParser):
  307. def __init__(self):
  308. htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
  309. self.in_p = False
  310. self.logged_in = False
  311. def start_p(self, attrs):
  312. self.in_p = True
  313. def end_p(self):
  314. self.in_p = False
  315. def handle_data(self, data):
  316. if self.in_p:
  317. if data == "You are currently not logged in.":
  318. self.logged_in = False
  319. elif data.startswith( "You are currently logged in as " ):
  320. self.logged_in = True
  321. def dprint(str):
  322. if debug:
  323. print str
  324. # do stuff here
  325. if __name__ == "__main__":
  326. dprint("checking %s" % server)
  327. b = Browser()
  328. # login (or not)
  329. if b.check_if_logged_in():
  330. dprint("we are already logged in (via cookies), hooray!")
  331. else:
  332. dprint("not logged in... logging in")
  333. b.login(username, password)
  334. for tool, params in tools.iteritems():
  335. check_file = ""
  336. # make sure history and state is clean
  337. b.reset()
  338. b.tool = tool
  339. # get all the tool run conditions
  340. for dict in params:
  341. for k, v in dict.items():
  342. if k == 'inputs':
  343. for file in v:
  344. b.upload(file)
  345. elif k == 'check_file':
  346. b.check_file = v
  347. elif k == 'tool_run_options':
  348. b.tool_opts = v
  349. else:
  350. raise Exception, "Unknown key in tools dict: %s" % k
  351. b.runtool()
  352. b.wait()
  353. b.check_status()
  354. b.diff()
  355. b.delete_datasets()
  356. # by this point, everything else has succeeded. there should be no maint.
  357. is_maint = b.check_maint()
  358. if is_maint:
  359. print "Galaxy is up and fully functional, but a maint file is in place."
  360. sys.exit(1)
  361. sys.exit(0)