/ruffus/test/simpler.py

https://code.google.com/p/ruffus/ · Python · 235 lines · 123 code · 60 blank · 52 comment · 16 complexity · 6592304c1dc5f3b0d872b8230bbb513e MD5 · raw file

  1. #!/usr/bin/env python
  2. """
  3. test_tasks.py
  4. """
  5. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  6. # options
  7. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  8. from optparse import OptionParser
  9. import sys, os
  10. import os.path
  11. import StringIO
  12. # add self to search path for testing
  13. exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
  14. sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
  15. if __name__ == '__main__':
  16. module_name = os.path.split(sys.argv[0])[1]
  17. module_name = os.path.splitext(module_name)[0];
  18. else:
  19. module_name = __name__
  20. parser = OptionParser(version="%prog 1.0")
  21. parser.add_option("-t", "--target_tasks", dest="target_tasks",
  22. action="append",
  23. default = list(),
  24. metavar="JOBNAME",
  25. type="string",
  26. help="Target task(s) of pipeline.")
  27. parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
  28. action="append",
  29. default = list(),
  30. metavar="JOBNAME",
  31. type="string",
  32. help="Pipeline task(s) which will be included even if they are up to date.")
  33. parser.add_option("-j", "--jobs", dest="jobs",
  34. default=1,
  35. metavar="jobs",
  36. type="int",
  37. help="Specifies the number of jobs (commands) to run simultaneously.")
  38. parser.add_option("-v", "--verbose", dest = "verbose",
  39. action="count", default=0,
  40. help="Print more verbose messages for each additional verbose level.")
  41. parser.add_option("-d", "--dependency", dest="dependency_file",
  42. metavar="FILE",
  43. type="string",
  44. help="Print a dependency graph of the pipeline that would be executed "
  45. "to FILE, but do not execute it.")
  46. parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
  47. metavar="FORMAT",
  48. type="string",
  49. default = 'svg',
  50. help="format of dependency graph file. Can be 'ps' (PostScript), "+
  51. "'svg' 'svgz' (Structured Vector Graphics), " +
  52. "'png' 'gif' (bitmap graphics) etc ")
  53. parser.add_option("-n", "--just_print", dest="just_print",
  54. action="store_true", default=False,
  55. help="Print a description of the jobs that would be executed, "
  56. "but do not execute them.")
  57. parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
  58. action="store_true", default=False,
  59. help="Rebuild a minimum of tasks necessary for the target. "
  60. "Ignore upstream out of date tasks if intervening tasks are fine.")
  61. parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
  62. action="store_true", default=False,
  63. help="Do not print out legend and key for dependency graph.")
  64. parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
  65. action="store_true", default=False,
  66. help="Draw horizontal dependency graph.")
  67. parameters = [
  68. ]
  69. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  70. # imports
  71. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  72. import StringIO
  73. import re
  74. import operator
  75. import sys,os
  76. from collections import defaultdict
  77. sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
  78. from ruffus import *
  79. # use simplejson in place of json for python < 2.6
  80. try:
  81. import json
  82. except ImportError:
  83. import simplejson
  84. json = simplejson
  85. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  86. # Functions
  87. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  88. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  89. # Main logic
  90. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  91. # get help string
  92. f =StringIO.StringIO()
  93. parser.print_help(f)
  94. helpstr = f.getvalue()
  95. (options, remaining_args) = parser.parse_args()
  96. import time
  97. def sleep_a_while ():
  98. time.sleep(1)
  99. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  100. # Tasks
  101. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  102. #
  103. # task1
  104. #
  105. @files(None, 'a.1')
  106. def task1(infile, outfile):
  107. """
  108. First task
  109. """
  110. output_text = ""
  111. output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
  112. open(outfile, "w").write(output_text)
  113. time.sleep(1)
  114. #
  115. # task2
  116. #
  117. @transform(task1, suffix(".1"), ".2")
  118. def task2(infile, outfile):
  119. """
  120. Second task
  121. """
  122. output_text = open(infile).read() if infile else ""
  123. output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
  124. open(outfile, "w").write(output_text)
  125. time.sleep(1)
  126. #
  127. # task3
  128. #
  129. @transform(task2, suffix(".2"), ".3")
  130. def task3(infile, outfile):
  131. """
  132. Third task
  133. """
  134. output_text = open(infile).read() if infile else ""
  135. output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
  136. open(outfile, "w").write(output_text)
  137. time.sleep(1)
  138. #
  139. # task4
  140. #
  141. @transform(task3, suffix(".3"), ".4")
  142. def task4(infile, outfile):
  143. """
  144. Fourth task
  145. """
  146. output_text = open(infile).read() if infile else ""
  147. output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
  148. open(outfile, "w").write(output_text)
  149. time.sleep(1)
  150. #
  151. # Necessary to protect the "entry point" of the program under windows.
  152. # see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
  153. #
  154. if __name__ == '__main__':
  155. try:
  156. if options.just_print:
  157. pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
  158. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
  159. verbose = options.verbose)
  160. elif options.dependency_file:
  161. pipeline_printout_graph ( open(options.dependency_file, "w"),
  162. options.dependency_graph_format,
  163. options.target_tasks,
  164. options.forced_tasks,
  165. draw_vertically = not options.draw_horizontally,
  166. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
  167. no_key_legend = options.no_key_legend_in_graph)
  168. else:
  169. pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
  170. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
  171. verbose = options.verbose)
  172. except Exception, e:
  173. print e.args