/ruffus/test/auto_generated_pipeline_examples/simpler.py

https://code.google.com/p/ruffus/ · Python · 270 lines · 137 code · 73 blank · 60 comment · 14 complexity · d2be36cf608adbd7c7faca6a69d1acf6 MD5 · raw file

  1. #!/usr/bin/env python2.5
  2. """
  3. test_tasks.py
  4. """
  5. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  6. # options
  7. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  8. from optparse import OptionParser
  9. import sys, os
  10. import os.path
  11. import StringIO
  12. # add self to search path for testing
  13. exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
  14. if __name__ == '__main__':
  15. module_name = os.path.split(sys.argv[0])[1]
  16. module_name = os.path.splitext(module_name)[0];
  17. else:
  18. module_name = __name__
  19. # graph, task etc are one directory down
  20. if __name__ == '__main__':
  21. sys.path.append("/net/cpp-group/Leo/inprogress/pipeline/installation/src/ruffus")
  22. parser = OptionParser(version="%prog 1.0")
  23. parser.add_option("-t", "--target_tasks", dest="target_tasks",
  24. action="append",
  25. default = list(),
  26. metavar="JOBNAME",
  27. type="string",
  28. help="Target task(s) of pipeline.")
  29. parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
  30. action="append",
  31. default = list(),
  32. metavar="JOBNAME",
  33. type="string",
  34. help="Pipeline task(s) which will be included even if they are up to date.")
  35. parser.add_option("-j", "--jobs", dest="jobs",
  36. default=5,
  37. metavar="jobs",
  38. type="int",
  39. help="Specifies the number of jobs (commands) to run simultaneously.")
  40. parser.add_option("-v", "--verbose", dest = "verbose",
  41. action="count", default=0,
  42. help="Print more verbose messages for each additional verbose level.")
  43. parser.add_option("-d", "--dependency", dest="dependency_file",
  44. default="simple.svg",
  45. metavar="FILE",
  46. type="string",
  47. help="Print a dependency graph of the pipeline that would be executed "
  48. "to FILE, but do not execute it.")
  49. parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
  50. metavar="FORMAT",
  51. type="string",
  52. default = 'svg',
  53. help="format of dependency graph file. Can be 'ps' (PostScript), "+
  54. "'svg' 'svgz' (Structured Vector Graphics), " +
  55. "'png' 'gif' (bitmap graphics) etc ")
  56. parser.add_option("-n", "--just_print", dest="just_print",
  57. action="store_true", default=False,
  58. help="Print a description of the jobs that would be executed, "
  59. "but do not execute them.")
  60. parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
  61. action="store_true", default=False,
  62. help="Rebuild a minimum of tasks necessary for the target. "
  63. "Ignore upstream out of date tasks if intervening tasks are fine.")
  64. parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
  65. action="store_true", default=False,
  66. help="Do not print out legend and key for dependency graph.")
  67. parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
  68. action="store_true", default=False,
  69. help="Draw horizontal dependency graph.")
  70. parameters = [
  71. ]
  72. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  73. # imports
  74. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  75. import StringIO
  76. import re
  77. import operator
  78. import sys
  79. from collections import defaultdict
  80. from graph import *
  81. from task import *
  82. import task
  83. from print_dependencies import *
  84. # use simplejson in place of json for python < 2.6
  85. try:
  86. import json
  87. except ImportError:
  88. import simplejson
  89. json = simplejson
  90. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  91. # Functions
  92. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  93. def create_custom_file_func(params):
  94. """
  95. creates function which can be used as input to @files_func
  96. """
  97. def cust_func ():
  98. for job_param in params:
  99. yield job_param
  100. return cust_func
  101. def is_job_uptodate (infiles, outfiles, *extra_params):
  102. """
  103. assumes first two parameters are files, checks if they are up to date
  104. """
  105. return task.needs_update_check_modify_time (infiles, outfiles, *extra_params)
  106. def test_post_task_function ():
  107. print "Hooray"
  108. import time
  109. def test_job_io(infiles, outfiles, extra_params):
  110. """
  111. cat input files content to output files
  112. after writing out job parameters
  113. """
  114. # dump parameters
  115. params = (infiles, outfiles) + extra_params
  116. sys.stdout.write(' job = %s\n' % json.dumps(params))
  117. if isinstance(infiles, str):
  118. infiles = [infiles]
  119. elif infiles == None:
  120. infiles = []
  121. if isinstance(outfiles, str):
  122. outfiles = [outfiles]
  123. output_text = list()
  124. for f in infiles:
  125. output_text.append(open(f).read())
  126. output_text = "".join(sorted(output_text))
  127. output_text += json.dumps(infiles) + " -> " + json.dumps(outfiles) + "\n"
  128. for f in outfiles:
  129. open(f, "w").write(output_text)
  130. time.sleep(1)
  131. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  132. # Main logic
  133. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  134. # get help string
  135. f =StringIO.StringIO()
  136. parser.print_help(f)
  137. helpstr = f.getvalue()
  138. (options, remaining_args) = parser.parse_args()
  139. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  140. # Tasks
  141. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  142. #
  143. # task1
  144. #
  145. @files(None, 'a.1')
  146. def task1(infiles, outfiles, *extra_params):
  147. """
  148. First task
  149. """
  150. test_job_io(infiles, outfiles, extra_params)
  151. #
  152. # task2
  153. #
  154. @files_re('*.1', '(.*).1', r'\1.1', r'\1.2')
  155. @follows(task1)
  156. def task2(infiles, outfiles, *extra_params):
  157. """
  158. Second task
  159. """
  160. test_job_io(infiles, outfiles, extra_params)
  161. #
  162. # task3
  163. #
  164. @files_re('*.1', '(.*).1', r'\1.2', r'\1.3')
  165. @follows(task2)
  166. def task3(infiles, outfiles, *extra_params):
  167. """
  168. Third task
  169. """
  170. test_job_io(infiles, outfiles, extra_params)
  171. #
  172. # task4
  173. #
  174. @files_re('*.1', '(.*).1', r'\1.3', r'\1.4')
  175. @follows(task3)
  176. def task4(infiles, outfiles, *extra_params):
  177. """
  178. Fourth task
  179. """
  180. test_job_io(infiles, outfiles, extra_params)
  181. if options.just_print:
  182. pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
  183. long_winded=True,
  184. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
  185. elif options.dependency_file:
  186. pipeline_printout_graph ( open(options.dependency_file, "w"),
  187. options.dependency_graph_format,
  188. options.target_tasks,
  189. options.forced_tasks,
  190. draw_vertically = not options.draw_horizontally,
  191. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
  192. no_key_legend = options.no_key_legend_in_graph)
  193. else:
  194. pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs,
  195. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)