/ruffus/test/test_transform_inputs.py

https://code.google.com/p/ruffus/ · Python · 207 lines · 128 code · 53 blank · 26 comment · 11 complexity · ce5eabd2891c0964a0614191979395ff MD5 · raw file

  1. #!/usr/bin/env python
  2. """
  3. test_transform_with_no_re_matches.py
  4. test messages with no regular expression matches
  5. """
  6. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  7. # options
  8. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  9. from optparse import OptionParser
  10. import sys, os
  11. import os.path
  12. import StringIO
  13. import re,time
  14. # add self to search path for testing
  15. exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
  16. sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
  17. if __name__ == '__main__':
  18. module_name = os.path.split(sys.argv[0])[1]
  19. module_name = os.path.splitext(module_name)[0];
  20. else:
  21. module_name = __name__
  22. import ruffus
  23. print "\tRuffus Version = ", ruffus.__version__
  24. parser = OptionParser(version="%%prog v1.0, ruffus v%s" % ruffus.ruffus_version.__version)
  25. parser.add_option("-t", "--target_tasks", dest="target_tasks",
  26. action="append",
  27. default = list(),
  28. metavar="JOBNAME",
  29. type="string",
  30. help="Target task(s) of pipeline.")
  31. parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
  32. action="append",
  33. default = list(),
  34. metavar="JOBNAME",
  35. type="string",
  36. help="Pipeline task(s) which will be included even if they are up to date.")
  37. parser.add_option("-j", "--jobs", dest="jobs",
  38. default=1,
  39. metavar="jobs",
  40. type="int",
  41. help="Specifies the number of jobs (commands) to run simultaneously.")
  42. parser.add_option("-v", "--verbose", dest = "verbose",
  43. action="count", default=0,
  44. help="Print more verbose messages for each additional verbose level.")
  45. parser.add_option("-d", "--dependency", dest="dependency_file",
  46. #default="simple.svg",
  47. metavar="FILE",
  48. type="string",
  49. help="Print a dependency graph of the pipeline that would be executed "
  50. "to FILE, but do not execute it.")
  51. parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
  52. metavar="FORMAT",
  53. type="string",
  54. default = 'svg',
  55. help="format of dependency graph file. Can be 'ps' (PostScript), "+
  56. "'svg' 'svgz' (Structured Vector Graphics), " +
  57. "'png' 'gif' (bitmap graphics) etc ")
  58. parser.add_option("-n", "--just_print", dest="just_print",
  59. action="store_true", default=False,
  60. help="Print a description of the jobs that would be executed, "
  61. "but do not execute them.")
  62. parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
  63. action="store_true", default=False,
  64. help="Rebuild a minimum of tasks necessary for the target. "
  65. "Ignore upstream out of date tasks if intervening tasks are fine.")
  66. parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
  67. action="store_true", default=False,
  68. help="Do not print out legend and key for dependency graph.")
  69. parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
  70. action="store_true", default=False,
  71. help="Draw horizontal dependency graph.")
  72. parameters = [
  73. ]
  74. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  75. # imports
  76. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  77. import StringIO
  78. import re
  79. import operator
  80. import sys,os
  81. from collections import defaultdict
  82. import random
  83. sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
  84. from ruffus import *
  85. # use simplejson in place of json for python < 2.6
  86. try:
  87. import json
  88. except ImportError:
  89. import simplejson
  90. json = simplejson
  91. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  92. # Main logic
  93. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  94. # get help string
  95. f =StringIO.StringIO()
  96. parser.print_help(f)
  97. helpstr = f.getvalue()
  98. (options, remaining_args) = parser.parse_args()
  99. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  100. # Tasks
  101. #88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
  102. tempdir = "tempdir/"
  103. @follows(mkdir(tempdir))
  104. @files([[None, tempdir+ "a.1"], [None, tempdir+ "b.1"]])
  105. def task1(i, o):
  106. open(o, "w")
  107. @follows(mkdir(tempdir))
  108. @files([[None, tempdir+ "c.1"], [None, tempdir+ "d.1"]])
  109. def task2(i, o):
  110. open(o, "w")
  111. @transform(task1, regex(r"(.*)"), inputs(((r"\1"), task2, "test_transform_inputs.*")), r"\1.output")
  112. def task3(i, o):
  113. names = ",".join(sorted(i))
  114. for f in o:
  115. open(o, "w").write(names)
  116. @merge((task3), tempdir + "final.output")
  117. def task4(i, o):
  118. o_file = open(o, "w")
  119. for f in sorted(i):
  120. o_file.write(f +":" +open(f).read() + ";")
  121. import unittest
  122. class Test_task(unittest.TestCase):
  123. def tearDown (self):
  124. """
  125. """
  126. import glob
  127. for f in glob.glob(tempdir + "*"):
  128. os.unlink(f)
  129. os.rmdir(tempdir)
  130. def test_task (self):
  131. pipeline_run([task4], options.forced_tasks, multiprocess = options.jobs,
  132. verbose = options.verbose)
  133. correct_output = "tempdir/a.1.output:tempdir/a.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;tempdir/b.1.output:tempdir/b.1,tempdir/c.1,tempdir/d.1,test_transform_inputs.py;"
  134. real_output = open(tempdir + "final.output").read()
  135. self.assert_(correct_output == real_output)
  136. if __name__ == '__main__':
  137. if options.just_print:
  138. pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
  139. verbose = options.verbose,
  140. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
  141. elif options.dependency_file:
  142. pipeline_printout_graph ( open(options.dependency_file, "w"),
  143. options.dependency_graph_format,
  144. options.target_tasks,
  145. options.forced_tasks,
  146. draw_vertically = not options.draw_horizontally,
  147. gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
  148. no_key_legend = options.no_key_legend_in_graph)
  149. else:
  150. sys.argv= sys.argv[0:1]
  151. unittest.main()