/ruffus/test/simpler_at_runtime.py

  1#!/usr/bin/env python
  2"""
  3
  4    test_tasks.py
  5
  6"""
  7
  8
  9#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 10
 11#   options        
 12
 13
 14#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 15
 16from optparse import OptionParser
 17import sys, os
 18import os.path
 19import StringIO
 20
 21# add self to search path for testing
 22exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
 23sys.path.insert(0,os.path.abspath(os.path.join(exe_path,"..", "..")))
 24if __name__ == '__main__':
 25    module_name = os.path.split(sys.argv[0])[1]
 26    module_name = os.path.splitext(module_name)[0];
 27else:
 28    module_name = __name__
 29
 30
 31
 32
 33parser = OptionParser(version="%prog 1.0")
 34parser.add_option("-t", "--target_tasks", dest="target_tasks",
 35                  action="append",
 36                  default = list(),
 37                  metavar="JOBNAME", 
 38                  type="string",
 39                  help="Target task(s) of pipeline.")
 40parser.add_option("-R", "--runtime_files", dest="runtime_files",
 41                  action="append",
 42                  default = ["a.3"],
 43                  metavar="JOBNAME", 
 44                  type="string",
 45                  help="Target task(s) of pipeline.")
 46parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
 47                  action="append",
 48                  default = list(),
 49                  metavar="JOBNAME", 
 50                  type="string",
 51                  help="Pipeline task(s) which will be included even if they are up to date.")
 52parser.add_option("-j", "--jobs", dest="jobs",
 53                  default=1,
 54                  metavar="jobs", 
 55                  type="int",
 56                  help="Specifies  the number of jobs (commands) to run simultaneously.")
 57parser.add_option("-v", "--verbose", dest = "verbose",
 58                  action="count", default=0,
 59                  help="Print more verbose messages for each additional verbose level.")
 60parser.add_option("-d", "--debug", dest = "debug",
 61                  action="count", default=0,
 62                  help="Cleanup afterwards.")
 63parser.add_option("--dependency", dest="dependency_file",
 64                  metavar="FILE", 
 65                  type="string",
 66                  help="Print a dependency graph of the pipeline that would be executed "
 67                        "to FILE, but do not execute it.")
 68parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
 69                  metavar="FORMAT", 
 70                  type="string",
 71                  default = 'svg',
 72                  help="format of dependency graph file. Can be 'ps' (PostScript), "+
 73                  "'svg' 'svgz' (Structured Vector Graphics), " +
 74                  "'png' 'gif' (bitmap  graphics) etc ")
 75parser.add_option("-n", "--just_print", dest="just_print",
 76                    action="store_true", default=False,
 77                    help="Print a description of the jobs that would be executed, "
 78                        "but do not execute them.")
 79parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
 80                    action="store_true", default=False,
 81                    help="Rebuild a minimum of tasks necessary for the target. "
 82                    "Ignore upstream out of date tasks if intervening tasks are fine.")
 83parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
 84                    action="store_true", default=False,
 85                    help="Do not print out legend and key for dependency graph.")
 86parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
 87                    action="store_true", default=False,
 88                    help="Draw horizontal dependency graph.")
 89
 90parameters = [  
 91                ]
 92
 93
 94
 95
 96
 97
 98
 99#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
100
101#   imports        
102
103
104#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
105
106import StringIO
107import re
108import operator
109import sys,os
110from collections import defaultdict
111
112sys.path.append(os.path.abspath(os.path.join(exe_path,"..", "..")))
113from ruffus import *
114
115# use simplejson in place of json for python < 2.6
116try:
117    import json
118except ImportError:
119    import simplejson
120    json = simplejson
121
122#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
123
124#   Functions
125
126
127#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
128
129
130#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
131
132#   Main logic
133
134
135#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
136
137
138
139
140
141# get help string
142f =StringIO.StringIO()
143parser.print_help(f)
144helpstr = f.getvalue()
145(options, remaining_args) = parser.parse_args()
146
147
148
149
150
151#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
152
153#   Tasks
154
155
156#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
157
158
159#
160#    task1
161#
162@split(None, ['a.1'] + options.runtime_files)
163def task1(infile, outfiles):
164    """
165    First task
166    """
167    output_text  = ""
168    output_text += json.dumps(infile) + " -> " + json.dumps(outfiles) + "\n"
169    for outfile in outfiles:
170        open(outfile, "w").write(output_text)
171
172
173
174#
175#    task2
176#
177@transform(task1, suffix(".1"), ".2")
178def task2(infile, outfile):
179    """
180    Second task
181    """
182    output_text  = open(infile).read() if infile else ""
183    output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
184    open(outfile, "w").write(output_text)
185
186
187
188#
189#    task3
190#
191@transform(task2, suffix(".2"), ".3")
192def task3(infile, outfile):
193    """
194    Third task
195    """
196    output_text  = open(infile).read() if infile else ""
197    output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
198    open(outfile, "w").write(output_text)
199
200
201
202#
203#    task4
204#
205@follows(task3)
206@transform(runtime_parameter("a"), suffix(".3"), ".4")
207def task4(infile, outfile):
208    """
209    Fourth task
210    """
211    output_text  = open(infile).read() if infile else ""
212    output_text += json.dumps(infile) + " -> " + json.dumps(outfile) + "\n"
213    open(outfile, "w").write(output_text)
214
215# 
216#   Necessary to protect the "entry point" of the program under windows.
217#       see: http://docs.python.org/library/multiprocessing.html#multiprocessing-programming
218#
219if __name__ == '__main__':
220        if options.just_print:
221            pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks, 
222                                gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode,
223                            verbose = options.verbose, runtime_data = {"a": options.runtime_files})
224        
225        elif options.dependency_file:
226            pipeline_printout_graph (     open(options.dependency_file, "w"),
227                                 options.dependency_graph_format,
228                                 options.target_tasks, 
229                                 options.forced_tasks,
230                                 draw_vertically = not options.draw_horizontally,
231                                 gnu_make_maximal_rebuild_mode  = not options.minimal_rebuild_mode,
232                                 no_key_legend  = options.no_key_legend_in_graph)
233        elif options.debug:
234            import os
235            for f in ["a.1", "a.2","a.3","a.4"]:
236                if os.path.exists(f):
237                    os.unlink(f)
238            pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs, 
239                            gnu_make_maximal_rebuild_mode  = not options.minimal_rebuild_mode,
240                            verbose = options.verbose, runtime_data = {"a": options.runtime_files})
241            for f in ["a.1", "a.2","a.3","a.4"]:
242                if os.path.exists(f):
243                    os.unlink(f)
244                else:
245                    raise Exception("%s is missing" % f)
246            print "OK"
247        else:    
248            pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs, 
249                            gnu_make_maximal_rebuild_mode  = not options.minimal_rebuild_mode,
250                            verbose = options.verbose, runtime_data = {"a": options.runtime_files})