/ruffus/test/auto_generated_pipeline_examples/simpler.py

  1#!/usr/bin/env python2.5
  2"""
  3
  4    test_tasks.py
  5
  6"""
  7
  8
  9#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 10
 11#   options        
 12
 13
 14#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 15
 16from optparse import OptionParser
 17import sys, os
 18import os.path
 19import StringIO
 20
 21# add self to search path for testing
 22exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0]
 23if __name__ == '__main__':
 24    module_name = os.path.split(sys.argv[0])[1]
 25    module_name = os.path.splitext(module_name)[0];
 26else:
 27    module_name = __name__
 28
 29# graph, task etc are one directory down
 30if __name__ == '__main__':
 31    sys.path.append("/net/cpp-group/Leo/inprogress/pipeline/installation/src/ruffus")
 32
 33
 34
 35parser = OptionParser(version="%prog 1.0")
 36parser.add_option("-t", "--target_tasks", dest="target_tasks",
 37                  action="append",
 38                  default = list(),
 39                  metavar="JOBNAME", 
 40                  type="string",
 41                  help="Target task(s) of pipeline.")
 42parser.add_option("-f", "--forced_tasks", dest="forced_tasks",
 43                  action="append",
 44                  default = list(),
 45                  metavar="JOBNAME", 
 46                  type="string",
 47                  help="Pipeline task(s) which will be included even if they are up to date.")
 48parser.add_option("-j", "--jobs", dest="jobs",
 49                  default=5,
 50                  metavar="jobs", 
 51                  type="int",
 52                  help="Specifies  the number of jobs (commands) to run simultaneously.")
 53parser.add_option("-v", "--verbose", dest = "verbose",
 54                  action="count", default=0,
 55                  help="Print more verbose messages for each additional verbose level.")
 56parser.add_option("-d", "--dependency", dest="dependency_file",
 57                  default="simple.svg",
 58                  metavar="FILE", 
 59                  type="string",
 60                  help="Print a dependency graph of the pipeline that would be executed "
 61                        "to FILE, but do not execute it.")
 62parser.add_option("-F", "--dependency_graph_format", dest="dependency_graph_format",
 63                  metavar="FORMAT", 
 64                  type="string",
 65                  default = 'svg',
 66                  help="format of dependency graph file. Can be 'ps' (PostScript), "+
 67                  "'svg' 'svgz' (Structured Vector Graphics), " +
 68                  "'png' 'gif' (bitmap  graphics) etc ")
 69parser.add_option("-n", "--just_print", dest="just_print",
 70                    action="store_true", default=False,
 71                    help="Print a description of the jobs that would be executed, "
 72                        "but do not execute them.")
 73parser.add_option("-M", "--minimal_rebuild_mode", dest="minimal_rebuild_mode",
 74                    action="store_true", default=False,
 75                    help="Rebuild a minimum of tasks necessary for the target. "
 76                    "Ignore upstream out of date tasks if intervening tasks are fine.")
 77parser.add_option("-K", "--no_key_legend_in_graph", dest="no_key_legend_in_graph",
 78                    action="store_true", default=False,
 79                    help="Do not print out legend and key for dependency graph.")
 80parser.add_option("-H", "--draw_graph_horizontally", dest="draw_horizontally",
 81                    action="store_true", default=False,
 82                    help="Draw horizontal dependency graph.")
 83
 84parameters = [  
 85                ]
 86
 87
 88
 89
 90
 91
 92
 93#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 94
 95#   imports        
 96
 97
 98#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 99
100import StringIO
101import re
102import operator
103import sys
104from collections import defaultdict
105
106from graph import *
107from task import *
108import task
109from print_dependencies import *
110# use simplejson in place of json for python < 2.6
111try:
112    import json
113except ImportError:
114    import simplejson
115    json = simplejson
116
117#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
118
119#   Functions
120
121
122#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
123
124def create_custom_file_func(params):
125    """
126    creates function which can be used as input to @files_func
127    """
128    def cust_func ():
129        for job_param in params:
130            yield job_param
131    return cust_func
132    
133    
134def is_job_uptodate (infiles, outfiles, *extra_params):
135    """
136    assumes first two parameters are files, checks if they are up to date
137    """
138    return task.needs_update_check_modify_time (infiles, outfiles, *extra_params)
139    
140    
141    
142def test_post_task_function ():
143    print "Hooray"
144
145import time
146def test_job_io(infiles, outfiles, extra_params):
147    """
148    cat input files content to output files
149        after writing out job parameters
150    """
151    # dump parameters
152    params = (infiles, outfiles) + extra_params
153    sys.stdout.write('    job = %s\n' % json.dumps(params))
154
155    
156        
157    if isinstance(infiles, str):
158        infiles = [infiles]
159    elif infiles == None:
160        infiles = []
161    if isinstance(outfiles, str):
162        outfiles = [outfiles]
163    output_text = list()
164    for f in infiles:
165        output_text.append(open(f).read())
166    output_text = "".join(sorted(output_text))
167    output_text += json.dumps(infiles) + " -> " + json.dumps(outfiles) + "\n"
168    for f in outfiles:
169        open(f, "w").write(output_text)
170    time.sleep(1)
171
172
173#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
174
175#   Main logic
176
177
178#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
179
180
181
182
183
184# get help string
185f =StringIO.StringIO()
186parser.print_help(f)
187helpstr = f.getvalue()
188(options, remaining_args) = parser.parse_args()
189
190
191
192
193
194#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
195
196#   Tasks
197
198
199#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
200
201
202#
203#    task1
204#
205@files(None, 'a.1')
206def task1(infiles, outfiles, *extra_params):
207    """
208    First task
209    """
210    test_job_io(infiles, outfiles, extra_params)
211
212
213
214#
215#    task2
216#
217@files_re('*.1', '(.*).1', r'\1.1', r'\1.2')
218@follows(task1)
219def task2(infiles, outfiles, *extra_params):
220    """
221    Second task
222    """
223    test_job_io(infiles, outfiles, extra_params)
224
225
226
227#
228#    task3
229#
230@files_re('*.1', '(.*).1', r'\1.2', r'\1.3')
231@follows(task2)
232def task3(infiles, outfiles, *extra_params):
233    """
234    Third task
235    """
236    test_job_io(infiles, outfiles, extra_params)
237
238
239
240#
241#    task4
242#
243@files_re('*.1', '(.*).1', r'\1.3', r'\1.4')
244@follows(task3)
245def task4(infiles, outfiles, *extra_params):
246    """
247    Fourth task
248    """
249    test_job_io(infiles, outfiles, extra_params)
250
251
252
253        
254if options.just_print:
255    pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks, 
256                        long_winded=True, 
257                        gnu_make_maximal_rebuild_mode = not options.minimal_rebuild_mode)
258
259elif options.dependency_file:
260    pipeline_printout_graph (     open(options.dependency_file, "w"),
261                         options.dependency_graph_format,
262                         options.target_tasks, 
263                         options.forced_tasks,
264                         draw_vertically = not options.draw_horizontally,
265                         gnu_make_maximal_rebuild_mode  = not options.minimal_rebuild_mode,
266                         no_key_legend  = options.no_key_legend_in_graph)
267else:    
268    pipeline_run(options.target_tasks, options.forced_tasks, multiprocess = options.jobs, 
269                    gnu_make_maximal_rebuild_mode  = not options.minimal_rebuild_mode)
270