/doc/static_data/example_scripts/ruffus_template.py

  1#!/usr/bin/env python
  2"""
  3
  4    ruffus_template.py
  5                        [--log_file PATH]
  6                        [--verbose]
  7                        [--target_tasks]
  8                        [--jobs]
  9                        [--just_print]
 10                        [--flowchart]
 11                        [--key_legend_in_graph]
 12                        [--forced_tasks]
 13
 14"""
 15import sys, os
 16
 17
 18#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 19
 20#   options
 21
 22
 23#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
 24
 25
 26if __name__ == '__main__':
 27    from optparse import OptionParser
 28    import StringIO
 29
 30    parser = OptionParser(version="%prog 1.0", usage = "\n\n    %progs [options]")
 31
 32
 33
 34    #
 35    #   general options: verbosity / logging
 36    #
 37    parser.add_option("-v", "--verbose", dest = "verbose",
 38                      action="count", default=0,
 39                      help="Print more verbose messages for each additional verbose level.")
 40    parser.add_option("-L", "--log_file", dest="log_file",
 41                      metavar="FILE",
 42                      type="string",
 43                      help="Name and path of log file")
 44
 45
 46
 47
 48    #
 49    #   pipeline
 50    #
 51    parser.add_option("-t", "--target_tasks", dest="target_tasks",
 52                        action="append",
 53                        default = list(),
 54                        metavar="JOBNAME",
 55                        type="string",
 56                        help="Target task(s) of pipeline.")
 57    parser.add_option("-j", "--jobs", dest="jobs",
 58                        default=1,
 59                        metavar="N",
 60                        type="int",
 61                        help="Allow N jobs (commands) to run simultaneously.")
 62    parser.add_option("-n", "--just_print", dest="just_print",
 63                        action="store_true", default=False,
 64                        help="Don't actually run any commands; just print the pipeline.")
 65    parser.add_option("--flowchart", dest="flowchart",
 66                        metavar="FILE",
 67                        type="string",
 68                        help="Don't actually run any commands; just print the pipeline "
 69                             "as a flowchart.")
 70
 71    #
 72    #   Less common pipeline options
 73    #
 74    parser.add_option("--key_legend_in_graph", dest="key_legend_in_graph",
 75                        action="store_true", default=False,
 76                        help="Print out legend and key for dependency graph.")
 77    parser.add_option("--forced_tasks", dest="forced_tasks",
 78                        action="append",
 79                        default = list(),
 80                        metavar="JOBNAME",
 81                        type="string",
 82                        help="Pipeline task(s) which will be included even if they are up to date.")
 83
 84    # get help string
 85    f =StringIO.StringIO()
 86    parser.print_help(f)
 87    helpstr = f.getvalue()
 88    (options, remaining_args) = parser.parse_args()
 89
 90
 91    #vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
 92    #                                             #
 93    #   Change this if necessary                  #
 94    #                                             #
 95    #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 96
 97    #
 98    #   Add names of mandatory options,
 99    #       strings corresponding to the "dest" parameter
100    #       in the options defined above
101    #
102    mandatory_options = [ ]
103
104    #vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
105    #                                             #
106    #   Change this if necessary                  #
107    #                                             #
108    #^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
109
110
111    def check_mandatory_options (options, mandatory_options, helpstr):
112        """
113        Check if specified mandatory options have b een defined
114        """
115        missing_options = []
116        for o in mandatory_options:
117            if not getattr(options, o):
118                missing_options.append("--" + o)
119
120        if not len(missing_options):
121            return
122
123        raise Exception("Missing mandatory parameter%s: %s.\n\n%s\n\n" %
124                        ("s" if len(missing_options) > 1 else "",
125                         ", ".join(missing_options),
126                         helpstr))
127    check_mandatory_options (options, mandatory_options, helpstr)
128
129
130#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
131
132#   imports
133
134
135#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
136
137from ruffus import *
138from ruffus.ruffus_exceptions import JobSignalledBreak
139
140#from json import dumps
141#from collections import defaultdict
142
143
144
145#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
146
147#   Functions
148
149
150#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
151
152
153#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
154
155#   Logger
156
157
158#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
159
160if __name__ == '__main__':
161    import logging
162    import logging.handlers
163
164    MESSAGE = 15
165    logging.addLevelName(MESSAGE, "MESSAGE")
166
167    def setup_std_logging (logger, log_file, verbose):
168        """
169        set up logging using programme options
170        """
171        class debug_filter(logging.Filter):
172            """
173            Ignore INFO messages
174            """
175            def filter(self, record):
176                return logging.INFO != record.levelno
177
178        class NullHandler(logging.Handler):
179            """
180            for when there is no logging
181            """
182            def emit(self, record):
183                pass
184
185        # We are interesting in all messages
186        logger.setLevel(logging.DEBUG)
187        has_handler = False
188
189        # log to file if that is specified
190        if log_file:
191            handler = logging.FileHandler(log_file, delay=False)
192            handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)6s - %(message)s"))
193            handler.setLevel(MESSAGE)
194            logger.addHandler(handler)
195            has_handler = True
196
197        # log to stderr if verbose
198        if verbose:
199            stderrhandler = logging.StreamHandler(sys.stderr)
200            stderrhandler.setFormatter(logging.Formatter("    %(message)s"))
201            stderrhandler.setLevel(logging.DEBUG)
202            if log_file:
203                stderrhandler.addFilter(debug_filter())
204            logger.addHandler(stderrhandler)
205            has_handler = True
206
207        # no logging
208        if not has_handler:
209            logger.addHandler(NullHandler())
210
211
212    #
213    #   set up log
214    #
215    logger = logging.getLogger(module_name)
216    setup_std_logging(logger, options.log_file, options.verbose)
217
218    #
219    #   Allow logging across Ruffus pipeline
220    #
221    def get_logger (logger_name, args):
222        return logger
223
224    from ruffus.proxy_logger import *
225    (logger_proxy,
226     logging_mutex) = make_shared_logger_and_proxy (get_logger,
227                                                    module_name,
228                                                    {})
229
230
231#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
232
233#   Pipeline
234
235
236#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
237
238#       Put pipeline code here
239
240
241
242
243
244
245
246#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
247
248#   Main logic
249
250
251#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888
252if __name__ == '__main__':
253    if options.just_print:
254        pipeline_printout(sys.stdout, options.target_tasks, options.forced_tasks,
255                            verbose=options.verbose)
256
257    elif options.flowchart:
258        pipeline_printout_graph (   open(options.flowchart, "w"),
259                                    # use flowchart file name extension to decide flowchart format
260                                    #   e.g. svg, jpg etc.
261                                    os.path.splitext(options.flowchart)[1][1:],
262                                    options.target_tasks,
263                                    options.forced_tasks,
264                                    no_key_legend   = not options.key_legend_in_graph)
265    else:
266        pipeline_run(options.target_tasks, options.forced_tasks,
267                            multiprocess    = options.jobs,
268                            logger          = stderr_logger,
269                            verbose         = options.verbose)
270