/ruffus/test/test_ruffus_utility.py
Python | 509 lines | 240 code | 112 blank | 157 comment | 2 complexity | 2b9b4e9bef0ac98ba33db1635b1097f4 MD5 | raw file
1#!/usr/bin/env python 2################################################################################ 3# 4# test_ruffus_utility.py 5# 6# 7# Copyright (c) 2009 Leo Goodstadt 8# 9# Permission is hereby granted, free of charge, to any person obtaining a copy 10# of this software and associated documentation files (the "Software"), to deal 11# in the Software without restriction, including without limitation the rights 12# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13# copies of the Software, and to permit persons to whom the Software is 14# furnished to do so, subject to the following conditions: 15# 16# The above copyright notice and this permission notice shall be included in 17# all copies or substantial portions of the Software. 18# 19# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 25# THE SOFTWARE. 26################################################################################# 27 28""" 29 test_ruffus_utility.py 30""" 31 32# use simplejson in place of json for python < 2.6 33try: 34 import json 35except ImportError: 36 import simplejson 37 json = simplejson 38import unittest, os,sys 39if __name__ != '__main__': 40 raise Exception ("This is not a callable module [%s]" % __main__) 41 42 43exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0] 44sys.path.insert(0, os.path.abspath(os.path.join(exe_path,"..", ".."))) 45from ruffus import * 46from ruffus.ruffus_utility import * 47 48 49os.chdir(exe_path) 50 51import unittest, time 52 53#_________________________________________________________________________________________ 54 55# get_nested_tasks_or_globs 56 57#_________________________________________________________________________________________ 58class Test_get_nested_tasks_or_globs(unittest.TestCase): 59 def setUp(self): 60 exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0] 61 os.chdir(exe_path) 62 63 # self.assertEqual(self.seq, range(10)) 64 # self.assert_(element in self.seq) 65 # self.assertRaises(ValueError, random.sample, self.seq, 20) 66 67 def check_equal (self, a,b): 68 self.assertEqual(get_nested_tasks_or_globs(a), b) 69 70 def test_get_nested_tasks_or_globs(self): 71 72 # 73 # test strings 74 # 75 self.check_equal("test", (set(), set(), set())) 76 self.check_equal([("test1",), "test2", 3], (set(), set(), set())) 77 78 # 79 # test missing 80 # 81 self.check_equal((1,3, [5]), (set(), set(), set())) 82 self.check_equal(None, (set(), set(), set())) 83 84 # 85 # test glob 86 # 87 self.check_equal([("test1.*",), "test?2", 3], (set(), set(['test1.*', 'test?2']), set())) 88 89 # 90 # test glob and string 91 # 92 self.check_equal([("test*1",), (("test3",),),"test2", 3], (set(), set(['test*1']), set())) 93 94 # 95 # test task function 96 # 97 self.check_equal(is_glob, (set([is_glob]), set([]), set())) 98 self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]], ( 99 set([is_glob, get_strings_in_nested_sequence]), set(["that*"]), set())) 100 # 101 # test wrapper 102 # 103 self.check_equal(output_from(is_glob, ["what", 7], 5), (set([is_glob, "what"]), set([]), set())) 104 105#_________________________________________________________________________________________ 106 107# replace_func_names_with_tasks 108 109#_________________________________________________________________________________________ 110class Test_replace_func_names_with_tasks(unittest.TestCase): 111 def setUp(self): 112 exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0] 113 os.chdir(exe_path) 114 115 # self.assertEqual(self.seq, range(10)) 116 # self.assert_(element in self.seq) 117 # self.assertRaises(ValueError, random.sample, self.seq, 20) 118 119 def check_equal (self, a,b, d): 120 self.assertEqual(replace_func_names_with_tasks(a, d), b) 121 122 def test_replace_func_names_with_tasks(self): 123 func_or_name_to_task = {is_glob: "FF is_glob", "what" : "FF what", get_strings_in_nested_sequence: "FF get_strings_in_nested_sequence"} 124 125 126 # 127 # test strings 128 # 129 self.check_equal("test", "test", func_or_name_to_task) 130 self.check_equal( [("test1",), "test2", 3], 131 [("test1",), "test2", 3], 132 func_or_name_to_task) 133 134 # 135 # test missing 136 # 137 self.check_equal((1,3, [5]), (1,3, [5]), func_or_name_to_task) 138 self.check_equal(None, None, func_or_name_to_task) 139 140 141 142 # 143 # test task function 144 # 145 self.check_equal(is_glob, "FF is_glob", func_or_name_to_task) 146 self.check_equal([is_glob, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]], 147 ["FF is_glob", [1, "this", ["that*", 5]], [("FF get_strings_in_nested_sequence",)]], 148 func_or_name_to_task) 149 # 150 # test wrapper 151 # 152 self.check_equal(output_from(is_glob, ["what", 7], 5), 153 ["FF is_glob", ["FF what", 7], 5], 154 func_or_name_to_task) 155 self.check_equal(output_from(is_glob), 156 "FF is_glob", 157 func_or_name_to_task) 158 159 self.check_equal([1, output_from(is_glob, ["what", 7], 5)], 160 [1, "FF is_glob", ["FF what", 7], 5], 161 func_or_name_to_task) 162 163 self.check_equal([1, output_from(is_glob), ["what", 7], 5], 164 [1, "FF is_glob", ["what", 7], 5], 165 func_or_name_to_task) 166 167 168 169#_________________________________________________________________________________________ 170 171# non_str_sequence 172 173#_________________________________________________________________________________________ 174class Test_non_str_sequence(unittest.TestCase): 175 176 def test_non_str_sequence (self): 177 """ 178 non_str_sequence() 179 """ 180 test_str1 = "asfas" 181 class inherited_str (str): 182 # 183 # use __new__ instead of init because str is immutable 184 # 185 def __new__( cls, a): 186 obj = super( inherited_str, cls).__new__( inherited_str, a ) 187 return obj 188 189 test_str2 = inherited_str("test") 190 class inherited_list (list): 191 def __init__ (self, *param): 192 list.__init__(self, *param) 193 test_str3 = list(test_str1) 194 test_str4 = inherited_list(test_str2) 195 self.assert_(not non_str_sequence(test_str1)) 196 self.assert_(not non_str_sequence(test_str2)) 197 self.assert_(non_str_sequence(test_str3)) 198 self.assert_(non_str_sequence(test_str4)) 199 200#_________________________________________________________________________________________ 201 202# get_strings_in_nested_sequence 203 204#_________________________________________________________________________________________ 205class Test_get_strings_in_nested_sequence(unittest.TestCase): 206 207 def test_get_strings_in_nested_sequence (self): 208 """ 209 get_strings_in_nested_sequence() 210 """ 211 class inherited_str (str): 212 # 213 # use __new__ instead of init because str is immutable 214 # 215 def __new__( cls, a): 216 obj = super( inherited_str, cls).__new__( inherited_str, a ) 217 return obj 218 219 class inherited_list (list): 220 def __init__ (self, *param): 221 list.__init__(self, *param) 222 223 self.assertEqual(get_strings_in_nested_sequence("one"), ["one"]) 224 self.assertEqual(get_strings_in_nested_sequence(["one", "two"]), ["one", "two"]) 225 self.assertEqual(get_strings_in_nested_sequence(["one", 1, "two"]), ["one", "two"]) 226 self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"]]]), ["one", "two"]) 227 self.assertEqual(get_strings_in_nested_sequence([inherited_str("one"), [1, ["two"]]]), [inherited_str("one"), "two"]) 228 self.assertEqual(get_strings_in_nested_sequence(inherited_list([inherited_str("one"), [1, ["two"]]])), 229 inherited_list([inherited_str("one"), "two"])) 230 231 232#_________________________________________________________________________________________ 233 234# get_first_strings_in_nested_sequence 235 236#_________________________________________________________________________________________ 237class Test_get_first_strings_in_nested_sequence(unittest.TestCase): 238 239 def test_get_first_strings_in_nested_sequence (self): 240 """ 241 get_first_strings_in_nested_sequence() 242 """ 243 class inherited_str (str): 244 # 245 # use __new__ instead of init because str is immutable 246 # 247 def __new__( cls, a): 248 obj = super( inherited_str, cls).__new__( inherited_str, a ) 249 return obj 250 251 class inherited_list (list): 252 def __init__ (self, *param): 253 list.__init__(self, *param) 254 255 self.assertEqual(get_strings_in_nested_sequence("one", True), ["one"]) 256 self.assertEqual(get_strings_in_nested_sequence(["one", "two"], True), ["one", "two"]) 257 self.assertEqual(get_strings_in_nested_sequence(["one", 1, "two"], True), ["one", "two"]) 258 self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"]]], True), ["one", "two"]) 259 self.assertEqual(get_strings_in_nested_sequence([inherited_str("one"), [1, ["two"]]], True), [inherited_str("one"), "two"]) 260 self.assertEqual(get_strings_in_nested_sequence(inherited_list([inherited_str("one"), [1, ["two"]]]), True), 261 inherited_list([inherited_str("one"), "two"])) 262 self.assertEqual(get_strings_in_nested_sequence(["one", [1, ["two"], "three"]], True), ["one", "two"]) 263 d = {"four" :4} 264 self.assertEqual(get_strings_in_nested_sequence(["one", [1, [d, "two"], "three"]], True), ["one", "two"]) 265 self.assertEqual(get_strings_in_nested_sequence(None, True), []) 266 self.assertEqual(get_strings_in_nested_sequence([], True), []) 267 self.assertEqual(get_strings_in_nested_sequence([1,2,3, d], True), []) 268 269 270#_________________________________________________________________________________________ 271 272# Test_compile_regex 273 274#_________________________________________________________________________________________ 275class Test_compile_regex (unittest.TestCase): 276 def test_compile_regex (self): 277 compile_regex("Dummy Task", regex(".*"), Exception, "test1") 278 279 # bad regex 280 self.assertRaises(Exception, compile_regex, "Dummy Task", regex(".*)"), Exception, "test1") 281 try: 282 compile_regex("Dummy Task", regex(".*)"), Exception, "test1") 283 except Exception, e: 284 self.assertEqual(e.args, ('Dummy Task', "test1: regular expression regex('.*)') is malformed\n[sre_constants.error: (unbalanced parenthesis)]")) 285 286 # bad number of items regex 287 self.assertRaises(Exception, compile_regex, "Dummy Task", regex(".*", "o"), Exception, "test1") 288 try: 289 compile_regex("Dummy Task", regex(".*", "o"), Exception, "test1") 290 except Exception, e: 291 self.assertEqual(e.args, ('Dummy Task', "test1: regex('('.*', 'o')') is malformed\nregex(...) should only be used to wrap a single regular expression string")) 292 293 # 0 number of items regex 294 self.assertRaises(Exception, compile_regex, "Dummy Task", regex(), Exception, "test1") 295 try: 296 compile_regex("Dummy Task", regex(), Exception, "test1") 297 except Exception, e: 298 self.assertEqual(e.args, ('Dummy Task', 'test1: regex() is malformed\nregex(...) should be used to wrap a regular expression string')) 299 300 # bad number of items suffix 301 self.assertRaises(Exception, compile_suffix, "Dummy Task", suffix(".*", "o"), Exception, "test1") 302 try: 303 compile_suffix("Dummy Task", suffix(".*", "o"), Exception, "test1") 304 except Exception, e: 305 self.assertEqual(e.args, ('Dummy Task', "test1: suffix('('.*', 'o')') is malformed.\nsuffix(...) should only be used to wrap a single string matching the suffices of file names")) 306 307 # 0 number of items suffix 308 self.assertRaises(Exception, compile_suffix, "Dummy Task", suffix(), Exception, "test1") 309 try: 310 compile_suffix("Dummy Task", suffix(), Exception, "test1") 311 except Exception, e: 312 self.assertEqual(e.args, ('Dummy Task', 'test1: suffix() is malformed.\nsuffix(...) should be used to wrap a string matching the suffices of file names')) 313 314#_________________________________________________________________________________________ 315 316# Test_check_files_io_parameters 317 318#_________________________________________________________________________________________ 319class Test_check_files_io_parameters (unittest.TestCase): 320 def test_check_files_io_parameters(self): 321 322 323 class t_fake_task(object): 324 def __init__ (self): 325 self._action_type = None 326 self._name = "fake task" 327 fake_task = t_fake_task() 328 329 single_job_params = [["input", "output"]] 330 multiple_job_params = [["input1", "output1"], ["input2", "output2"]] 331 332 check_files_io_parameters (fake_task, single_job_params, error_task_files) 333 check_files_io_parameters (fake_task, multiple_job_params, error_task_files) 334 335 336 #Bad format 337 bad_single_job_params = ["input", "output"] 338 self.assertRaises(error_task_files, check_files_io_parameters, fake_task, bad_single_job_params, error_task_files) 339 340 #Missing output files for job 341 bad_multiple_job_params = [["input1", "output1"], ["input2"]] 342 self.assertRaises(error_task_files, check_files_io_parameters, fake_task, bad_multiple_job_params, error_task_files) 343 344 #Missing input files for job 345 bad_multiple_job_params = [["input1", "output1"], []] 346 self.assertRaises(error_task_files, check_files_io_parameters, fake_task, bad_multiple_job_params, error_task_files) 347 348 #Input or output file parameters should contain at least one or more file names strings 349 bad_multiple_job_params = [[1, 2]] 350 self.assertRaises(error_task_files, check_files_io_parameters, fake_task, bad_multiple_job_params, error_task_files) 351 352#_________________________________________________________________________________________ 353 354# Test_get_first_string_in_nested_sequence 355 356#_________________________________________________________________________________________ 357class Test_get_first_string_in_nested_sequence (unittest.TestCase): 358 def test_get_first_string_in_nested_sequence(self): 359 360 self.assertEqual(get_first_string_in_nested_sequence("a") , "a") 361 self.assertEqual(get_first_string_in_nested_sequence(None) , None) 362 self.assertEqual(get_first_string_in_nested_sequence(1) , None) 363 self.assertEqual(get_first_string_in_nested_sequence((1,2)) , None) 364 self.assertEqual(get_first_string_in_nested_sequence((1,2, "a")) , "a") 365 self.assertEqual(get_first_string_in_nested_sequence((1,2, "a")) , "a") 366 self.assertEqual(get_first_string_in_nested_sequence((1,[2,"b"], "a")) , "b") 367 self.assertEqual(get_first_string_in_nested_sequence((1,set([2,"b"]), "a")) , "b") 368 369#_________________________________________________________________________________________ 370 371# Test_check_parallel_parameters 372 373#_________________________________________________________________________________________ 374class Test_check_parallel_parameters (unittest.TestCase): 375 def test_check_parallel_parameters(self): 376 377 378 class t_fake_task(object): 379 def __init__ (self): 380 self._action_type = None 381 self._name = "fake task" 382 fake_task = t_fake_task() 383 384 single_job_params = [["input", "output"]] 385 multiple_job_params = [["input1", "output1"], ["input2", "output2"]] 386 387 check_parallel_parameters (fake_task, single_job_params, error_task_files) 388 check_parallel_parameters (fake_task, multiple_job_params, error_task_files) 389 390 391 #Bad format 392 bad_single_job_params = ["input", "output"] 393 self.assertRaises(error_task_parallel, check_parallel_parameters, fake_task, bad_single_job_params, error_task_parallel) 394 395#_________________________________________________________________________________________ 396 397# expand_nested_tasks_or_globs 398 399#_________________________________________________________________________________________ 400class Test_expand_nested_tasks_or_globs(unittest.TestCase): 401 def setUp(self): 402 exe_path = os.path.split(os.path.abspath(sys.argv[0]))[0] 403 os.chdir(exe_path) 404 t1 = task._task("module", "func1"); 405 t2 = task._task("module", "func2"); 406 t3 = task._task("module", "func3"); 407 self.tasks = [t1, t2, t3] 408 409 # self.assertEqual(self.seq, range(10)) 410 # self.assert_(element in self.seq) 411 # self.assertRaises(ValueError, random.sample, self.seq, 20) 412 413 def check_equal (self, a,b): 414 415 tasks, globs, runtime_data_names = get_nested_tasks_or_globs(a) 416 func_or_name_to_task = dict(zip((non_str_sequence, get_strings_in_nested_sequence, "what"), self.tasks)) 417 418 task_or_glob_to_files = dict() 419 #for f in func_or_name_to_task: 420 # print f, task_or_glob_to_files[func_or_name_to_task[f]] 421 422 task_or_glob_to_files[self.tasks[0] ] = ["t1a", "t1b"] # non_str_sequence 423 task_or_glob_to_files[self.tasks[1] ] = ["t2"] # get_strings_in_nested_sequence 424 task_or_glob_to_files[self.tasks[2] ] = ["t3"] # "what" 425 task_or_glob_to_files["that*" ] = ["that1", "that2"] 426 task_or_glob_to_files["test*1" ] = ["test11","test21"] 427 task_or_glob_to_files["test1.*"] = ["test1.1", "test1.2"] 428 task_or_glob_to_files["test?2" ] = ["test12"] 429 430 431 param_a = replace_func_names_with_tasks(a, func_or_name_to_task) 432 self.assertEqual(expand_nested_tasks_or_globs(param_a, task_or_glob_to_files), b) 433 434 def test_expand_nested_tasks_or_globs(self): 435 436 # 437 # test strings 438 # 439 self.check_equal("test", "test") 440 self.check_equal([("test1",), "test2", 3], [("test1",), "test2", 3]) 441 442 # 443 # test missing 444 # 445 self.check_equal(None, None) 446 447 # 448 # test glob 449 # 450 self.check_equal([("test1.*",), "test?2", 3], 451 [("test1.1","test1.2"), "test12", 3]) 452 self.check_equal(["test1.*", "test?2", 3], 453 ["test1.1","test1.2", "test12", 3]) 454 455 # 456 # test glob and string 457 # 458 self.check_equal([("test*1",), (("test3",),),"test2", 3], 459 [("test11","test21"), (("test3",),),"test2", 3]) 460 461 # 462 # test task function 463 # 464 self.check_equal(non_str_sequence, ["t1a", "t1b"]) 465 self.check_equal(get_strings_in_nested_sequence, ["t2"]) 466 self.check_equal([get_strings_in_nested_sequence, non_str_sequence], ["t2", "t1a", "t1b"]) 467 self.check_equal([non_str_sequence, [1, "this", ["that*", 5]], [(get_strings_in_nested_sequence,)]], 468 ['t1a', 't1b', [1, 'this', ['that1', 'that2', 5]], [('t2',)]]) 469 # 470 # test wrapper 471 # 472 self.check_equal(output_from(non_str_sequence, ["what", 7], 5), 473 ['t1a', 't1b', ['t3', 7], 5]) 474# 475# 476#_________________________________________________________________________________________ 477 478# Test_regex_replace 479 480#_________________________________________________________________________________________ 481class Test_regex_replace (unittest.TestCase): 482 def helper (self, data, result): 483 try_result = regex_replace("aaa.bbb.ccc.aaa", 484 re.compile("([a-z]+)\.([a-z]+)\.([a-z]+)\.([a-z]+)"), 485 data) 486 self.assertEqual(try_result , result) 487 488 def test_regex_replace(self): 489 self.helper(r"\3.\2.\1", "ccc.bbb.aaa") 490 self.helper(None, None) 491 self.helper(1, 1) 492 self.helper([r"\3.\2.\1", 1], ["ccc.bbb.aaa", 1]) 493 # note set is constructed with substituted results! 494 self.helper([r"\3.\2.\1", 1, (set([r"\1\2", r"\4\2", "aaabbb"]), "whatever", {1:2, 3:4})], 495 ['ccc.bbb.aaa', 1, (set(['aaabbb']), 'whatever', {1: 2, 3: 4})]) 496 497 498 499 500# 501# debug parameter ignored if called as a module 502# 503if sys.argv.count("--debug"): 504 sys.argv.remove("--debug") 505#sys.argv.append("Test_regex_replace") 506unittest.main() 507 508 509