PageRenderTime 59ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 0ms

/lib/common_test/src/ct_framework.erl

https://github.com/bsmr-erlang/otp
Erlang | 1701 lines | 1385 code | 109 blank | 207 comment | 66 complexity | be777359b70302d2fd76bfeb19a9bd7e MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-2.1, MPL-2.0-no-copyleft-exception, Apache-2.0

Large files files are truncated, but you can click here to view the full file

  1. %%
  2. %% %CopyrightBegin%
  3. %%
  4. %% Copyright Ericsson AB 2004-2018. All Rights Reserved.
  5. %%
  6. %% Licensed under the Apache License, Version 2.0 (the "License");
  7. %% you may not use this file except in compliance with the License.
  8. %% You may obtain a copy of the License at
  9. %%
  10. %% http://www.apache.org/licenses/LICENSE-2.0
  11. %%
  12. %% Unless required by applicable law or agreed to in writing, software
  13. %% distributed under the License is distributed on an "AS IS" BASIS,
  14. %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. %% See the License for the specific language governing permissions and
  16. %% limitations under the License.
  17. %%
  18. %% %CopyrightEnd%
  19. %%
  20. %%% Common Test Framework callback module.
  21. %%%
  22. %%% This module exports framework callback functions which are
  23. %%% called from the test_server.
  24. -module(ct_framework).
  25. -export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]).
  26. -export([report/2, warn/1, error_notification/4]).
  27. -export([get_log_dir/0, get_logopts/0, format_comment/1, get_html_wrapper/4]).
  28. -export([error_in_suite/1, init_per_suite/1, end_per_suite/1,
  29. init_per_group/2, end_per_group/2]).
  30. -include("ct.hrl").
  31. -include("ct_event.hrl").
  32. -include("ct_util.hrl").
  33. -define(val(Key, List), proplists:get_value(Key, List)).
  34. -define(val(Key, List, Def), proplists:get_value(Key, List, Def)).
  35. -define(rev(L), lists:reverse(L)).
  36. %%%-----------------------------------------------------------------
  37. %%% -spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} |
  38. %%% {skip,Reason} | {auto_skip,Reason}
  39. %%% Mod = atom()
  40. %%% Func = atom()
  41. %%% Args = list()
  42. %%% NewArgs = list()
  43. %%% Reason = term()
  44. %%%
  45. %%% Test server framework callback, called by the test_server
  46. %%% when a new test case is started.
  47. init_tc(_,{end_per_testcase_not_run,_},[Config]) ->
  48. %% Testcase is completed (skipped or failed), but end_per_testcase
  49. %% is not run - don't call pre-hook.
  50. {ok,[Config]};
  51. init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
  52. %% in case Mod == ct_framework, lookup the suite name
  53. Suite = get_suite_name(Mod, Config),
  54. case ct_hooks:init_tc(Suite,EPTC,Config) of
  55. NewConfig when is_list(NewConfig) ->
  56. {ok,[NewConfig]};
  57. Other->
  58. Other
  59. end;
  60. init_tc(Mod,Func0,Args) ->
  61. %% in case Mod == ct_framework, lookup the suite name
  62. Suite = get_suite_name(Mod, Args),
  63. {Func,HookFunc} = case Func0 of
  64. {init_per_testcase,F} -> {F,Func0};
  65. _ -> {Func0,Func0}
  66. end,
  67. %% check if previous testcase was interpreted and has left
  68. %% a "dead" trace window behind - if so, kill it
  69. case ct_util:get_testdata(interpret) of
  70. {What,kill,{TCPid,AttPid}} ->
  71. ct_util:kill_attached(TCPid,AttPid),
  72. ct_util:set_testdata({interpret,{What,kill,{undefined,undefined}}});
  73. _ ->
  74. ok
  75. end,
  76. case Func=/=end_per_suite
  77. andalso Func=/=end_per_group
  78. andalso ct_util:get_testdata(skip_rest) of
  79. true ->
  80. initialize(false,Mod,Func,Args),
  81. {auto_skip,"Repeated test stopped by force_stop option"};
  82. _ ->
  83. case ct_util:get_testdata(curr_tc) of
  84. {Suite,{suite0_failed,{require,Reason}}} ->
  85. initialize(false,Mod,Func,Args),
  86. {auto_skip,{require_failed_in_suite0,Reason}};
  87. {Suite,{suite0_failed,_}=Failure} ->
  88. initialize(false,Mod,Func,Args),
  89. {fail,Failure};
  90. _ ->
  91. ct_util:update_testdata(curr_tc,
  92. fun(undefined) ->
  93. [{Suite,Func}];
  94. (Running) ->
  95. [{Suite,Func}|Running]
  96. end, [create]),
  97. case ct_util:read_suite_data({seq,Suite,Func}) of
  98. undefined ->
  99. init_tc1(Mod,Suite,Func,HookFunc,Args);
  100. Seq when is_atom(Seq) ->
  101. case ct_util:read_suite_data({seq,Suite,Seq}) of
  102. [Func|TCs] -> % this is the 1st case in Seq
  103. %% make sure no cases in this seq are
  104. %% marked as failed from an earlier execution
  105. %% in the same suite
  106. lists:foreach(
  107. fun(TC) ->
  108. ct_util:save_suite_data(
  109. {seq,Suite,TC},
  110. Seq)
  111. end, TCs);
  112. _ ->
  113. ok
  114. end,
  115. init_tc1(Mod,Suite,Func,HookFunc,Args);
  116. {failed,Seq,BadFunc} ->
  117. initialize(false,Mod,Func,Args),
  118. {auto_skip,{sequence_failed,Seq,BadFunc}}
  119. end
  120. end
  121. end.
  122. init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) ->
  123. initialize(false,?MODULE,error_in_suite),
  124. _ = ct_suite_init(?MODULE,error_in_suite,[],Config0),
  125. case ?val(error,Config0) of
  126. undefined ->
  127. {fail,"unknown_error_in_suite"};
  128. Reason ->
  129. {fail,Reason}
  130. end;
  131. init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->
  132. Config1 =
  133. case ct_util:read_suite_data(last_saved_config) of
  134. {{Suite,LastFunc},SavedConfig} -> % last testcase
  135. [{saved_config,{LastFunc,SavedConfig}} |
  136. lists:keydelete(saved_config,1,Config0)];
  137. {{LastSuite,InitOrEnd},
  138. SavedConfig} when InitOrEnd == init_per_suite ;
  139. InitOrEnd == end_per_suite ->
  140. %% last suite
  141. [{saved_config,{LastSuite,SavedConfig}} |
  142. lists:keydelete(saved_config,1,Config0)];
  143. undefined ->
  144. lists:keydelete(saved_config,1,Config0)
  145. end,
  146. ct_util:delete_suite_data(last_saved_config),
  147. Config = lists:keydelete(watchdog,1,Config1),
  148. if Func == init_per_suite ->
  149. %% delete all default values used in previous suite
  150. ct_config:delete_default_config(suite),
  151. %% release all name -> key bindings (once per suite)
  152. ct_config:release_allocated();
  153. Func /= init_per_suite ->
  154. ok
  155. end,
  156. GroupPath = ?val(tc_group_path, Config, []),
  157. AllGroups = [?val(tc_group_properties, Config, []) | GroupPath],
  158. %% clear all config data default values set by previous
  159. %% testcase info function (these should only survive the
  160. %% testcase, not the whole suite)
  161. FuncSpec = group_or_func(Func,Config0),
  162. HookFunc1 =
  163. if is_tuple(FuncSpec) -> % group
  164. FuncSpec;
  165. true ->
  166. ct_config:delete_default_config(testcase),
  167. HookFunc
  168. end,
  169. case add_defaults(Mod,Func,AllGroups) of
  170. Error = {suite0_failed,_} ->
  171. initialize(false,Mod,FuncSpec),
  172. ct_util:set_testdata({curr_tc,{Suite,Error}}),
  173. {error,Error};
  174. Error = {group0_failed,_} ->
  175. initialize(false,Mod,FuncSpec),
  176. {auto_skip,Error};
  177. Error = {testcase0_failed,_} ->
  178. initialize(false,Mod,FuncSpec),
  179. {auto_skip,Error};
  180. {SuiteInfo,MergeResult} ->
  181. case MergeResult of
  182. {error,Reason} ->
  183. initialize(false,Mod,FuncSpec),
  184. {fail,Reason};
  185. _ ->
  186. init_tc2(Mod,Suite,Func,HookFunc1,
  187. SuiteInfo,MergeResult,Config)
  188. end
  189. end;
  190. init_tc1(_Mod,_Suite,_Func,_HookFunc,Args) ->
  191. {ok,Args}.
  192. init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
  193. %% timetrap must be handled before require
  194. MergedInfo = timetrap_first(MergeResult, [], []),
  195. %% tell logger to use specified style sheet
  196. _ = case lists:keysearch(stylesheet,1,MergeResult++Config) of
  197. {value,{stylesheet,SSFile}} ->
  198. ct_logs:set_stylesheet(Func,add_data_dir(SSFile,Config));
  199. _ ->
  200. case ct_util:get_testdata(stylesheet) of
  201. undefined ->
  202. ct_logs:clear_stylesheet(Func);
  203. SSFile ->
  204. ct_logs:set_stylesheet(Func,SSFile)
  205. end
  206. end,
  207. %% suppress output for connections (Conns is a
  208. %% list of {Type,Bool} tuples, e.g. {telnet,true}),
  209. case ct_util:get_overridden_silenced_connections() of
  210. undefined ->
  211. case lists:keysearch(silent_connections,1,MergeResult++Config) of
  212. {value,{silent_connections,Conns}} ->
  213. ct_util:silence_connections(Conns);
  214. _ ->
  215. ok
  216. end;
  217. Conns ->
  218. ct_util:silence_connections(Conns)
  219. end,
  220. FuncSpec = group_or_func(Func,Config),
  221. initialize((Func==init_per_suite),Mod,FuncSpec),
  222. case catch configure(MergedInfo,MergedInfo,SuiteInfo,
  223. FuncSpec,[],Config) of
  224. {suite0_failed,Reason} ->
  225. ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
  226. {require,Reason}}}}),
  227. {auto_skip,{require_failed_in_suite0,Reason}};
  228. {error,Reason} ->
  229. {auto_skip,{require_failed,Reason}};
  230. {'EXIT',Reason} ->
  231. {fail,Reason};
  232. {ok,PostInitHook,Config1} ->
  233. case get('$test_server_framework_test') of
  234. undefined ->
  235. ct_suite_init(Suite,HookFunc,PostInitHook,Config1);
  236. Fun ->
  237. PostInitHookResult = do_post_init_hook(PostInitHook,
  238. Config1),
  239. case Fun(init_tc, [PostInitHookResult ++ Config1]) of
  240. NewConfig when is_list(NewConfig) ->
  241. {ok,NewConfig};
  242. Else ->
  243. Else
  244. end
  245. end
  246. end.
  247. initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) ->
  248. initialize(RefreshLogs,Mod,group_or_func(Func,Config));
  249. initialize(RefreshLogs,Mod,Func,_) ->
  250. initialize(RefreshLogs,Mod,Func).
  251. initialize(RefreshLogs,Mod,FuncSpec) ->
  252. ct_logs:init_tc(RefreshLogs),
  253. ct_event:notify(#event{name=tc_start,
  254. node=node(),
  255. data={Mod,FuncSpec}}).
  256. ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->
  257. case ct_hooks:init_tc(Suite,HookFunc,Config) of
  258. NewConfig when is_list(NewConfig) ->
  259. PostInitHookResult = do_post_init_hook(PostInitHook,NewConfig),
  260. {ok, [PostInitHookResult ++ NewConfig]};
  261. Else ->
  262. Else
  263. end.
  264. do_post_init_hook(PostInitHook,Config) ->
  265. lists:flatmap(fun({Tag,Fun}) ->
  266. case lists:keysearch(Tag,1,Config) of
  267. {value,_} ->
  268. [];
  269. false ->
  270. case Fun() of
  271. {error,_} -> [];
  272. Result -> [{Tag,Result}]
  273. end
  274. end
  275. end, PostInitHook).
  276. add_defaults(Mod,Func, GroupPath) ->
  277. Suite = get_suite_name(Mod, GroupPath),
  278. case (catch Suite:suite()) of
  279. {'EXIT',{undef,_}} ->
  280. SuiteInfo = merge_with_suite_defaults(Suite,[]),
  281. SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
  282. case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of
  283. Error = {group0_failed,_} -> Error;
  284. Error = {testcase0_failed,_} -> Error;
  285. Error = {error,_} -> {SuiteInfo,Error};
  286. MergedInfo -> {SuiteInfo,MergedInfo}
  287. end;
  288. {'EXIT',Reason} ->
  289. ErrStr = io_lib:format("~n*** ERROR *** "
  290. "~w:suite/0 failed: ~tp~n",
  291. [Suite,Reason]),
  292. io:format("~ts", [ErrStr]),
  293. io:format(?def_gl, "~ts", [ErrStr]),
  294. {suite0_failed,{exited,Reason}};
  295. SuiteInfo when is_list(SuiteInfo) ->
  296. case lists:all(fun(E) when is_tuple(E) -> true;
  297. (_) -> false
  298. end, SuiteInfo) of
  299. true ->
  300. SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo),
  301. SuiteInfoNoCTH = [I || I <- SuiteInfo1,
  302. element(1,I) =/= ct_hooks],
  303. case add_defaults1(Mod,Func, GroupPath,
  304. SuiteInfoNoCTH) of
  305. Error = {group0_failed,_} -> Error;
  306. Error = {testcase0_failed,_} -> Error;
  307. Error = {error,_} -> {SuiteInfo1,Error};
  308. MergedInfo -> {SuiteInfo1,MergedInfo}
  309. end;
  310. false ->
  311. ErrStr = io_lib:format("~n*** ERROR *** "
  312. "Invalid return value from "
  313. "~w:suite/0: ~tp~n",
  314. [Suite,SuiteInfo]),
  315. io:format("~ts", [ErrStr]),
  316. io:format(?def_gl, "~ts", [ErrStr]),
  317. {suite0_failed,bad_return_value}
  318. end;
  319. SuiteInfo ->
  320. ErrStr = io_lib:format("~n*** ERROR *** "
  321. "Invalid return value from "
  322. "~w:suite/0: ~tp~n", [Suite,SuiteInfo]),
  323. io:format("~ts", [ErrStr]),
  324. io:format(?def_gl, "~ts", [ErrStr]),
  325. {suite0_failed,bad_return_value}
  326. end.
  327. add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
  328. Suite = get_suite_name(Mod, GroupPath),
  329. %% GroupPathInfo (for subgroup on level X) =
  330. %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
  331. GroupPathInfo =
  332. lists:map(fun(GroupProps) ->
  333. case ?val(name, GroupProps) of
  334. undefined ->
  335. [];
  336. Name ->
  337. case catch Suite:group(Name) of
  338. GrInfo when is_list(GrInfo) -> GrInfo;
  339. {'EXIT',{undef,_}} -> [];
  340. BadGr0 -> {error,BadGr0,Name}
  341. end
  342. end
  343. end, GroupPath),
  344. case lists:keysearch(error, 1, GroupPathInfo) of
  345. {value,{error,BadGr0Val,GrName}} ->
  346. Gr0ErrStr = io_lib:format("~n*** ERROR *** "
  347. "Invalid return value from "
  348. "~w:group(~tw): ~tp~n",
  349. [Mod,GrName,BadGr0Val]),
  350. io:format("~ts", [Gr0ErrStr]),
  351. io:format(?def_gl, "~ts", [Gr0ErrStr]),
  352. {group0_failed,bad_return_value};
  353. _ ->
  354. Args = if Func == init_per_group ; Func == end_per_group ->
  355. [?val(name, hd(GroupPath))];
  356. true ->
  357. []
  358. end,
  359. TestCaseInfo =
  360. case catch apply(Mod,Func,Args) of
  361. TCInfo when is_list(TCInfo) -> TCInfo;
  362. {'EXIT',{undef,_}} -> [];
  363. BadTC0 -> {error,BadTC0}
  364. end,
  365. case TestCaseInfo of
  366. {error,BadTC0Val} ->
  367. TC0ErrStr = io_lib:format("~n*** ERROR *** "
  368. "Invalid return value from "
  369. "~w:~tw/0: ~tp~n",
  370. [Mod,Func,BadTC0Val]),
  371. io:format("~ts", [TC0ErrStr]),
  372. io:format(?def_gl, "~ts", [TC0ErrStr]),
  373. {testcase0_failed,bad_return_value};
  374. _ ->
  375. %% let test case info (also for all config funcs) override
  376. %% group info, and lower level group info override higher
  377. %% level info
  378. TCAndGroupInfo =
  379. [TestCaseInfo | remove_info_in_prev(TestCaseInfo,
  380. GroupPathInfo)],
  381. %% find and save require terms found in suite info
  382. SuiteReqs =
  383. [SDDef || SDDef <- SuiteInfo,
  384. ((require == element(1,SDDef))
  385. or (default_config == element(1,SDDef)))],
  386. case check_for_clashes(TestCaseInfo, GroupPathInfo,
  387. SuiteReqs) of
  388. [] ->
  389. add_defaults2(Mod,Func, TCAndGroupInfo,
  390. SuiteInfo,SuiteReqs);
  391. Clashes ->
  392. {error,{config_name_already_in_use,Clashes}}
  393. end
  394. end
  395. end.
  396. get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] ->
  397. get_suite_name(?MODULE, Cfg);
  398. get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] ->
  399. case ?val(tc_group_properties, Cfg) of
  400. undefined ->
  401. case ?val(suite, Cfg) of
  402. undefined -> ?MODULE;
  403. Suite -> Suite
  404. end;
  405. GrProps ->
  406. case ?val(suite, GrProps) of
  407. undefined -> ?MODULE;
  408. Suite -> Suite
  409. end
  410. end;
  411. get_suite_name(Mod, _) ->
  412. Mod.
  413. %% Check that alias names are not already in use
  414. check_for_clashes(TCInfo, [CurrGrInfo|Path], SuiteInfo) ->
  415. ReqNames = fun(Info) -> [element(2,R) || R <- Info,
  416. size(R) == 3,
  417. require == element(1,R)]
  418. end,
  419. ExistingNames = lists:flatten([ReqNames(L) || L <- [SuiteInfo|Path]]),
  420. CurrGrReqNs = ReqNames(CurrGrInfo),
  421. GrClashes = [Name || Name <- CurrGrReqNs,
  422. true == lists:member(Name, ExistingNames)],
  423. AllReqNs = CurrGrReqNs ++ ExistingNames,
  424. TCClashes = [Name || Name <- ReqNames(TCInfo),
  425. true == lists:member(Name, AllReqNs)],
  426. TCClashes ++ GrClashes.
  427. %% Delete the info terms in Terms from all following info lists
  428. remove_info_in_prev(Terms, [[] | Rest]) ->
  429. [[] | remove_info_in_prev(Terms, Rest)];
  430. remove_info_in_prev(Terms, [Info | Rest]) ->
  431. UniqueInInfo = [U || U <- Info,
  432. ((timetrap == element(1,U)) and
  433. (not lists:keymember(timetrap,1,Terms))) or
  434. ((require == element(1,U)) and
  435. (not lists:member(U,Terms))) or
  436. ((default_config == element(1,U)) and
  437. (not keysmember([default_config,1,
  438. element(2,U),2], Terms)))],
  439. OtherTermsInInfo = [T || T <- Info,
  440. timetrap /= element(1,T),
  441. require /= element(1,T),
  442. default_config /= element(1,T),
  443. false == lists:keymember(element(1,T),1,
  444. Terms)],
  445. KeptInfo = UniqueInInfo ++ OtherTermsInInfo,
  446. [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)];
  447. remove_info_in_prev(_, []) ->
  448. [].
  449. keysmember([Key,Pos|Next], List) ->
  450. case [Elem || Elem <- List, Key == element(Pos,Elem)] of
  451. [] -> false;
  452. Found -> keysmember(Next, Found)
  453. end;
  454. keysmember([], _) -> true.
  455. add_defaults2(_Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs) ->
  456. Info = lists:flatten([IPSInfo, SuiteReqs]),
  457. lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]);
  458. add_defaults2(_Mod,init_per_group, IPGAndGroupInfo, SuiteInfo,SuiteReqs) ->
  459. SuiteInfo1 =
  460. remove_info_in_prev(lists:flatten([IPGAndGroupInfo,
  461. SuiteReqs]), [SuiteInfo]),
  462. %% don't require terms in prev groups (already processed)
  463. case IPGAndGroupInfo of
  464. [IPGInfo] ->
  465. lists:flatten([IPGInfo,SuiteInfo1]);
  466. [IPGInfo | [CurrGroupInfo | PrevGroupInfo]] ->
  467. PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
  468. lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1,
  469. SuiteInfo1])
  470. end;
  471. add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs) ->
  472. %% Include require elements from test case info and current group,
  473. %% but not from previous groups or suite/0 (since we've already required
  474. %% those vars). Let test case info elements override group and suite
  475. %% info elements.
  476. SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo,
  477. SuiteReqs]), [SuiteInfo]),
  478. %% don't require terms in prev groups (already processed)
  479. case TCAndGroupInfo of
  480. [TCInfo] ->
  481. lists:flatten([TCInfo,SuiteInfo1]);
  482. [TCInfo | [CurrGroupInfo | PrevGroupInfo]] ->
  483. PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
  484. lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1,
  485. SuiteInfo1])
  486. end.
  487. delete_require_terms([Info | Prev]) ->
  488. Info1 = [T || T <- Info,
  489. require /= element(1,T),
  490. default_config /= element(1,T)],
  491. [Info1 | delete_require_terms(Prev)];
  492. delete_require_terms([]) ->
  493. [].
  494. merge_with_suite_defaults(Mod,SuiteInfo) ->
  495. case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of
  496. {value,{suite_defaults,Defaults}} ->
  497. SDReqs =
  498. [SDDef || SDDef <- Defaults,
  499. require == element(1,SDDef),
  500. false == lists:keymember(element(2,SDDef),2,
  501. SuiteInfo)],
  502. SuiteInfo ++ SDReqs ++
  503. [SDDef || SDDef <- Defaults,
  504. require /= element(1,SDDef),
  505. false == lists:keymember(element(1,SDDef),1,
  506. SuiteInfo)];
  507. false ->
  508. SuiteInfo
  509. end.
  510. timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) ->
  511. timetrap_first(Rest,Info,[Trap | Found]);
  512. timetrap_first([Other | Rest],Info,Found) ->
  513. timetrap_first(Rest,[Other | Info],Found);
  514. timetrap_first([],Info,[]) ->
  515. [{timetrap,{minutes,30}} | ?rev(Info)];
  516. timetrap_first([],Info,Found) ->
  517. ?rev(Found) ++ ?rev(Info).
  518. configure([{require,Required}|Rest],
  519. Info,SuiteInfo,Scope,PostInitHook,Config) ->
  520. case ct:require(Required) of
  521. ok ->
  522. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  523. Error = {error,Reason} ->
  524. case required_default('_UNDEF',Required,Info,
  525. SuiteInfo,Scope) of
  526. ok ->
  527. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  528. _ ->
  529. case lists:keymember(Required,2,SuiteInfo) of
  530. true ->
  531. {suite0_failed,Reason};
  532. false ->
  533. Error
  534. end
  535. end
  536. end;
  537. configure([{require,Name,Required}|Rest],
  538. Info,SuiteInfo,Scope,PostInitHook,Config) ->
  539. case ct:require(Name,Required) of
  540. ok ->
  541. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  542. Error = {error,Reason} ->
  543. case required_default(Name,Required,Info,SuiteInfo,Scope) of
  544. ok ->
  545. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  546. _ ->
  547. case lists:keymember(Name,2,SuiteInfo) of
  548. true ->
  549. {suite0_failed,Reason};
  550. false ->
  551. Error
  552. end
  553. end
  554. end;
  555. configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
  556. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  557. configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
  558. PostInitHook1 =
  559. [{watchdog,fun() -> case test_server:get_timetrap_info() of
  560. undefined ->
  561. test_server:timetrap(Time);
  562. _ ->
  563. {error,already_set}
  564. end
  565. end} | PostInitHook],
  566. configure(Rest,Info,SuiteInfo,Scope,PostInitHook1,Config);
  567. configure([{ct_hooks,Hook}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
  568. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,[{ct_hooks,Hook}|Config]);
  569. configure([_|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
  570. configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
  571. configure([],_,_,_,PostInitHook,Config) ->
  572. {ok,PostInitHook,Config}.
  573. %% the require element in Info may come from suite/0 and
  574. %% should be scoped 'suite', or come from the group info
  575. %% function and be scoped 'group', or come from the testcase
  576. %% info function and then be scoped 'testcase'
  577. required_default(Name,Key,Info,_,init_per_suite) ->
  578. try_set_default(Name,Key,Info,suite);
  579. required_default(Name,Key,Info,_,{init_per_group,GrName,_}) ->
  580. try_set_default(Name,Key,Info,{group,GrName});
  581. required_default(Name,Key,Info,_,_FuncSpec) ->
  582. try_set_default(Name,Key,Info,testcase).
  583. try_set_default(Name,Key,Info,Where) ->
  584. CfgElems =
  585. case lists:keysearch(Name,1,Info) of
  586. {value,{Name,Val}} ->
  587. [Val];
  588. false ->
  589. case catch [{Key,element(3,Elem)} || Elem <- Info,
  590. element(1,Elem)==default_config,
  591. element(2,Elem)==Key] of
  592. {'EXIT',_} -> [];
  593. Result -> Result
  594. end
  595. end,
  596. case {Name,CfgElems} of
  597. {_,[]} ->
  598. no_default;
  599. {'_UNDEF',_} ->
  600. _ = [ct_config:set_default_config([CfgVal],Where) || CfgVal <- CfgElems],
  601. ok;
  602. _ ->
  603. _ = [ct_config:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems],
  604. ok
  605. end.
  606. %%%-----------------------------------------------------------------
  607. %%% -spec end_tc(Mod,Func,Args) -> {ok,NewArgs}| {error,Reason} |
  608. %%% {skip,Reason} | {auto_skip,Reason}
  609. %%% Mod = atom()
  610. %%% Func = atom()
  611. %%% Args = list()
  612. %%% NewArgs = list()
  613. %%% Reason = term()
  614. %%%
  615. %%% Test server framework callback, called by the test_server
  616. %%% when a test case is finished.
  617. end_tc(Mod, Fun, Args) ->
  618. %% Have to keep end_tc/3 for backwards compatibility issues
  619. end_tc(Mod, Fun, Args, '$end_tc_dummy').
  620. end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) ->
  621. %% this clause gets called if CT has encountered a suite that
  622. %% can't be executed
  623. FinalNotify =
  624. case ct_hooks:end_tc(?MODULE, error_in_suite, Args, Result, Return) of
  625. '$ct_no_change' ->
  626. Result;
  627. HookResult ->
  628. HookResult
  629. end,
  630. Event = #event{name=tc_done,
  631. node=node(),
  632. data={?MODULE,error_in_suite,tag(FinalNotify)}},
  633. ct_event:sync_notify(Event),
  634. ok;
  635. end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) ->
  636. end_tc(Mod,Func,TCPid,Result,Args,Return);
  637. end_tc(Mod,Func,{Result,[Args]}, Return) ->
  638. end_tc(Mod,Func,self(),Result,Args,Return).
  639. end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
  640. case end_hook_func(IPTC,Return,IPTC) of
  641. undefined -> ok;
  642. _ ->
  643. %% in case Mod == ct_framework, lookup the suite name
  644. Suite = get_suite_name(Mod, Args),
  645. case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
  646. '$ct_no_change' ->
  647. ok;
  648. HookResult ->
  649. HookResult
  650. end
  651. end;
  652. end_tc(Mod,Func00,TCPid,Result,Args,Return) ->
  653. %% in case Mod == ct_framework, lookup the suite name
  654. Suite = get_suite_name(Mod, Args),
  655. {OnlyCleanup,Func0} =
  656. case Func00 of
  657. {cleanup,F0} ->
  658. {true,F0};
  659. _ ->
  660. {false,Func00}
  661. end,
  662. {Func,FuncSpec,HookFunc} =
  663. case Func0 of
  664. {end_per_testcase_not_run,F} ->
  665. %% Testcase is completed (skipped or failed), but
  666. %% end_per_testcase is not run - don't call post-hook.
  667. {F,F,undefined};
  668. {end_per_testcase,F} ->
  669. {F,F,Func0};
  670. _ ->
  671. FS = group_or_func(Func0,Args),
  672. HF = end_hook_func(Func0,Return,FS),
  673. {Func0,FS,HF}
  674. end,
  675. test_server:timetrap_cancel(),
  676. %% save the testcase process pid so that it can be used
  677. %% to look up the attached trace window later
  678. case ct_util:get_testdata(interpret) of
  679. {What,kill,_} ->
  680. AttPid = ct_util:get_attached(self()),
  681. ct_util:set_testdata({interpret,{What,kill,{self(),AttPid}}});
  682. _ ->
  683. ok
  684. end,
  685. if Func == end_per_group; Func == end_per_suite ->
  686. %% clean up any saved comments
  687. ct_util:match_delete_testdata({comment,'_'});
  688. true ->
  689. %% attemp to delete any saved comment for this TC
  690. case process_info(TCPid, group_leader) of
  691. {group_leader,TCGL} ->
  692. ct_util:delete_testdata({comment,TCGL});
  693. _ ->
  694. ok
  695. end
  696. end,
  697. ct_util:delete_suite_data(last_saved_config),
  698. {Result1,FinalNotify} =
  699. case HookFunc of
  700. undefined ->
  701. {ok,Result};
  702. _ when OnlyCleanup ->
  703. {ok,Result};
  704. _ ->
  705. case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
  706. '$ct_no_change' ->
  707. {ok,Result};
  708. HookResult ->
  709. {HookResult,HookResult}
  710. end
  711. end,
  712. FinalResult =
  713. case get('$test_server_framework_test') of
  714. _ when OnlyCleanup ->
  715. Result1;
  716. undefined ->
  717. %% send sync notification so that event handlers may print
  718. %% in the log file before it gets closed
  719. Event = #event{name=tc_done,
  720. node=node(),
  721. data={Mod,FuncSpec,
  722. tag(FinalNotify)}},
  723. ct_event:sync_notify(Event),
  724. Result1;
  725. Fun ->
  726. %% send sync notification so that event handlers may print
  727. %% in the log file before it gets closed
  728. Event = #event{name=tc_done,
  729. node=node(),
  730. data={Mod,FuncSpec,
  731. tag({'$test_server_framework_test',
  732. FinalNotify})}},
  733. ct_event:sync_notify(Event),
  734. Fun(end_tc, Return)
  735. end,
  736. case FuncSpec of
  737. {_,GroupName,_Props} ->
  738. if Func == end_per_group ->
  739. ct_config:delete_default_config({group,GroupName});
  740. true -> ok
  741. end,
  742. case lists:keysearch(save_config,1,Args) of
  743. {value,{save_config,SaveConfig}} ->
  744. ct_util:save_suite_data(last_saved_config,
  745. {Suite,{group,GroupName}},
  746. SaveConfig);
  747. false ->
  748. ok
  749. end;
  750. _ ->
  751. case lists:keysearch(save_config,1,Args) of
  752. {value,{save_config,SaveConfig}} ->
  753. ct_util:save_suite_data(last_saved_config,
  754. {Suite,Func},SaveConfig);
  755. false ->
  756. ok
  757. end
  758. end,
  759. ct_util:reset_silent_connections(),
  760. %% reset the curr_tc state, or delete this TC from the list of
  761. %% executing cases (if in a parallel group)
  762. ClearCurrTC = fun(Running = [_,_|_]) ->
  763. lists:keydelete(Func,2,Running);
  764. ({_,{suite0_failed,_}}) ->
  765. undefined;
  766. ([{_,CurrTC}]) when CurrTC == Func ->
  767. undefined;
  768. (undefined) ->
  769. undefined;
  770. (Unexpected) ->
  771. {error,{reset_curr_tc,{Mod,Func},Unexpected}}
  772. end,
  773. case ct_util:update_testdata(curr_tc, ClearCurrTC) of
  774. {error,_} = ClearError ->
  775. exit(ClearError);
  776. _ ->
  777. ok
  778. end,
  779. case FinalResult of
  780. {auto_skip,{sequence_failed,_,_}} ->
  781. %% ct_logs:init_tc is never called for a skipped test case
  782. %% in a failing sequence, so neither should end_tc
  783. ok;
  784. _ ->
  785. case ct_logs:end_tc(TCPid) of
  786. {error,Reason} ->
  787. exit({error,{logger,Reason}});
  788. _ ->
  789. ok
  790. end
  791. end,
  792. case Func of
  793. end_per_suite ->
  794. ct_util:match_delete_suite_data({seq,Suite,'_'});
  795. _ ->
  796. ok
  797. end,
  798. FinalResult.
  799. %% This is to make sure that no post_init_per_* is ever called if the
  800. %% corresponding pre_init_per_* was not called.
  801. %% The skip or fail reasons are those that can be returned from
  802. %% init_tc above in situations where we never came to call
  803. %% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call
  804. %% ct_hooks:init_tc for init_per_suite, and thus we must not call
  805. %% ct_hooks:end_tc for init_per_suite either.
  806. end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) ->
  807. undefined;
  808. end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) ->
  809. undefined;
  810. end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) ->
  811. undefined;
  812. end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_)
  813. when InfoFuncError==testcase0_failed;
  814. InfoFuncError==require_failed ->
  815. undefined;
  816. end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_)
  817. when InfoFuncError==group0_failed;
  818. InfoFuncError==require_failed ->
  819. undefined;
  820. end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) ->
  821. undefined;
  822. end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) ->
  823. undefined;
  824. end_hook_func(_,_,Default) ->
  825. Default.
  826. %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |
  827. %% {testcase_aborted,Reason} | testcase_aborted_or_killed |
  828. %% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} |
  829. %% {user_timetrap_error,Reason} |
  830. %% Other (ignored return value, e.g. 'ok')
  831. tag({'$test_server_framework_test',Result}) ->
  832. case tag(Result) of
  833. ok -> Result;
  834. Failure -> Failure
  835. end;
  836. tag({skipped,Reason={failed,{_,init_per_testcase,_}}}) ->
  837. {auto_skipped,Reason};
  838. tag({STag,Reason}) when STag == skip; STag == skipped ->
  839. case Reason of
  840. {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
  841. _ -> {skipped,Reason}
  842. end;
  843. tag({auto_skip,Reason}) ->
  844. {auto_skipped,Reason};
  845. tag({fail,Reason}) ->
  846. {failed,{error,Reason}};
  847. tag(Failed = {failed,_Reason}) ->
  848. Failed;
  849. tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT';
  850. ETag == timetrap_timeout;
  851. ETag == testcase_aborted ->
  852. {failed,E};
  853. tag(E = testcase_aborted_or_killed) ->
  854. {failed,E};
  855. tag(UserTimetrap = {user_timetrap_error,_Reason}) ->
  856. UserTimetrap;
  857. tag(_Other) ->
  858. ok.
  859. %%%-----------------------------------------------------------------
  860. %%% -spec error_notification(Mod,Func,Args,Error) -> ok
  861. %%% Mod = atom()
  862. %%% Func = atom()
  863. %%% Args = list()
  864. %%% Error = term()
  865. %%%
  866. %%% This function is called as the result of testcase
  867. %%% Func in suite Mod crashing.
  868. %%% Error specifies the reason for failing.
  869. error_notification(Mod,Func,_Args,{Error,Loc}) ->
  870. ErrorSpec = case Error of
  871. {What={_E,_R},Trace} when is_list(Trace) ->
  872. What;
  873. What ->
  874. What
  875. end,
  876. ErrorStr = case ErrorSpec of
  877. {badmatch,Descr} ->
  878. Descr1 = io_lib:format("~tP",[Descr,10]),
  879. DescrLength = string:length(Descr1),
  880. if DescrLength > 50 ->
  881. Descr2 = string:slice(Descr1,0,50),
  882. io_lib:format("{badmatch,~ts...}",[Descr2]);
  883. true ->
  884. io_lib:format("{badmatch,~ts}",[Descr1])
  885. end;
  886. {test_case_failed,Reason} ->
  887. case (catch io_lib:format("{test_case_failed,~ts}", [Reason])) of
  888. {'EXIT',_} ->
  889. io_lib:format("{test_case_failed,~tp}", [Reason]);
  890. Result -> Result
  891. end;
  892. {'EXIT',_Reason} = EXIT ->
  893. io_lib:format("~tP", [EXIT,5]);
  894. {Spec,_Reason} when is_atom(Spec) ->
  895. io_lib:format("~tw", [Spec]);
  896. Other ->
  897. io_lib:format("~tP", [Other,5])
  898. end,
  899. ErrorHtml =
  900. "<font color=\"brown\">" ++ ct_logs:escape_chars(ErrorStr) ++ "</font>",
  901. case {Mod,Error} of
  902. %% some notifications come from the main test_server process
  903. %% and for these cases the existing comment may not be modified
  904. {_,{timetrap_timeout,_TVal}} ->
  905. ok;
  906. {_,{testcase_aborted,_Info}} ->
  907. ok;
  908. {_,testcase_aborted_or_killed} ->
  909. ok;
  910. {undefined,_OtherError} ->
  911. ok;
  912. _ ->
  913. %% this notification comes from the test case process, so
  914. %% we can add error info to comment with test_server:comment/1
  915. case ct_util:get_testdata({comment,group_leader()}) of
  916. undefined ->
  917. test_server:comment(ErrorHtml);
  918. Comment ->
  919. CommentHtml =
  920. "<font color=\"green\">" ++ "(" ++ "</font>"
  921. ++ Comment ++
  922. "<font color=\"green\">" ++ ")" ++ "</font>",
  923. Str = io_lib:format("~ts ~ts", [ErrorHtml,CommentHtml]),
  924. test_server:comment(Str)
  925. end
  926. end,
  927. PrintError = fun(ErrorFormat, ErrorArgs) ->
  928. Div = "\n- - - - - - - - - - - - - - - - - - - "
  929. "- - - - - - - - - - - - - - - - - - - - -\n",
  930. ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs),
  931. io:format(?def_gl, "~ts~n", [lists:concat([Div,ErrorStr2,Div])]),
  932. Link =
  933. "\n\n<a href=\"#end\">"
  934. "Full error description and stacktrace"
  935. "</a>",
  936. ErrorHtml2 = ct_logs:escape_chars(ErrorStr2),
  937. ct_logs:tc_log(ct_error_notify,
  938. ?MAX_IMPORTANCE,
  939. "CT Error Notification",
  940. "~ts", [ErrorHtml2++Link],
  941. [])
  942. end,
  943. case Loc of
  944. [{?MODULE,error_in_suite}] ->
  945. PrintError("Error in suite detected: ~ts", [ErrorStr]);
  946. R when R == unknown; R == undefined ->
  947. PrintError("Error detected: ~ts", [ErrorStr]);
  948. %% if a function specified by all/0 does not exist, we
  949. %% pick up undef here
  950. [{LastMod,LastFunc}|_] when ErrorStr == "undef" ->
  951. PrintError("~w:~tw could not be executed~nReason: ~ts",
  952. [LastMod,LastFunc,ErrorStr]);
  953. [{LastMod,LastFunc}|_] ->
  954. PrintError("~w:~tw failed~nReason: ~ts", [LastMod,LastFunc,ErrorStr]);
  955. [{LastMod,LastFunc,LastLine}|_] ->
  956. %% print error to console, we are only
  957. %% interested in the last executed expression
  958. PrintError("~w:~tw failed on line ~w~nReason: ~ts",
  959. [LastMod,LastFunc,LastLine,ErrorStr]),
  960. case ct_util:read_suite_data({seq,Mod,Func}) of
  961. undefined ->
  962. ok;
  963. Seq ->
  964. SeqTCs = ct_util:read_suite_data({seq,Mod,Seq}),
  965. mark_as_failed(Seq,Mod,Func,SeqTCs)
  966. end
  967. end,
  968. ok.
  969. %% cases in seq that have already run
  970. mark_as_failed(Seq,Mod,Func,[Func|TCs]) ->
  971. mark_as_failed1(Seq,Mod,Func,TCs);
  972. mark_as_failed(Seq,Mod,Func,[_TC|TCs]) ->
  973. mark_as_failed(Seq,Mod,Func,TCs);
  974. mark_as_failed(_,_,_,[]) ->
  975. ok;
  976. mark_as_failed(_,_,_,undefined) ->
  977. ok.
  978. %% mark rest of cases in seq to be skipped
  979. mark_as_failed1(Seq,Mod,Func,[TC|TCs]) ->
  980. ct_util:save_suite_data({seq,Mod,TC},{failed,Seq,Func}),
  981. mark_as_failed1(Seq,Mod,Func,TCs);
  982. mark_as_failed1(_,_,_,[]) ->
  983. ok.
  984. group_or_func(Func, Config) when Func == init_per_group;
  985. Func == end_per_group ->
  986. case ?val(tc_group_properties, Config) of
  987. undefined ->
  988. {Func,unknown,[]};
  989. GrProps ->
  990. GrName = ?val(name,GrProps),
  991. {Func,GrName,proplists:delete(name,GrProps)}
  992. end;
  993. group_or_func(Func, _Config) ->
  994. Func.
  995. %%%-----------------------------------------------------------------
  996. %%% -spec get_suite(Mod, Func) -> Tests
  997. %%%
  998. %%% Called from test_server for every suite (Func==all)
  999. %%% and every test case. If the former, all test cases in the suite
  1000. %%% should be returned.
  1001. get_suite(Mod, all) ->
  1002. case safe_apply_groups_0(Mod,{ok,[]}) of
  1003. {ok,GroupDefs} ->
  1004. try ct_groups:find_groups(Mod, all, all, GroupDefs) of
  1005. ConfTests when is_list(ConfTests) ->
  1006. get_all(Mod, ConfTests)
  1007. catch
  1008. throw:{error,Error} ->
  1009. [{?MODULE,error_in_suite,[[{error,Error}]]}];
  1010. _:Error:S ->
  1011. [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
  1012. end;
  1013. {error,{bad_return,_Bad}} ->
  1014. E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
  1015. [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
  1016. {error,{bad_hook_return,Bad}} ->
  1017. E = "Bad return value from post_groups/2 hook function",
  1018. [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
  1019. {error,{failed,ExitReason}} ->
  1020. case ct_util:get_testdata({error_in_suite,Mod}) of
  1021. undefined ->
  1022. ErrStr = io_lib:format("~n*** ERROR *** "
  1023. "~w:groups/0 failed: ~p~n",
  1024. [Mod,ExitReason]),
  1025. io:format(?def_gl, ErrStr, []),
  1026. %% save the error info so it doesn't get printed twice
  1027. ct_util:set_testdata_async({{error_in_suite,Mod},
  1028. ExitReason});
  1029. _ExitReason ->
  1030. ct_util:delete_testdata({error_in_suite,Mod})
  1031. end,
  1032. Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
  1033. [{?MODULE,error_in_suite,[[{error,Reason}]]}];
  1034. {error,What} ->
  1035. [{?MODULE,error_in_suite,[[{error,What}]]}]
  1036. end;
  1037. %%!============================================================
  1038. %%! Note: The handling of sequences in get_suite/2 and get_all/2
  1039. %%! is deprecated and should be removed at some point...
  1040. %%!============================================================
  1041. %% group
  1042. get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
  1043. case safe_apply_groups_0(Mod,{ok,[Group]}) of
  1044. {ok,GroupDefs} ->
  1045. Name = ?val(name, Props),
  1046. try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
  1047. [] ->
  1048. [];
  1049. ConfTests when is_list(ConfTests) ->
  1050. case lists:member(skipped, Props) of
  1051. true ->
  1052. %% a *subgroup* specified *only* as skipped (and not
  1053. %% as an explicit test) should not be returned, or
  1054. %% init/end functions for top groups will be executed
  1055. try ?val(name, element(2, hd(ConfTests))) of
  1056. Name -> % top group
  1057. ct_groups:delete_subs(ConfTests, ConfTests);
  1058. _ -> []
  1059. catch
  1060. _:_ -> []
  1061. end;
  1062. false ->
  1063. ConfTests1 = ct_groups:delete_subs(ConfTests,
  1064. ConfTests),
  1065. case ?val(override, Props) of
  1066. undefined ->
  1067. ConfTests1;
  1068. [] ->
  1069. ConfTests1;
  1070. ORSpec ->
  1071. ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
  1072. true -> ORSpec end,
  1073. ct_groups:search_and_override(ConfTests1,
  1074. ORSpec1, Mod)
  1075. end
  1076. end
  1077. catch
  1078. throw:{error,Error} ->
  1079. [{?MODULE,error_in_suite,[[{error,Error}]]}];
  1080. _:Error:S ->
  1081. [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
  1082. end;
  1083. {error,{bad_return,_Bad}} ->
  1084. E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
  1085. [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
  1086. {error,{bad_hook_return,Bad}} ->
  1087. E = "Bad return value from post_groups/2 hook function",
  1088. [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
  1089. {error,{failed,ExitReason}} ->
  1090. case ct_util:get_testdata({error_in_suite,Mod}) of
  1091. undefined ->
  1092. ErrStr = io_lib:format("~n*** ERROR *** "
  1093. "~w:groups/0 failed: ~p~n",
  1094. [Mod,ExitReason]),
  1095. io:format(?def_gl, ErrStr, []),
  1096. %% save the error info so it doesn't get printed twice
  1097. ct_util:set_testdata_async({{error_in_suite,Mod},
  1098. ExitReason});
  1099. _ExitReason ->
  1100. ct_util:delete_testdata({error_in_suite,Mod})
  1101. end,
  1102. Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
  1103. [{?MODULE,error_in_suite,[[{error,Reason}]]}];
  1104. {error,What} ->
  1105. [{?MODULE,error_in_suite,[[{error,What}]]}]
  1106. end;
  1107. %% testcase
  1108. get_suite(Mod, Name) ->
  1109. get_seq(Mod, Name).
  1110. %%%-----------------------------------------------------------------
  1111. get_all_cases(Suite) ->
  1112. case get_suite(Suite, all) of
  1113. [{?MODULE,error_in_suite,[[{error,_}=Error]]}] ->
  1114. Error;
  1115. [{?MODULE,error_in_suite,[[Error]]}] ->
  1116. {error,Error};
  1117. Tests ->
  1118. Cases = get_all_cases1(Suite, Tests),
  1119. ?rev(lists:foldl(fun(TC, TCs) ->
  1120. case lists:member(TC, TCs) of
  1121. true -> TCs;
  1122. false -> [TC | TCs]
  1123. end
  1124. end, [], Cases))
  1125. end.
  1126. get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) ->
  1127. get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests);
  1128. get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) ->
  1129. [{Suite,Test} | get_all_cases1(Suite, Tests)];
  1130. get_all_cases1(Suite, [Test | Tests]) ->
  1131. [Test | get_all_cases1(Suite, Tests)];
  1132. get_all_cases1(_, []) ->
  1133. [].
  1134. %%%-----------------------------------------------------------------
  1135. get_all(Mod, ConfTests) ->
  1136. case safe_apply_all_0(Mod) of
  1137. {ok,AllTCs} ->
  1138. %% expand group references using ConfTests
  1139. try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of
  1140. {error,_} = Error ->
  1141. [{?MODULE,error_in_suite,[[Error]]}];
  1142. Tests0 ->
  1143. Tests = ct_groups:delete_subs(Tests0, Tests0),
  1144. expand_tests(Mod, Tests)
  1145. catch
  1146. throw:{error,Error} ->
  1147. [{?MODULE,error_in_suite,[[{error,Error}]]}];
  1148. _:Error:S ->
  1149. [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
  1150. end;
  1151. Skip = {skip,_Reason} ->
  1152. Skip;
  1153. {error,undef} ->
  1154. Reason =
  1155. case code:which(Mod) of
  1156. non_existing ->
  1157. list_to_atom(
  1158. atom_to_list(Mod)++
  1159. " cannot be compiled or loaded");
  1160. _ ->
  1161. list_to_atom(
  1162. atom_to_list(Mod)++":all/0 is missing")
  1163. end,
  1164. %% this makes test_server call error_in_suite as first
  1165. %% (and only) test case so we can report Reason properly
  1166. [{?MODULE,error_in_suite,[[{error,Reason}]]}];
  1167. {error,{bad_return,_Bad}} ->
  1168. Reason =
  1169. list_to_atom("Bad return value from "++
  1170. atom_to_list(Mod)++":all/0"),
  1171. [{?MODULE,error_in_suite,[[{error,Reason}]]}];
  1172. {error,{bad_hook_return,Bad}} ->
  1173. Reason =
  1174. list_to_atom("Bad return value from post_all/3 hook function"),
  1175. [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}];
  1176. {error,{failed,ExitReason}} ->
  1177. case ct_util:get_testdata({error_in_suite,Mod}) of
  1178. undefined ->
  1179. ErrStr = io_lib:format("~n*** ERROR *** "
  1180. "~w:all/0 failed: ~tp~n",
  1181. [Mod,ExitReason]),
  1182. io:format(?def_gl, "~ts", [ErrStr]),
  1183. %% save the error info so it doesn't get printed twice
  1184. ct_util:set_testdata_async({{error_in_suite,Mod},
  1185. ExitReason});
  1186. _ExitReason ->
  1187. ct_util:delete_testdata({error_in_suite,Mod})
  1188. end,
  1189. Reason = list_to_atom(atom_to_list(Mod)++":all/0 failed"),
  1190. %% this makes test_server call error_in_suite as first
  1191. %% (and only) test case so we can report Reason properly
  1192. [{?MODULE,error_in_suite,[[{error,Reason}]]}];
  1193. {error,What} ->
  1194. [{?MODULE,error_in_suite,[[{error,What}]]}]
  1195. end.
  1196. %%!============================================================
  1197. %%! The support for sequences by means of using sequences/0
  1198. %%! will be removed in OTP R15. The code below is only kept
  1199. %%! for backwards compatibility. From OTP R13 groups with
  1200. %%! sequence property should be used instead!
  1201. %%!============================================================
  1202. %%!============================================================
  1203. %%! START OF DEPRECATED SUPPORT FOR SEQUENCES --->
  1204. get_seq(Mod, Func) ->
  1205. case ct_util:read_suite_data({seq,Mod,Func}) of
  1206. undefined ->
  1207. case catch apply(Mod,sequences,[]) of
  1208. {'EXIT',_} ->
  1209. [];
  1210. Seqs ->
  1211. case lists:keysearch(Func,1,Seqs) of
  1212. {value,{Func,SeqTCs}} ->
  1213. case catch save_seq(Mod,Func,SeqTCs) of
  1214. {error,What} ->
  1215. [{?MODULE,error_in_suite,[[{error,What}]]}];
  1216. _ ->
  1217. SeqTCs
  1218. end;
  1219. false ->
  1220. []
  1221. end
  1222. end;
  1223. TCs when is_list(TCs) ->
  1224. TCs;
  1225. _ ->
  1226. []
  1227. end.
  1228. save_seqs(Mod,AllTCs) ->
  1229. case lists:keymember(sequence,1,AllTCs) of
  1230. true ->
  1231. case catch apply(Mod,sequences,[]) of
  1232. {'EXIT',_} ->
  1233. Reason = list_to_atom(atom_to_list(Mod)++
  1234. ":sequences/0 is missing"),
  1235. throw({error,Reason});
  1236. Seqs ->
  1237. save_seqs(Mod,AllTCs,Seqs,AllTCs)
  1238. end;
  1239. false ->
  1240. AllTCs
  1241. end.
  1242. save_seqs(Mod,[{sequence,Seq}|TCs],Seqs,All) ->
  1243. case lists:keysearch(Seq,1,Seqs) of
  1244. {value,{Seq,SeqTCs}} ->
  1245. save_seq(Mod,Seq,SeqTCs,All),
  1246. [Seq|save_seqs(Mod,TCs,Seqs,All)];
  1247. false ->
  1248. Reason = list_to_atom(
  1249. atom_to_list(Seq)++" is missing in "++
  1250. atom_to_list(Mod)),
  1251. throw({error,Reason})
  1252. end;
  1253. save_seqs(Mod,[TC|TCs],Seqs,All) ->
  1254. [TC|save_seqs(Mod,TCs,Seqs,All)];
  1255. save_seqs(_,[],_,_) ->
  1256. [].
  1257. save_seq(Mod,Seq,SeqTCs) ->
  1258. save_seq(Mod,Seq,SeqTCs,apply(Mod,all,[])).
  1259. save_seq(Mod,Seq,SeqTCs,All) ->
  1260. check_private(Seq,SeqTCs,All),
  1261. check_multiple(Mod,Seq,SeqTCs),
  1262. ct_util:save_suite_data({seq,Mod,Seq},SeqTCs),
  1263. lists:foreach(fun(TC) ->
  1264. ct_util:save_suite_data({seq,Mod,TC},Seq)
  1265. end, SeqTCs).
  1266. check_private(Seq,TCs,All) ->
  1267. Bad = lists:filter(fun(TC) -> lists:member(TC,All) end, TCs),
  1268. if Bad /= [] ->
  1269. Reason = io_lib:format("regular test cases not allowed in sequence ~tp: "
  1270. "~tp",[Seq,Bad]),
  1271. throw({error,list_to_atom(lists:flatten(Reason))});
  1272. true ->
  1273. ok
  1274. end.
  1275. check_multiple(Mod,Seq,TCs) ->
  1276. Bad = lists:filter(fun(TC) ->
  1277. case ct_util:read_suite_data({seq,Mod,TC}) of
  1278. Seq1 when Seq1 /= undefined, Seq1 /= Seq ->
  1279. true;
  1280. _ -> false
  1281. end
  1282. end,TCs),
  1283. if Bad /= [] ->
  1284. Reason = io_lib:format("test cases found in multiple sequences: "
  1285. "~tp",[Bad]),
  1286. throw({error,list_to_atom(lists:flatten(Reason))});
  1287. true ->
  1288. ok
  1289. end.
  1290. %%! <--- END OF DEPRECATED SUPPORT FOR SEQUENCES
  1291. %%!============================================================
  1292. %% let test_server call this function as a testcase only so that
  1293. %% the user may see info about what's missing in the suite
  1294. error_in_suite(Config) ->
  1295. Reason = test_server:lookup_config(error,Config),
  1296. exit(Reason).
  1297. %% if init_per_suite and end_per_suite are missing in the suite,
  1298. %% these will be called instead (without any trace of them in the
  1299. %% log files), only so that it's possible to call hook functions
  1300. %% for configuration
  1301. init_per_suite(Config) ->
  1302. Config.
  1303. end_per_suite(_Config) ->
  1304. ok.
  1305. %% if the group config functions are missing in the suite,
  1306. %% use these instead
  1307. init_per_group(GroupName, Config) ->
  1308. ct:comment(io_lib:format("start of ~tp", [GroupName])),
  1309. ct_logs:log("TEST INFO", "init_per_group/2 for ~tw missing "
  1310. "in suite, using default.",
  1311. [GroupName]),
  1312. Config.
  1313. end_per_group(GroupName, _) ->
  1314. ct:comment(io_lib:format("end of ~tp", [GroupName])),
  1315. ct_logs:log("TEST INFO", "end_per_group/2 for ~tw missing "
  1316. "in suite, using default.",
  1317. [GroupName]),
  1318. ok.
  1319. %%%-----------------------------------------------------------------
  1320. %%% -spec report(What,Data) -> ok
  1321. report(What,Data) ->
  1322. case What of
  1323. loginfo ->
  1324. %% logfiles and direcories have been created for a test and the
  1325. %% top level test index page needs to be refreshed
  1326. TestName = filename:basename(?val(topdir, Data), ".logs"),
  1327. RunDir = ?val(rundir, Data),
  1328. _ = ct_logs:make_all_suites_index({TestName,RunDir}),
  1329. ok;
  1330. tests_start ->
  1331. ok;
  1332. tests_done ->
  1333. ok;
  1334. severe_error ->
  1335. ct_event:sync_notify(#event{name=What,
  1336. node=node(),
  1337. data=Data}),
  1338. ct_util:set_testdata({What,Data}),
  1339. ok;
  1340. tc_start ->
  1341. %% Data = {{Suite,{Func,GroupName}},LogFileName}
  1342. Data1 = case Data of
  1343. {{Suite,{Func,undefined}},LFN} -> {{Suite,Func},LFN};
  1344. _ -> Data
  1345. end,
  1346. ct_event:sync_notify(#event{name=tc_logfile,
  1347. node=node(),
  1348. data=Data1}),
  1349. ok;
  1350. tc_done ->
  1351. {Suite,{Func,GrName},Result} = Data,
  1352. FuncSpec = if GrName == undefined -> Func;
  1353. true -> {Func,GrName}
  1354. end,
  1355. %% Register the group leader for the process calling the report
  1356. %% function, making it possible for a hook function to print
  1357. %% in the test case log file
  1358. ReportingPid = self(),
  1359. ct_logs:register_groupleader(ReportingPid, group_leader()),
  1360. case Result of
  1361. {failed, Reason} ->
  1362. ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason});
  1363. {skipped,{failed,{_,init_per_testcase,_}}=Reason} ->
  1364. ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
  1365. {skipped,{require_failed,_}=Reason} ->
  1366. ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
  1367. {skipped,Reason} ->
  1368. ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason});
  1369. {auto_skipped,Reason} ->
  1370. ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
  1371. _Else ->
  1372. ok
  1373. end,
  1374. ct_logs:unregister_groupleader(ReportingPid),
  1375. case {Func,Result} of
  1376. {error_in_suite,_} when Suite == ?MODULE ->
  1377. ok;
  1378. {init_per_suite,_} ->
  1379. ok;
  1380. {end_per_suite,_} ->
  1381. ok;
  1382. {init_per_group,_} ->
  1383. ok;
  1384. {end_per_group,_} ->
  1385. ok;
  1386. {_,ok} ->
  1387. add_to_stats(ok);
  1388. {_,{skipped,{failed,{_,init_per_testcase,_}}}} ->
  1389. add_to_stats(auto_skipped);
  1390. {_,{skipped,{require_failed,_}}} ->
  1391. add_to_stats(auto_skipped);
  1392. {_,{skipped,{timetrap_error,_}}} ->
  1393. add_to_stats(auto_skipped);
  1394. {_,{skipped,{invalid_time_format,_}}} ->
  1395. add_to_stats(auto_skipped);
  1396. {_,{skipped,_}} ->
  1397. add_to_stats(user_skipped);
  1398. {_,{auto_skipped,_}} ->
  1399. add_to_stats(auto_skipped);
  1400. {_,{SkipOrFail,_Reason}} ->
  1401. add_to_stats(SkipOrFail)
  1402. end;
  1403. tc_user_skip ->
  1404. %% test case or config function specified as skipped in testspe

Large files files are truncated, but you can click here to view the full file