PageRenderTime 112ms CodeModel.GetById 2ms app.highlight 96ms RepoModel.GetById 1ms app.codeStats 1ms

/lib/common_test/src/ct_framework.erl

https://github.com/bsmr-erlang/otp
Erlang | 1701 lines | 1385 code | 109 blank | 207 comment | 66 complexity | be777359b70302d2fd76bfeb19a9bd7e MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1%%
   2%% %CopyrightBegin%
   3%%
   4%% Copyright Ericsson AB 2004-2018. All Rights Reserved.
   5%%
   6%% Licensed under the Apache License, Version 2.0 (the "License");
   7%% you may not use this file except in compliance with the License.
   8%% You may obtain a copy of the License at
   9%%
  10%%     http://www.apache.org/licenses/LICENSE-2.0
  11%%
  12%% Unless required by applicable law or agreed to in writing, software
  13%% distributed under the License is distributed on an "AS IS" BASIS,
  14%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15%% See the License for the specific language governing permissions and
  16%% limitations under the License.
  17%%
  18%% %CopyrightEnd%
  19%%
  20
  21%%% Common Test Framework callback module.
  22%%%
  23%%% This module exports framework callback functions which are
  24%%% called from the test_server.
  25
  26-module(ct_framework).
  27
  28-export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]).
  29-export([report/2, warn/1, error_notification/4]).
  30
  31-export([get_log_dir/0, get_logopts/0, format_comment/1, get_html_wrapper/4]).
  32
  33-export([error_in_suite/1, init_per_suite/1, end_per_suite/1,
  34	 init_per_group/2, end_per_group/2]).
  35
  36-include("ct.hrl").
  37-include("ct_event.hrl").
  38-include("ct_util.hrl").
  39
  40-define(val(Key, List), proplists:get_value(Key, List)). 
  41-define(val(Key, List, Def), proplists:get_value(Key, List, Def)).
  42-define(rev(L), lists:reverse(L)).
  43
  44%%%-----------------------------------------------------------------
  45%%% -spec init_tc(Mod,Func,Args) -> {ok,NewArgs} | {error,Reason} |
  46%%%         {skip,Reason} | {auto_skip,Reason}
  47%%%      Mod = atom()
  48%%%      Func = atom()
  49%%%      Args = list()
  50%%%      NewArgs = list()
  51%%%      Reason = term()
  52%%%
  53%%% Test server framework callback, called by the test_server
  54%%% when a new test case is started.
  55init_tc(_,{end_per_testcase_not_run,_},[Config]) ->
  56    %% Testcase is completed (skipped or failed), but end_per_testcase
  57    %% is not run - don't call pre-hook.
  58    {ok,[Config]};
  59init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
  60    %% in case Mod == ct_framework, lookup the suite name
  61    Suite = get_suite_name(Mod, Config),
  62    case ct_hooks:init_tc(Suite,EPTC,Config) of
  63	NewConfig when is_list(NewConfig) ->
  64	    {ok,[NewConfig]};
  65	Other->
  66	    Other
  67    end;
  68
  69init_tc(Mod,Func0,Args) ->
  70    %% in case Mod == ct_framework, lookup the suite name
  71    Suite = get_suite_name(Mod, Args),
  72    {Func,HookFunc} = case Func0 of
  73			  {init_per_testcase,F} -> {F,Func0};
  74			  _                     -> {Func0,Func0}
  75		      end,
  76
  77    %% check if previous testcase was interpreted and has left
  78    %% a "dead" trace window behind - if so, kill it
  79    case ct_util:get_testdata(interpret) of
  80	{What,kill,{TCPid,AttPid}} ->
  81	    ct_util:kill_attached(TCPid,AttPid),
  82	    ct_util:set_testdata({interpret,{What,kill,{undefined,undefined}}});
  83	_ ->
  84	    ok
  85    end,
  86
  87    case Func=/=end_per_suite
  88	andalso Func=/=end_per_group
  89	andalso ct_util:get_testdata(skip_rest) of
  90	true ->
  91            initialize(false,Mod,Func,Args),
  92	    {auto_skip,"Repeated test stopped by force_stop option"};
  93	_ ->
  94	    case ct_util:get_testdata(curr_tc) of
  95		{Suite,{suite0_failed,{require,Reason}}} ->
  96                    initialize(false,Mod,Func,Args),
  97		    {auto_skip,{require_failed_in_suite0,Reason}};
  98		{Suite,{suite0_failed,_}=Failure} ->
  99                    initialize(false,Mod,Func,Args),
 100		    {fail,Failure};
 101		_ ->
 102		    ct_util:update_testdata(curr_tc,
 103					    fun(undefined) ->
 104						    [{Suite,Func}];
 105					       (Running) ->
 106						    [{Suite,Func}|Running]
 107					    end, [create]),
 108		    case ct_util:read_suite_data({seq,Suite,Func}) of
 109			undefined ->
 110			    init_tc1(Mod,Suite,Func,HookFunc,Args);
 111			Seq when is_atom(Seq) ->
 112			    case ct_util:read_suite_data({seq,Suite,Seq}) of
 113				[Func|TCs] -> % this is the 1st case in Seq
 114				    %% make sure no cases in this seq are
 115				    %% marked as failed from an earlier execution
 116				    %% in the same suite
 117				    lists:foreach(
 118				      fun(TC) ->
 119					      ct_util:save_suite_data(
 120						{seq,Suite,TC},
 121						Seq)
 122				      end, TCs);
 123				_ ->
 124				    ok
 125			    end,
 126			    init_tc1(Mod,Suite,Func,HookFunc,Args);
 127			{failed,Seq,BadFunc} ->
 128                            initialize(false,Mod,Func,Args),
 129                            {auto_skip,{sequence_failed,Seq,BadFunc}}
 130		    end
 131	    end
 132    end.
 133
 134init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) ->
 135    initialize(false,?MODULE,error_in_suite),
 136    _ = ct_suite_init(?MODULE,error_in_suite,[],Config0),
 137    case ?val(error,Config0) of
 138	undefined ->
 139	    {fail,"unknown_error_in_suite"};
 140	Reason ->
 141	    {fail,Reason}
 142    end;
 143
 144init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->
 145    Config1 = 
 146	case ct_util:read_suite_data(last_saved_config) of
 147	    {{Suite,LastFunc},SavedConfig} ->	% last testcase
 148		[{saved_config,{LastFunc,SavedConfig}} | 
 149		 lists:keydelete(saved_config,1,Config0)];
 150	    {{LastSuite,InitOrEnd},
 151	     SavedConfig} when InitOrEnd == init_per_suite ;
 152			       InitOrEnd == end_per_suite ->
 153		%% last suite
 154		[{saved_config,{LastSuite,SavedConfig}} | 
 155		 lists:keydelete(saved_config,1,Config0)];
 156	    undefined ->
 157		lists:keydelete(saved_config,1,Config0)
 158	end,
 159    ct_util:delete_suite_data(last_saved_config),
 160    Config = lists:keydelete(watchdog,1,Config1),
 161
 162    if Func == init_per_suite ->
 163	    %% delete all default values used in previous suite
 164	    ct_config:delete_default_config(suite),
 165	    %% release all name -> key bindings (once per suite)
 166	    ct_config:release_allocated();
 167       Func /= init_per_suite ->
 168	    ok
 169    end,
 170
 171    GroupPath = ?val(tc_group_path, Config, []),
 172    AllGroups =	[?val(tc_group_properties, Config, []) | GroupPath],
 173
 174    %% clear all config data default values set by previous
 175    %% testcase info function (these should only survive the
 176    %% testcase, not the whole suite)
 177    FuncSpec = group_or_func(Func,Config0),
 178    HookFunc1 =
 179	if is_tuple(FuncSpec) ->		% group
 180	    FuncSpec;
 181	   true ->
 182		ct_config:delete_default_config(testcase),
 183		HookFunc
 184	end,
 185    case add_defaults(Mod,Func,AllGroups) of
 186	Error = {suite0_failed,_} ->
 187	    initialize(false,Mod,FuncSpec),
 188	    ct_util:set_testdata({curr_tc,{Suite,Error}}),
 189	    {error,Error};
 190	Error = {group0_failed,_} ->
 191	    initialize(false,Mod,FuncSpec),
 192	    {auto_skip,Error};
 193	Error = {testcase0_failed,_} ->
 194	    initialize(false,Mod,FuncSpec),
 195	    {auto_skip,Error};
 196	{SuiteInfo,MergeResult} ->
 197	    case MergeResult of
 198		{error,Reason} ->
 199		    initialize(false,Mod,FuncSpec),
 200		    {fail,Reason};
 201		_ ->
 202		    init_tc2(Mod,Suite,Func,HookFunc1,
 203			     SuiteInfo,MergeResult,Config)
 204	    end
 205    end;
 206
 207init_tc1(_Mod,_Suite,_Func,_HookFunc,Args) ->
 208    {ok,Args}.
 209
 210init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
 211    %% timetrap must be handled before require
 212    MergedInfo = timetrap_first(MergeResult, [], []),
 213    %% tell logger to use specified style sheet
 214    _ = case lists:keysearch(stylesheet,1,MergeResult++Config) of
 215	{value,{stylesheet,SSFile}} ->
 216	    ct_logs:set_stylesheet(Func,add_data_dir(SSFile,Config));
 217	_ ->
 218	    case ct_util:get_testdata(stylesheet) of
 219		undefined ->
 220		    ct_logs:clear_stylesheet(Func);
 221		SSFile ->
 222		    ct_logs:set_stylesheet(Func,SSFile)
 223	    end
 224    end,
 225    %% suppress output for connections (Conns is a 
 226    %% list of {Type,Bool} tuples, e.g. {telnet,true}),		
 227    case ct_util:get_overridden_silenced_connections() of
 228	undefined ->
 229	    case lists:keysearch(silent_connections,1,MergeResult++Config) of
 230		{value,{silent_connections,Conns}} ->
 231		    ct_util:silence_connections(Conns);
 232		_ ->
 233		    ok
 234	    end;
 235	Conns ->
 236	    ct_util:silence_connections(Conns)
 237    end,
 238    FuncSpec = group_or_func(Func,Config),
 239    initialize((Func==init_per_suite),Mod,FuncSpec),
 240
 241    case catch configure(MergedInfo,MergedInfo,SuiteInfo,
 242			 FuncSpec,[],Config) of
 243	{suite0_failed,Reason} ->
 244	    ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,
 245						{require,Reason}}}}),
 246	    {auto_skip,{require_failed_in_suite0,Reason}};
 247	{error,Reason} ->
 248	    {auto_skip,{require_failed,Reason}};
 249	{'EXIT',Reason} ->
 250	    {fail,Reason};
 251	{ok,PostInitHook,Config1} ->
 252	    case get('$test_server_framework_test') of
 253		undefined ->
 254		    ct_suite_init(Suite,HookFunc,PostInitHook,Config1);
 255		Fun ->
 256		    PostInitHookResult = do_post_init_hook(PostInitHook,
 257							   Config1),
 258		    case Fun(init_tc, [PostInitHookResult ++ Config1]) of
 259			NewConfig when is_list(NewConfig) ->
 260			    {ok,NewConfig};
 261			Else ->
 262			    Else
 263		    end
 264	    end
 265    end.
 266
 267initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) ->
 268    initialize(RefreshLogs,Mod,group_or_func(Func,Config));
 269initialize(RefreshLogs,Mod,Func,_) ->
 270    initialize(RefreshLogs,Mod,Func).
 271
 272initialize(RefreshLogs,Mod,FuncSpec) ->
 273    ct_logs:init_tc(RefreshLogs),
 274    ct_event:notify(#event{name=tc_start,
 275			   node=node(),
 276			   data={Mod,FuncSpec}}).
 277
 278
 279ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->
 280    case ct_hooks:init_tc(Suite,HookFunc,Config) of
 281	NewConfig when is_list(NewConfig) ->
 282	    PostInitHookResult = do_post_init_hook(PostInitHook,NewConfig),
 283	    {ok, [PostInitHookResult ++ NewConfig]};
 284	Else ->
 285	    Else
 286    end.
 287
 288do_post_init_hook(PostInitHook,Config) ->
 289    lists:flatmap(fun({Tag,Fun}) ->
 290			  case lists:keysearch(Tag,1,Config) of
 291			      {value,_} ->
 292				  [];
 293			      false -> 
 294				  case Fun() of
 295				      {error,_} -> [];
 296				      Result    -> [{Tag,Result}]
 297				  end
 298			  end
 299		  end, PostInitHook).
 300
 301add_defaults(Mod,Func, GroupPath) ->
 302    Suite = get_suite_name(Mod, GroupPath),
 303    case (catch Suite:suite()) of
 304	{'EXIT',{undef,_}} ->
 305	    SuiteInfo = merge_with_suite_defaults(Suite,[]),
 306	    SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks],
 307	    case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of
 308		Error = {group0_failed,_} -> Error;
 309		Error = {testcase0_failed,_} -> Error;
 310		Error = {error,_} -> {SuiteInfo,Error};
 311		MergedInfo -> {SuiteInfo,MergedInfo}
 312	    end;
 313	{'EXIT',Reason} ->
 314	    ErrStr = io_lib:format("~n*** ERROR *** "
 315				   "~w:suite/0 failed: ~tp~n",
 316				   [Suite,Reason]),
 317	    io:format("~ts", [ErrStr]),
 318	    io:format(?def_gl, "~ts", [ErrStr]),
 319	    {suite0_failed,{exited,Reason}};
 320	SuiteInfo when is_list(SuiteInfo) ->
 321	    case lists:all(fun(E) when is_tuple(E) -> true;
 322			      (_) -> false
 323			   end, SuiteInfo) of
 324		true ->
 325		    SuiteInfo1 = merge_with_suite_defaults(Suite, SuiteInfo),
 326		    SuiteInfoNoCTH = [I || I <- SuiteInfo1,
 327					   element(1,I) =/= ct_hooks],
 328		    case add_defaults1(Mod,Func, GroupPath,
 329				       SuiteInfoNoCTH) of
 330			Error = {group0_failed,_} -> Error;
 331			Error = {testcase0_failed,_} -> Error;
 332			Error = {error,_} -> {SuiteInfo1,Error};
 333			MergedInfo -> {SuiteInfo1,MergedInfo}
 334		    end;
 335		false ->
 336		    ErrStr = io_lib:format("~n*** ERROR *** "
 337					   "Invalid return value from "
 338					   "~w:suite/0: ~tp~n",
 339					   [Suite,SuiteInfo]),
 340		    io:format("~ts", [ErrStr]),
 341		    io:format(?def_gl, "~ts", [ErrStr]),
 342		    {suite0_failed,bad_return_value}
 343	    end;
 344	SuiteInfo ->
 345	    ErrStr = io_lib:format("~n*** ERROR *** "
 346				   "Invalid return value from "
 347				   "~w:suite/0: ~tp~n", [Suite,SuiteInfo]),
 348	    io:format("~ts", [ErrStr]),
 349	    io:format(?def_gl, "~ts", [ErrStr]),
 350	    {suite0_failed,bad_return_value}
 351    end.
 352
 353add_defaults1(Mod,Func, GroupPath, SuiteInfo) ->
 354    Suite = get_suite_name(Mod, GroupPath),
 355    %% GroupPathInfo (for subgroup on level X) =
 356    %%     [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo]
 357    GroupPathInfo =  
 358	lists:map(fun(GroupProps) ->
 359			  case ?val(name, GroupProps) of
 360			      undefined ->
 361				  [];
 362			      Name ->
 363				  case catch Suite:group(Name) of
 364				      GrInfo when is_list(GrInfo) -> GrInfo;
 365				      {'EXIT',{undef,_}} -> [];
 366				      BadGr0 -> {error,BadGr0,Name}
 367				  end
 368			  end
 369		  end, GroupPath),
 370    case lists:keysearch(error, 1, GroupPathInfo) of
 371	{value,{error,BadGr0Val,GrName}} ->
 372	    Gr0ErrStr = io_lib:format("~n*** ERROR *** "
 373				      "Invalid return value from "
 374				      "~w:group(~tw): ~tp~n",
 375				      [Mod,GrName,BadGr0Val]),
 376	    io:format("~ts", [Gr0ErrStr]),
 377	    io:format(?def_gl, "~ts", [Gr0ErrStr]),
 378	    {group0_failed,bad_return_value};
 379	_ ->
 380	    Args = if Func == init_per_group ; Func == end_per_group ->
 381			   [?val(name, hd(GroupPath))];
 382		      true ->
 383			   []
 384		   end,
 385	    TestCaseInfo =
 386		case catch apply(Mod,Func,Args) of
 387		    TCInfo when is_list(TCInfo) -> TCInfo;
 388		    {'EXIT',{undef,_}} -> [];
 389		    BadTC0 -> {error,BadTC0}
 390		end,
 391	    
 392	    case TestCaseInfo of
 393		{error,BadTC0Val} ->
 394		    TC0ErrStr = io_lib:format("~n*** ERROR *** "
 395					      "Invalid return value from "
 396					      "~w:~tw/0: ~tp~n",
 397					      [Mod,Func,BadTC0Val]),
 398		    io:format("~ts", [TC0ErrStr]),
 399		    io:format(?def_gl, "~ts", [TC0ErrStr]),
 400		    {testcase0_failed,bad_return_value};
 401		_ ->
 402		    %% let test case info (also for all config funcs) override
 403		    %% group info, and lower level group info override higher
 404		    %% level info
 405		    TCAndGroupInfo =
 406			[TestCaseInfo | remove_info_in_prev(TestCaseInfo,
 407							    GroupPathInfo)],
 408		    %% find and save require terms found in suite info
 409		    SuiteReqs = 
 410			[SDDef || SDDef <- SuiteInfo,
 411				  ((require == element(1,SDDef))
 412				   or (default_config == element(1,SDDef)))],
 413		    case check_for_clashes(TestCaseInfo, GroupPathInfo,
 414					   SuiteReqs) of
 415			[] ->
 416			    add_defaults2(Mod,Func, TCAndGroupInfo,
 417					  SuiteInfo,SuiteReqs);
 418			Clashes ->
 419			    {error,{config_name_already_in_use,Clashes}}
 420		    end
 421	    end
 422    end.
 423
 424get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] ->
 425    get_suite_name(?MODULE, Cfg);
 426
 427get_suite_name(?MODULE, Cfg) when is_list(Cfg), Cfg /= [] ->
 428    case ?val(tc_group_properties, Cfg) of
 429	undefined ->
 430	    case ?val(suite, Cfg) of
 431		undefined -> ?MODULE;
 432		Suite -> Suite
 433	    end;
 434	GrProps ->
 435	    case ?val(suite, GrProps) of
 436		undefined -> ?MODULE;
 437		Suite -> Suite
 438	    end
 439    end;
 440get_suite_name(Mod, _) ->
 441    Mod.
 442
 443%% Check that alias names are not already in use
 444check_for_clashes(TCInfo, [CurrGrInfo|Path], SuiteInfo) ->
 445    ReqNames = fun(Info) -> [element(2,R) || R <- Info,
 446					     size(R) == 3,
 447					     require == element(1,R)]
 448	       end,
 449    ExistingNames = lists:flatten([ReqNames(L)  || L <- [SuiteInfo|Path]]),
 450    CurrGrReqNs = ReqNames(CurrGrInfo),
 451    GrClashes = [Name || Name <- CurrGrReqNs,
 452			 true == lists:member(Name, ExistingNames)],
 453    AllReqNs = CurrGrReqNs ++ ExistingNames,
 454    TCClashes = [Name || Name <- ReqNames(TCInfo),
 455			 true == lists:member(Name, AllReqNs)],
 456    TCClashes ++ GrClashes.
 457
 458%% Delete the info terms in Terms from all following info lists
 459remove_info_in_prev(Terms, [[] | Rest]) ->
 460    [[] | remove_info_in_prev(Terms, Rest)];
 461remove_info_in_prev(Terms, [Info | Rest]) ->
 462    UniqueInInfo = [U || U <- Info,
 463			  ((timetrap == element(1,U)) and
 464			   (not lists:keymember(timetrap,1,Terms))) or 
 465			  ((require == element(1,U)) and
 466			   (not lists:member(U,Terms))) or
 467			  ((default_config == element(1,U)) and
 468                           (not keysmember([default_config,1,
 469					    element(2,U),2], Terms)))],
 470    OtherTermsInInfo = [T || T <- Info,
 471			     timetrap /= element(1,T),
 472			     require /= element(1,T),
 473			     default_config /= element(1,T),
 474			     false == lists:keymember(element(1,T),1,
 475						      Terms)],
 476    KeptInfo = UniqueInInfo ++ OtherTermsInInfo,
 477    [KeptInfo | remove_info_in_prev(Terms ++ KeptInfo, Rest)];
 478remove_info_in_prev(_, []) ->
 479    [].
 480
 481keysmember([Key,Pos|Next], List) ->
 482    case [Elem || Elem <- List, Key == element(Pos,Elem)] of
 483	[]    -> false;
 484	Found -> keysmember(Next, Found)
 485    end;
 486keysmember([], _) -> true.
 487
 488
 489add_defaults2(_Mod,init_per_suite, IPSInfo, SuiteInfo,SuiteReqs) ->
 490    Info = lists:flatten([IPSInfo, SuiteReqs]),
 491    lists:flatten([Info,remove_info_in_prev(Info, [SuiteInfo])]);
 492
 493add_defaults2(_Mod,init_per_group, IPGAndGroupInfo, SuiteInfo,SuiteReqs) ->
 494    SuiteInfo1 =
 495	remove_info_in_prev(lists:flatten([IPGAndGroupInfo,
 496					   SuiteReqs]), [SuiteInfo]),
 497    %% don't require terms in prev groups (already processed)
 498    case IPGAndGroupInfo of
 499	[IPGInfo] ->
 500	    lists:flatten([IPGInfo,SuiteInfo1]);
 501	[IPGInfo | [CurrGroupInfo | PrevGroupInfo]] ->
 502	    PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
 503	    lists:flatten([IPGInfo,CurrGroupInfo,PrevGroupInfo1,
 504			   SuiteInfo1])
 505    end;
 506
 507add_defaults2(_Mod,_Func, TCAndGroupInfo, SuiteInfo,SuiteReqs) ->
 508    %% Include require elements from test case info and current group,
 509    %% but not from previous groups or suite/0 (since we've already required
 510    %% those vars). Let test case info elements override group and suite
 511    %% info elements.
 512    SuiteInfo1 = remove_info_in_prev(lists:flatten([TCAndGroupInfo,
 513						    SuiteReqs]), [SuiteInfo]),
 514    %% don't require terms in prev groups (already processed)
 515    case TCAndGroupInfo of
 516	[TCInfo] ->
 517	    lists:flatten([TCInfo,SuiteInfo1]);
 518	[TCInfo | [CurrGroupInfo | PrevGroupInfo]] ->
 519	    PrevGroupInfo1 = delete_require_terms(PrevGroupInfo),
 520	    lists:flatten([TCInfo,CurrGroupInfo,PrevGroupInfo1,
 521			   SuiteInfo1])
 522    end.
 523
 524delete_require_terms([Info | Prev]) ->
 525    Info1 = [T || T <- Info, 
 526		  require /= element(1,T),
 527		  default_config /= element(1,T)],
 528    [Info1 | delete_require_terms(Prev)];
 529delete_require_terms([]) ->
 530    [].
 531
 532merge_with_suite_defaults(Mod,SuiteInfo) ->
 533    case lists:keysearch(suite_defaults,1,Mod:module_info(attributes)) of
 534	{value,{suite_defaults,Defaults}} ->
 535	    SDReqs =
 536		[SDDef || SDDef <- Defaults,
 537			  require == element(1,SDDef),
 538			  false == lists:keymember(element(2,SDDef),2,
 539						   SuiteInfo)],
 540	    SuiteInfo ++ SDReqs ++
 541		[SDDef || SDDef <- Defaults,
 542			  require /= element(1,SDDef),
 543			  false == lists:keymember(element(1,SDDef),1,
 544						   SuiteInfo)];		    
 545	false ->
 546	    SuiteInfo
 547    end.
 548
 549timetrap_first([Trap = {timetrap,_} | Rest],Info,Found) ->
 550    timetrap_first(Rest,Info,[Trap | Found]);
 551timetrap_first([Other | Rest],Info,Found) ->
 552    timetrap_first(Rest,[Other | Info],Found);
 553timetrap_first([],Info,[]) ->
 554    [{timetrap,{minutes,30}} | ?rev(Info)];
 555timetrap_first([],Info,Found) ->
 556    ?rev(Found) ++ ?rev(Info).
 557
 558configure([{require,Required}|Rest],
 559	  Info,SuiteInfo,Scope,PostInitHook,Config) ->
 560    case ct:require(Required) of
 561	ok ->
 562	    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 563	Error = {error,Reason} ->
 564	    case required_default('_UNDEF',Required,Info,
 565				  SuiteInfo,Scope) of
 566		ok ->
 567		    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 568		_ ->
 569		    case lists:keymember(Required,2,SuiteInfo) of
 570			true ->
 571			    {suite0_failed,Reason};
 572			false ->
 573			    Error
 574		    end
 575	    end
 576    end;
 577configure([{require,Name,Required}|Rest],
 578	  Info,SuiteInfo,Scope,PostInitHook,Config) ->
 579    case ct:require(Name,Required) of
 580	ok ->
 581	    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 582	Error = {error,Reason} ->
 583	    case required_default(Name,Required,Info,SuiteInfo,Scope) of
 584		ok ->
 585		    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 586		_ ->
 587		    case lists:keymember(Name,2,SuiteInfo) of
 588			true -> 
 589			    {suite0_failed,Reason};
 590			false ->
 591			    Error
 592		    end
 593	    end
 594    end;
 595configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
 596    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 597configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
 598    PostInitHook1 = 
 599	[{watchdog,fun() -> case test_server:get_timetrap_info() of
 600				undefined ->
 601				    test_server:timetrap(Time);
 602				_ ->
 603				    {error,already_set}
 604			    end
 605		   end} | PostInitHook],
 606    configure(Rest,Info,SuiteInfo,Scope,PostInitHook1,Config);
 607configure([{ct_hooks,Hook}|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
 608    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,[{ct_hooks,Hook}|Config]);
 609configure([_|Rest],Info,SuiteInfo,Scope,PostInitHook,Config) ->
 610    configure(Rest,Info,SuiteInfo,Scope,PostInitHook,Config);
 611configure([],_,_,_,PostInitHook,Config) ->
 612    {ok,PostInitHook,Config}.
 613
 614%% the require element in Info may come from suite/0 and
 615%% should be scoped 'suite', or come from the group info
 616%% function and be scoped 'group', or come from the testcase
 617%% info function and then be scoped 'testcase'
 618
 619required_default(Name,Key,Info,_,init_per_suite) ->
 620    try_set_default(Name,Key,Info,suite);
 621required_default(Name,Key,Info,_,{init_per_group,GrName,_}) ->
 622    try_set_default(Name,Key,Info,{group,GrName});
 623required_default(Name,Key,Info,_,_FuncSpec) ->
 624    try_set_default(Name,Key,Info,testcase).
 625
 626try_set_default(Name,Key,Info,Where) ->
 627    CfgElems = 
 628	case lists:keysearch(Name,1,Info) of
 629	    {value,{Name,Val}} ->
 630		[Val];
 631	    false ->
 632		case catch [{Key,element(3,Elem)} || Elem <- Info,
 633						     element(1,Elem)==default_config,
 634						     element(2,Elem)==Key] of
 635		    {'EXIT',_} -> [];
 636		    Result -> Result
 637		end
 638	end,
 639    case {Name,CfgElems} of
 640	{_,[]} -> 
 641	    no_default;
 642	{'_UNDEF',_} ->
 643	    _ = [ct_config:set_default_config([CfgVal],Where) || CfgVal <- CfgElems],
 644	    ok;
 645	_ ->
 646	    _ = [ct_config:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems],
 647	    ok
 648    end.
 649	    
 650
 651%%%-----------------------------------------------------------------
 652%%% -spec end_tc(Mod,Func,Args) -> {ok,NewArgs}| {error,Reason} |
 653%%%         {skip,Reason} | {auto_skip,Reason}
 654%%%      Mod = atom()
 655%%%      Func = atom()
 656%%%      Args = list()
 657%%%      NewArgs = list()
 658%%%      Reason = term()
 659%%%
 660%%% Test server framework callback, called by the test_server
 661%%% when a test case is finished.
 662end_tc(Mod, Fun, Args) ->
 663    %% Have to keep end_tc/3 for backwards compatibility issues
 664    end_tc(Mod, Fun, Args, '$end_tc_dummy').
 665end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) ->
 666    %% this clause gets called if CT has encountered a suite that
 667    %% can't be executed
 668    FinalNotify =
 669	case ct_hooks:end_tc(?MODULE, error_in_suite, Args, Result, Return) of
 670	    '$ct_no_change' ->
 671		Result;
 672	    HookResult ->
 673		HookResult
 674	end,
 675    Event = #event{name=tc_done,
 676		   node=node(),
 677		   data={?MODULE,error_in_suite,tag(FinalNotify)}},
 678    ct_event:sync_notify(Event),
 679    ok;
 680end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) ->
 681    end_tc(Mod,Func,TCPid,Result,Args,Return);
 682end_tc(Mod,Func,{Result,[Args]}, Return) ->
 683    end_tc(Mod,Func,self(),Result,Args,Return).
 684
 685end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
 686    case end_hook_func(IPTC,Return,IPTC) of
 687        undefined -> ok;
 688        _ ->
 689            %% in case Mod == ct_framework, lookup the suite name
 690            Suite = get_suite_name(Mod, Args),
 691            case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
 692                '$ct_no_change' ->
 693                    ok;
 694                HookResult ->
 695                    HookResult
 696            end
 697    end;
 698
 699end_tc(Mod,Func00,TCPid,Result,Args,Return) ->
 700    %% in case Mod == ct_framework, lookup the suite name
 701    Suite = get_suite_name(Mod, Args),
 702    {OnlyCleanup,Func0} =
 703        case Func00 of
 704            {cleanup,F0} ->
 705                {true,F0};
 706            _ ->
 707                {false,Func00}
 708        end,
 709    {Func,FuncSpec,HookFunc} =
 710        case Func0 of
 711            {end_per_testcase_not_run,F} ->
 712                %% Testcase is completed (skipped or failed), but
 713                %% end_per_testcase is not run - don't call post-hook.
 714                {F,F,undefined};
 715            {end_per_testcase,F} ->
 716                {F,F,Func0};
 717            _ ->
 718                FS = group_or_func(Func0,Args),
 719                HF = end_hook_func(Func0,Return,FS),
 720                {Func0,FS,HF}
 721        end,
 722
 723    test_server:timetrap_cancel(),
 724
 725    %% save the testcase process pid so that it can be used
 726    %% to look up the attached trace window later
 727    case ct_util:get_testdata(interpret) of
 728	{What,kill,_} ->
 729	    AttPid = ct_util:get_attached(self()),
 730	    ct_util:set_testdata({interpret,{What,kill,{self(),AttPid}}});
 731	_ ->
 732	    ok
 733    end,
 734    if Func == end_per_group; Func == end_per_suite ->
 735	    %% clean up any saved comments
 736	    ct_util:match_delete_testdata({comment,'_'});
 737       true ->
 738	    %% attemp to delete any saved comment for this TC
 739	    case process_info(TCPid, group_leader) of
 740		{group_leader,TCGL} ->
 741		    ct_util:delete_testdata({comment,TCGL});
 742		_ ->
 743		    ok
 744	    end
 745    end,
 746    ct_util:delete_suite_data(last_saved_config),
 747
 748    {Result1,FinalNotify} =
 749        case HookFunc of
 750            undefined ->
 751                {ok,Result};
 752            _ when OnlyCleanup ->
 753                {ok,Result};
 754            _ ->
 755                case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
 756                    '$ct_no_change' ->
 757                        {ok,Result};
 758                    HookResult ->
 759                        {HookResult,HookResult}
 760                end
 761        end,
 762    FinalResult =
 763	case get('$test_server_framework_test') of
 764            _ when OnlyCleanup ->
 765                Result1;
 766	    undefined ->
 767		%% send sync notification so that event handlers may print
 768		%% in the log file before it gets closed
 769		Event = #event{name=tc_done,
 770			       node=node(),
 771			       data={Mod,FuncSpec,
 772				     tag(FinalNotify)}},
 773		ct_event:sync_notify(Event),
 774		Result1;
 775	    Fun ->
 776		%% send sync notification so that event handlers may print
 777		%% in the log file before it gets closed
 778		Event = #event{name=tc_done,
 779			       node=node(),
 780			       data={Mod,FuncSpec,
 781				     tag({'$test_server_framework_test',
 782					  FinalNotify})}},
 783		ct_event:sync_notify(Event),
 784		Fun(end_tc, Return)
 785	end,    
 786
 787    case FuncSpec of
 788	{_,GroupName,_Props} ->
 789	    if Func == end_per_group ->
 790		    ct_config:delete_default_config({group,GroupName});
 791	       true -> ok
 792	    end,
 793	    case lists:keysearch(save_config,1,Args) of
 794		{value,{save_config,SaveConfig}} ->
 795		    ct_util:save_suite_data(last_saved_config,
 796					    {Suite,{group,GroupName}},
 797					    SaveConfig);
 798		false ->
 799		    ok
 800	    end;
 801	_ ->
 802	    case lists:keysearch(save_config,1,Args) of
 803		{value,{save_config,SaveConfig}} ->
 804		    ct_util:save_suite_data(last_saved_config,
 805					    {Suite,Func},SaveConfig);
 806		false ->
 807		    ok
 808	    end
 809    end,
 810
 811    ct_util:reset_silent_connections(),
 812
 813    %% reset the curr_tc state, or delete this TC from the list of
 814    %% executing cases (if in a parallel group)
 815    ClearCurrTC = fun(Running = [_,_|_]) ->
 816			  lists:keydelete(Func,2,Running);
 817		     ({_,{suite0_failed,_}}) ->
 818			  undefined;
 819		     ([{_,CurrTC}]) when CurrTC == Func ->
 820			  undefined;
 821		     (undefined) ->
 822			  undefined;
 823		     (Unexpected) ->
 824			  {error,{reset_curr_tc,{Mod,Func},Unexpected}}
 825		  end,
 826    case ct_util:update_testdata(curr_tc, ClearCurrTC) of
 827	{error,_} = ClearError ->
 828	    exit(ClearError);
 829	_ ->
 830	    ok
 831    end,
 832
 833    case FinalResult of
 834	{auto_skip,{sequence_failed,_,_}} ->
 835	    %% ct_logs:init_tc is never called for a skipped test case
 836	    %% in a failing sequence, so neither should end_tc	    
 837	    ok;
 838	_ ->
 839	    case ct_logs:end_tc(TCPid) of
 840		{error,Reason} ->
 841		    exit({error,{logger,Reason}});
 842		_ ->
 843		    ok
 844	    end
 845    end,
 846    case Func of
 847	end_per_suite -> 
 848	    ct_util:match_delete_suite_data({seq,Suite,'_'});
 849	_ -> 
 850	    ok
 851    end,
 852    FinalResult.	    
 853
 854%% This is to make sure that no post_init_per_* is ever called if the
 855%% corresponding pre_init_per_* was not called.
 856%% The skip or fail reasons are those that can be returned from
 857%% init_tc above in situations where we never came to call
 858%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call
 859%% ct_hooks:init_tc for init_per_suite, and thus we must not call
 860%% ct_hooks:end_tc for init_per_suite either.
 861end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) ->
 862    undefined;
 863end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) ->
 864    undefined;
 865end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) ->
 866    undefined;
 867end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_)
 868  when InfoFuncError==testcase0_failed;
 869       InfoFuncError==require_failed ->
 870    undefined;
 871end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_)
 872  when InfoFuncError==group0_failed;
 873       InfoFuncError==require_failed ->
 874    undefined;
 875end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) ->
 876    undefined;
 877end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) ->
 878    undefined;
 879end_hook_func(_,_,Default) ->
 880    Default.
 881
 882%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | 
 883%% {testcase_aborted,Reason} | testcase_aborted_or_killed | 
 884%% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} |
 885%% {user_timetrap_error,Reason} |
 886%% Other (ignored return value, e.g. 'ok')
 887tag({'$test_server_framework_test',Result}) ->
 888    case tag(Result) of
 889	ok      -> Result;
 890	Failure -> Failure
 891    end;	    
 892tag({skipped,Reason={failed,{_,init_per_testcase,_}}}) ->
 893    {auto_skipped,Reason};
 894tag({STag,Reason}) when STag == skip; STag == skipped -> 
 895    case Reason of
 896	{failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason};
 897	_ -> {skipped,Reason}
 898    end;
 899tag({auto_skip,Reason}) ->
 900    {auto_skipped,Reason};
 901tag({fail,Reason}) ->
 902    {failed,{error,Reason}};
 903tag(Failed = {failed,_Reason}) ->
 904    Failed;
 905tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; 
 906			   ETag == timetrap_timeout;
 907			   ETag == testcase_aborted -> 
 908    {failed,E};
 909tag(E = testcase_aborted_or_killed) ->
 910    {failed,E};
 911tag(UserTimetrap = {user_timetrap_error,_Reason}) ->
 912    UserTimetrap;
 913tag(_Other) ->
 914    ok.
 915
 916%%%-----------------------------------------------------------------
 917%%% -spec error_notification(Mod,Func,Args,Error) -> ok
 918%%%      Mod = atom()
 919%%%      Func = atom()
 920%%%      Args = list()
 921%%%      Error = term()
 922%%%
 923%%% This function is called as the result of testcase
 924%%% Func in suite Mod crashing.
 925%%% Error specifies the reason for failing.
 926error_notification(Mod,Func,_Args,{Error,Loc}) ->
 927    ErrorSpec = case Error of
 928		 {What={_E,_R},Trace} when is_list(Trace) ->
 929		      What;
 930		  What ->
 931		      What
 932	      end,
 933    ErrorStr = case ErrorSpec of
 934		 {badmatch,Descr} ->
 935                     Descr1 = io_lib:format("~tP",[Descr,10]),
 936                     DescrLength = string:length(Descr1),
 937                     if DescrLength > 50 ->
 938			     Descr2 = string:slice(Descr1,0,50),
 939			     io_lib:format("{badmatch,~ts...}",[Descr2]);
 940			true ->
 941			     io_lib:format("{badmatch,~ts}",[Descr1])
 942		     end;
 943		 {test_case_failed,Reason} ->
 944		     case (catch io_lib:format("{test_case_failed,~ts}", [Reason])) of
 945			 {'EXIT',_} ->
 946			     io_lib:format("{test_case_failed,~tp}", [Reason]);
 947			 Result -> Result
 948		     end;
 949		 {'EXIT',_Reason} = EXIT ->
 950		     io_lib:format("~tP", [EXIT,5]);
 951		 {Spec,_Reason} when is_atom(Spec) ->
 952		     io_lib:format("~tw", [Spec]);
 953		 Other ->
 954		     io_lib:format("~tP", [Other,5])
 955	     end,
 956    ErrorHtml =
 957	"<font color=\"brown\">" ++ ct_logs:escape_chars(ErrorStr) ++ "</font>",
 958    case {Mod,Error} of
 959	%% some notifications come from the main test_server process
 960	%% and for these cases the existing comment may not be modified
 961	{_,{timetrap_timeout,_TVal}} ->
 962	    ok;
 963	{_,{testcase_aborted,_Info}} ->
 964	    ok;
 965	{_,testcase_aborted_or_killed} ->
 966	    ok;
 967	{undefined,_OtherError} ->
 968	    ok;
 969	_ ->			     
 970	    %% this notification comes from the test case process, so
 971	    %% we can add error info to comment with test_server:comment/1
 972	    case ct_util:get_testdata({comment,group_leader()}) of
 973		undefined ->
 974		    test_server:comment(ErrorHtml);
 975		Comment ->
 976		    CommentHtml = 
 977			"<font color=\"green\">" ++ "(" ++ "</font>"
 978			++ Comment ++ 
 979			"<font color=\"green\">" ++ ")" ++ "</font>",
 980		    Str = io_lib:format("~ts   ~ts", [ErrorHtml,CommentHtml]),
 981		    test_server:comment(Str)
 982	    end
 983    end,
 984
 985    PrintError = fun(ErrorFormat, ErrorArgs) ->
 986                      Div = "\n- - - - - - - - - - - - - - - - - - - "
 987                            "- - - - - - - - - - - - - - - - - - - - -\n",
 988		       ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs),
 989                      io:format(?def_gl, "~ts~n", [lists:concat([Div,ErrorStr2,Div])]),
 990		       Link =
 991			   "\n\n<a href=\"#end\">"
 992			   "Full error description and stacktrace"
 993			   "</a>",
 994		       ErrorHtml2 = ct_logs:escape_chars(ErrorStr2),
 995		       ct_logs:tc_log(ct_error_notify,
 996				      ?MAX_IMPORTANCE,
 997				      "CT Error Notification",
 998                                      "~ts", [ErrorHtml2++Link],
 999                                      [])
1000	       end,
1001    case Loc of
1002	[{?MODULE,error_in_suite}] ->
1003	    PrintError("Error in suite detected: ~ts", [ErrorStr]);
1004
1005	R when R == unknown; R == undefined ->
1006	    PrintError("Error detected: ~ts", [ErrorStr]);
1007
1008	%% if a function specified by all/0 does not exist, we
1009	%% pick up undef here
1010	[{LastMod,LastFunc}|_] when ErrorStr == "undef" ->
1011	    PrintError("~w:~tw could not be executed~nReason: ~ts",
1012		     [LastMod,LastFunc,ErrorStr]);
1013
1014	[{LastMod,LastFunc}|_] ->
1015	    PrintError("~w:~tw failed~nReason: ~ts", [LastMod,LastFunc,ErrorStr]);
1016	    
1017	[{LastMod,LastFunc,LastLine}|_] ->
1018	    %% print error to console, we are only
1019	    %% interested in the last executed expression
1020	    PrintError("~w:~tw failed on line ~w~nReason: ~ts",
1021		     [LastMod,LastFunc,LastLine,ErrorStr]),
1022	    
1023	    case ct_util:read_suite_data({seq,Mod,Func}) of
1024		undefined ->
1025		    ok;
1026		Seq ->
1027		    SeqTCs = ct_util:read_suite_data({seq,Mod,Seq}),
1028		    mark_as_failed(Seq,Mod,Func,SeqTCs)
1029	    end	    
1030    end,
1031    ok.
1032
1033%% cases in seq that have already run
1034mark_as_failed(Seq,Mod,Func,[Func|TCs]) ->
1035    mark_as_failed1(Seq,Mod,Func,TCs);
1036mark_as_failed(Seq,Mod,Func,[_TC|TCs]) ->
1037    mark_as_failed(Seq,Mod,Func,TCs);
1038mark_as_failed(_,_,_,[]) ->
1039    ok;
1040mark_as_failed(_,_,_,undefined) ->
1041    ok.
1042
1043%% mark rest of cases in seq to be skipped
1044mark_as_failed1(Seq,Mod,Func,[TC|TCs]) ->
1045    ct_util:save_suite_data({seq,Mod,TC},{failed,Seq,Func}),
1046    mark_as_failed1(Seq,Mod,Func,TCs);
1047mark_as_failed1(_,_,_,[]) ->
1048    ok.
1049
1050group_or_func(Func, Config) when Func == init_per_group; 
1051				 Func == end_per_group ->
1052    case ?val(tc_group_properties, Config) of
1053	undefined ->
1054	    {Func,unknown,[]};
1055	GrProps ->
1056	    GrName = ?val(name,GrProps),
1057	    {Func,GrName,proplists:delete(name,GrProps)}
1058    end;	  
1059group_or_func(Func, _Config) ->
1060    Func.
1061
1062%%%-----------------------------------------------------------------
1063%%% -spec get_suite(Mod, Func) -> Tests
1064%%%
1065%%% Called from test_server for every suite (Func==all)
1066%%% and every test case. If the former, all test cases in the suite
1067%%% should be returned.
1068
1069get_suite(Mod, all) ->
1070    case safe_apply_groups_0(Mod,{ok,[]}) of
1071        {ok,GroupDefs} ->
1072            try ct_groups:find_groups(Mod, all, all, GroupDefs) of
1073                ConfTests when is_list(ConfTests) ->
1074                    get_all(Mod, ConfTests)
1075            catch
1076                throw:{error,Error} ->
1077                    [{?MODULE,error_in_suite,[[{error,Error}]]}];
1078                _:Error:S ->
1079                    [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
1080            end;
1081        {error,{bad_return,_Bad}} ->
1082	    E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
1083	    [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
1084        {error,{bad_hook_return,Bad}} ->
1085	    E = "Bad return value from post_groups/2 hook function",
1086	    [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
1087        {error,{failed,ExitReason}} ->
1088	    case ct_util:get_testdata({error_in_suite,Mod}) of
1089		undefined ->
1090		    ErrStr = io_lib:format("~n*** ERROR *** "
1091					   "~w:groups/0 failed: ~p~n",
1092					   [Mod,ExitReason]),
1093		    io:format(?def_gl, ErrStr, []),
1094		    %% save the error info so it doesn't get printed twice
1095		    ct_util:set_testdata_async({{error_in_suite,Mod},
1096						ExitReason});
1097		_ExitReason ->
1098		    ct_util:delete_testdata({error_in_suite,Mod})
1099	    end,
1100	    Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
1101	    [{?MODULE,error_in_suite,[[{error,Reason}]]}];
1102        {error,What} ->
1103            [{?MODULE,error_in_suite,[[{error,What}]]}]
1104    end;
1105
1106%%!============================================================
1107%%! Note: The handling of sequences in get_suite/2 and get_all/2
1108%%! is deprecated and should be removed at some point...
1109%%!============================================================
1110
1111%% group
1112get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
1113    case safe_apply_groups_0(Mod,{ok,[Group]}) of
1114        {ok,GroupDefs} ->
1115            Name = ?val(name, Props),
1116            try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
1117                [] ->
1118                    [];
1119                ConfTests when is_list(ConfTests) ->
1120                    case lists:member(skipped, Props) of
1121                        true ->
1122                            %% a *subgroup* specified *only* as skipped (and not
1123                            %% as an explicit test) should not be returned, or
1124                            %% init/end functions for top groups will be executed
1125                            try ?val(name, element(2, hd(ConfTests))) of
1126                                Name ->		% top group
1127                                    ct_groups:delete_subs(ConfTests, ConfTests);
1128                                _ -> []
1129                            catch
1130                                _:_ -> []
1131                            end;
1132                        false ->
1133                            ConfTests1 = ct_groups:delete_subs(ConfTests,
1134                                                               ConfTests),
1135                            case ?val(override, Props) of
1136                                undefined ->
1137                                    ConfTests1;
1138                                [] ->
1139                                    ConfTests1;
1140                                ORSpec ->
1141                                    ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
1142                                                 true -> ORSpec end,
1143                                    ct_groups:search_and_override(ConfTests1,
1144                                                                  ORSpec1, Mod)
1145                            end
1146                    end
1147            catch
1148                throw:{error,Error} ->
1149                    [{?MODULE,error_in_suite,[[{error,Error}]]}];
1150                _:Error:S ->
1151                    [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
1152            end;
1153        {error,{bad_return,_Bad}} ->
1154	    E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
1155	    [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
1156        {error,{bad_hook_return,Bad}} ->
1157            E = "Bad return value from post_groups/2 hook function",
1158	    [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
1159        {error,{failed,ExitReason}} ->
1160	    case ct_util:get_testdata({error_in_suite,Mod}) of
1161		undefined ->
1162		    ErrStr = io_lib:format("~n*** ERROR *** "
1163					   "~w:groups/0 failed: ~p~n",
1164					   [Mod,ExitReason]),
1165		    io:format(?def_gl, ErrStr, []),
1166		    %% save the error info so it doesn't get printed twice
1167		    ct_util:set_testdata_async({{error_in_suite,Mod},
1168						ExitReason});
1169		_ExitReason ->
1170		    ct_util:delete_testdata({error_in_suite,Mod})
1171	    end,
1172	    Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
1173	    [{?MODULE,error_in_suite,[[{error,Reason}]]}];
1174         {error,What} ->
1175            [{?MODULE,error_in_suite,[[{error,What}]]}]
1176    end;
1177
1178%% testcase
1179get_suite(Mod, Name) ->
1180    get_seq(Mod, Name).
1181
1182%%%-----------------------------------------------------------------
1183
1184get_all_cases(Suite) ->
1185    case get_suite(Suite, all) of
1186	[{?MODULE,error_in_suite,[[{error,_}=Error]]}] ->
1187		Error;
1188	[{?MODULE,error_in_suite,[[Error]]}] ->
1189	    {error,Error};
1190	Tests ->
1191	    Cases = get_all_cases1(Suite, Tests),
1192	    ?rev(lists:foldl(fun(TC, TCs) ->
1193				     case lists:member(TC, TCs) of
1194				      true  -> TCs;
1195					 false -> [TC | TCs]
1196				     end
1197			     end, [], Cases))
1198    end.
1199
1200get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) ->
1201    get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests);
1202
1203get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) ->
1204    [{Suite,Test} | get_all_cases1(Suite, Tests)];
1205
1206get_all_cases1(Suite, [Test | Tests]) ->
1207    [Test | get_all_cases1(Suite, Tests)];
1208
1209get_all_cases1(_, []) ->
1210    [].
1211
1212%%%-----------------------------------------------------------------
1213
1214get_all(Mod, ConfTests) ->
1215    case safe_apply_all_0(Mod) of
1216        {ok,AllTCs} ->
1217            %% expand group references using ConfTests
1218            try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of
1219                {error,_} = Error ->
1220                    [{?MODULE,error_in_suite,[[Error]]}];
1221                Tests0 ->
1222                    Tests = ct_groups:delete_subs(Tests0, Tests0),
1223                    expand_tests(Mod, Tests)
1224            catch
1225                throw:{error,Error} ->
1226                    [{?MODULE,error_in_suite,[[{error,Error}]]}];
1227                _:Error:S ->
1228                    [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
1229            end;
1230        Skip = {skip,_Reason} ->
1231	    Skip;
1232        {error,undef} ->
1233            Reason =
1234                case code:which(Mod) of
1235                    non_existing ->
1236                        list_to_atom(
1237                          atom_to_list(Mod)++
1238                              " cannot be compiled or loaded");
1239                    _ ->
1240                        list_to_atom(
1241                          atom_to_list(Mod)++":all/0 is missing")
1242                end,
1243            %% this makes test_server call error_in_suite as first
1244            %% (and only) test case so we can report Reason properly
1245            [{?MODULE,error_in_suite,[[{error,Reason}]]}];
1246	{error,{bad_return,_Bad}} ->
1247	    Reason =
1248		list_to_atom("Bad return value from "++
1249				 atom_to_list(Mod)++":all/0"),
1250	    [{?MODULE,error_in_suite,[[{error,Reason}]]}];
1251        {error,{bad_hook_return,Bad}} ->
1252	    Reason =
1253		list_to_atom("Bad return value from post_all/3 hook function"),
1254	    [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}];
1255        {error,{failed,ExitReason}} ->
1256	    case ct_util:get_testdata({error_in_suite,Mod}) of
1257		undefined ->
1258		    ErrStr = io_lib:format("~n*** ERROR *** "
1259					   "~w:all/0 failed: ~tp~n",
1260					   [Mod,ExitReason]),
1261		    io:format(?def_gl, "~ts", [ErrStr]),
1262		    %% save the error info so it doesn't get printed twice
1263		    ct_util:set_testdata_async({{error_in_suite,Mod},
1264						ExitReason});
1265		_ExitReason ->
1266		    ct_util:delete_testdata({error_in_suite,Mod})
1267	    end,
1268	    Reason = list_to_atom(atom_to_list(Mod)++":all/0 failed"),
1269	    %% this makes test_server call error_in_suite as first
1270	    %% (and only) test case so we can report Reason properly
1271	    [{?MODULE,error_in_suite,[[{error,Reason}]]}];
1272        {error,What} ->
1273            [{?MODULE,error_in_suite,[[{error,What}]]}]
1274    end.
1275
1276%%!============================================================
1277%%! The support for sequences by means of using sequences/0
1278%%! will be removed in OTP R15. The code below is only kept 
1279%%! for backwards compatibility. From OTP R13 groups with
1280%%! sequence property should be used instead!
1281%%!============================================================
1282%%!============================================================
1283%%! START OF DEPRECATED SUPPORT FOR SEQUENCES --->
1284
1285get_seq(Mod, Func) ->
1286    case ct_util:read_suite_data({seq,Mod,Func}) of
1287	undefined ->
1288	    case catch apply(Mod,sequences,[]) of
1289		{'EXIT',_} ->
1290		    [];
1291		Seqs ->
1292		    case lists:keysearch(Func,1,Seqs) of
1293			{value,{Func,SeqTCs}} ->			    
1294			    case catch save_seq(Mod,Func,SeqTCs) of
1295				{error,What} ->
1296				    [{?MODULE,error_in_suite,[[{error,What}]]}];
1297				_ ->
1298				    SeqTCs
1299			    end;
1300			false ->
1301			    []
1302		    end
1303	    end;
1304	TCs when is_list(TCs) ->
1305	    TCs;
1306	_ ->
1307	    []
1308    end.
1309
1310save_seqs(Mod,AllTCs) ->
1311    case lists:keymember(sequence,1,AllTCs) of
1312	true ->
1313	    case catch apply(Mod,sequences,[]) of
1314		{'EXIT',_} -> 
1315		    Reason = list_to_atom(atom_to_list(Mod)++
1316					  ":sequences/0 is missing"),
1317		    throw({error,Reason});
1318		Seqs ->
1319		    save_seqs(Mod,AllTCs,Seqs,AllTCs)
1320	    end;
1321	false ->
1322	    AllTCs
1323    end.
1324    
1325save_seqs(Mod,[{sequence,Seq}|TCs],Seqs,All) ->
1326    case lists:keysearch(Seq,1,Seqs) of
1327	{value,{Seq,SeqTCs}} ->
1328	    save_seq(Mod,Seq,SeqTCs,All),
1329	    [Seq|save_seqs(Mod,TCs,Seqs,All)];
1330	false ->
1331	    Reason = list_to_atom(
1332		       atom_to_list(Seq)++" is missing in "++
1333		       atom_to_list(Mod)),
1334	    throw({error,Reason})
1335    end;
1336save_seqs(Mod,[TC|TCs],Seqs,All) ->
1337    [TC|save_seqs(Mod,TCs,Seqs,All)];
1338save_seqs(_,[],_,_) ->
1339    [].
1340
1341save_seq(Mod,Seq,SeqTCs) ->
1342    save_seq(Mod,Seq,SeqTCs,apply(Mod,all,[])).
1343    
1344save_seq(Mod,Seq,SeqTCs,All) ->
1345    check_private(Seq,SeqTCs,All),
1346    check_multiple(Mod,Seq,SeqTCs),
1347    ct_util:save_suite_data({seq,Mod,Seq},SeqTCs),
1348    lists:foreach(fun(TC) -> 
1349			  ct_util:save_suite_data({seq,Mod,TC},Seq)
1350		  end, SeqTCs).
1351
1352check_private(Seq,TCs,All) ->    
1353    Bad = lists:filter(fun(TC) -> lists:member(TC,All) end, TCs),
1354    if Bad /= [] ->
1355	    Reason = io_lib:format("regular test cases not allowed in sequence ~tp: "
1356				   "~tp",[Seq,Bad]),
1357	    throw({error,list_to_atom(lists:flatten(Reason))});
1358       true ->
1359	    ok
1360    end.
1361
1362check_multiple(Mod,Seq,TCs) ->
1363    Bad = lists:filter(fun(TC) ->
1364			       case ct_util:read_suite_data({seq,Mod,TC}) of
1365				   Seq1 when Seq1 /= undefined, Seq1 /= Seq -> 
1366				       true;
1367
1368				   _ -> false
1369			       end
1370		       end,TCs),
1371    if Bad /= [] ->
1372	    Reason = io_lib:format("test cases found in multiple sequences: "
1373				   "~tp",[Bad]),
1374	    throw({error,list_to_atom(lists:flatten(Reason))});
1375       true ->
1376	    ok
1377    end.
1378
1379%%! <---  END OF DEPRECATED SUPPORT FOR SEQUENCES
1380%%!============================================================
1381
1382%% let test_server call this function as a testcase only so that
1383%% the user may see info about what's missing in the suite
1384error_in_suite(Config) ->
1385    Reason = test_server:lookup_config(error,Config),
1386    exit(Reason).
1387
1388%% if init_per_suite and end_per_suite are missing in the suite,
1389%% these will be called instead (without any trace of them in the
1390%% log files), only so that it's possible to call hook functions
1391%% for configuration
1392init_per_suite(Config) ->
1393    Config.
1394
1395end_per_suite(_Config) ->
1396    ok.
1397
1398%% if the group config functions are missing in the suite,
1399%% use these instead
1400init_per_group(GroupName, Config) ->
1401    ct:comment(io_lib:format("start of ~tp", [GroupName])),
1402    ct_logs:log("TEST INFO", "init_per_group/2 for ~tw missing "
1403		"in suite, using default.",
1404		[GroupName]),
1405    Config.
1406
1407end_per_group(GroupName, _) ->
1408    ct:comment(io_lib:format("end of ~tp", [GroupName])),
1409    ct_logs:log("TEST INFO", "end_per_group/2 for ~tw missing "
1410		"in suite, using default.",
1411		[GroupName]),
1412    ok.
1413    
1414%%%-----------------------------------------------------------------
1415%%% -spec report(What,Data) -> ok
1416report(What,Data) ->
1417    case What of
1418	loginfo ->
1419	    %% logfiles and direcories have been created for a test and the
1420	    %% top level test index page needs to be refreshed
1421	    TestName = filename:basename(?val(topdir, Data), ".logs"),
1422	    RunDir = ?val(rundir, Data),
1423	    _ = ct_logs:make_all_suites_index({TestName,RunDir}),
1424	    ok;
1425	tests_start ->
1426	    ok;
1427	tests_done ->
1428	    ok;
1429	severe_error ->
1430	    ct_event:sync_notify(#event{name=What,
1431					node=node(),
1432					data=Data}),
1433	    ct_util:set_testdata({What,Data}),
1434	    ok;
1435	tc_start ->
1436	    %% Data = {{Suite,{Func,GroupName}},LogFileName}
1437	    Data1 = case Data of
1438			{{Suite,{Func,undefined}},LFN} -> {{Suite,Func},LFN};
1439			_ -> Data
1440		    end,
1441	    ct_event:sync_notify(#event{name=tc_logfile,
1442					node=node(),
1443					data=Data1}),
1444	    ok;
1445	tc_done ->
1446	    {Suite,{Func,GrName},Result} = Data,
1447            FuncSpec = if GrName == undefined -> Func;
1448                          true                -> {Func,GrName}
1449                       end,
1450	    %% Register the group leader for the process calling the report
1451	    %% function, making it possible for a hook function to print
1452	    %% in the test case log file
1453	    ReportingPid = self(),
1454	    ct_logs:register_groupleader(ReportingPid, group_leader()),
1455	    case Result of
1456		{failed, Reason} ->
1457		    ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason});
1458		{skipped,{failed,{_,init_per_testcase,_}}=Reason} ->
1459		    ct_hooks:on_tc_skip(tc_auto_skip,  {Suite,FuncSpec,Reason});
1460		{skipped,{require_failed,_}=Reason} ->
1461		    ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
1462		{skipped,Reason} ->
1463		    ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason});
1464		{auto_skipped,Reason} ->
1465		    ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
1466		_Else ->
1467		    ok
1468	    end,
1469	    ct_logs:unregister_groupleader(ReportingPid),
1470	    case {Func,Result} of
1471		{error_in_suite,_} when Suite == ?MODULE ->
1472		    ok;
1473		{init_per_suite,_} ->
1474		    ok;
1475		{end_per_suite,_} ->
1476		    ok;
1477		{init_per_group,_} ->
1478		    ok;
1479		{end_per_group,_} ->
1480		    ok;
1481		{_,ok} ->
1482		    add_to_stats(ok);
1483		{_,{skipped,{failed,{_,init_per_testcase,_}}}} ->
1484		    add_to_stats(auto_skipped);
1485		{_,{skipped,{require_failed,_}}} ->
1486		    add_to_stats(auto_skipped);
1487		{_,{skipped,{timetrap_error,_}}} ->
1488		    add_to_stats(auto_skipped);
1489		{_,{skipped,{invalid_time_format,_}}} ->
1490		    add_to_stats(auto_skipped);
1491		{_,{skipped,_}} ->
1492		    add_to_stats(user_skipped);
1493		{_,{auto_skipped,_}} ->
1494		    add_to_stats(auto_skipped);
1495		{_,{SkipOrFail,_Reason}} ->
1496		    add_to_stats(SkipOrFail)
1497	    end;
1498	tc_user_skip ->
1499	    %% test case or config function specified as skipped in testspe

Large files files are truncated, but you can click here to view the full file