PageRenderTime 45ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 1ms

/testing/Test.m

http://simengine.googlecode.com/
Objective C | 429 lines | 391 code | 38 blank | 0 comment | 63 complexity | 90ba2f644dbe02bca7b40e81b42645cd MD5 | raw file
Possible License(s): CC-BY-SA-3.0
  1. % TEST class
  2. % Performs individual tests of simEngine
  3. %
  4. % Construction
  5. % t = TEST(NAME, FUNCTION)
  6. % This will create a new test with name NAME running FUNCTION.
  7. % FUNCTION should have no input arguments and return a boolean true or
  8. % false.
  9. %
  10. % t = TEST(NAME, FUNCTION, '-withouterror')
  11. % This test succeeds if the function executes without any errors. This
  12. % mode is useful for compilation tests.
  13. %
  14. % t = TEST(NAME, FUNCTION, '-equal', VAL)
  15. % This test succeeds if the return of FUNCTION is equivalent to VAL.
  16. % Equivalency is determined by the EQUIV helper function.
  17. %
  18. % t = TEST(NAME, FUNCTION, '-allequal')
  19. % This test succeeds if each parallel entry in the return structure of
  20. % FUNCTION is equal to every other. (returns true when running a single
  21. % model.
  22. %
  23. % t = TEST(NAME, FUNCTION, '-approxequal', VAL [, PRECISION])
  24. % This test verifies that the return of FUNCTION is equal to VAL within
  25. % +/- PRECISION percent.
  26. %
  27. % t = TEST(NAME, FUNCTION, '-range', LOW, HIGH)
  28. % This test verifies that the return of FUNCTION is greater or equal to
  29. % LOW and less or equal to HIGH.
  30. %
  31. % t = TEST(NAME, FUNCTION, '-regexpmatch', STRING)
  32. % This test verifies that the stdout matches some regular expression
  33. % pattern. This is often paired with the 'expectfail' option when we
  34. % are testing an error output.
  35. %
  36. % Usage
  37. %
  38. % t.Execute - this will execute the test manually. In generally, suites
  39. % will invoke this method of a test.
  40. %
  41. % Example
  42. % s = Suite('My Test Suite')
  43. % s.add(Test('Compile Sine Model', @()(simex('sine.dsl')),
  44. % '-withouterror'))
  45. % s.Execute
  46. %
  47. % Copyright 2009, 2010 Simatra Modeling Technologies, L.L.C.
  48. %
  49. classdef Test < handle
  50. properties
  51. Name
  52. Function
  53. Dir
  54. Mode
  55. ExpectFail = false
  56. ExpectError = false % testing an error condition
  57. Enabled = true
  58. end
  59. % enumeration for the Result
  60. properties (Constant = true)
  61. NOT_EXECUTED = 0;
  62. PASSED = 1;
  63. FAILED = 2;
  64. ERROR = 3;
  65. SKIPPED = 4;
  66. end
  67. % enumeration for the Mode
  68. properties (Constant = true)
  69. BOOLEAN = 0;
  70. NOTERROR = 1;
  71. EQUAL = 2;
  72. APPROXEQUAL = 3;
  73. RANGE = 4;
  74. REGEXP = 5;
  75. ALLEQUAL = 6;
  76. end
  77. properties (SetAccess = private)
  78. Executed
  79. Result
  80. Return
  81. Output = ''
  82. Time
  83. CompareOptions
  84. Message = ''
  85. end
  86. properties (Access = private)
  87. FunctionFile
  88. FunctionName
  89. FunctionLine
  90. OutputFileName
  91. end
  92. methods
  93. function t = Test(name, func, varargin)
  94. if nargin >= 2
  95. t.Name = name;
  96. t.Function = func;
  97. t.Dir = pwd;
  98. t.Executed = false;
  99. t.Result = t.NOT_EXECUTED;
  100. t.Time = 0;
  101. t.Mode = t.BOOLEAN;
  102. else
  103. error('Test:ArgumentError', 'Wrong number of arguments');
  104. end
  105. % Find the filename and line #
  106. try
  107. me = MException('Simatra:Test:Exception',['No Error ' ...
  108. 'Occurred']);
  109. throw(me);
  110. catch me
  111. if length(me.stack) >= 2
  112. FunctionFile = me.stack(2).file;
  113. t.FunctionName = me.stack(2).name;
  114. t.FunctionLine = me.stack(2).line;
  115. [fpath,fname,fext]=fileparts(FunctionFile);
  116. t.FunctionFile = [fname fext];
  117. else
  118. t.FunctionName = 'N/A';
  119. t.FunctionLine = 0;
  120. t.FunctionFile = 'Matlab Interpreter';
  121. end
  122. end
  123. if nargin >= 3 && ischar(varargin{1})
  124. switch lower(varargin{1})
  125. case '-boolean'
  126. t.Mode = t.BOOLEAN;
  127. case '-withouterror'
  128. t.Mode = t.NOTERROR;
  129. case '-equal'
  130. t.Mode = t.EQUAL;
  131. if nargin == 4
  132. t.CompareOptions = varargin(2);
  133. else
  134. error('Test:ArgumentError', 'When using the -equal option, there should be a fourth required option')
  135. end
  136. case '-allequal'
  137. t.Mode = t.ALLEQUAL;
  138. case '-approxequal'
  139. t.Mode = t.APPROXEQUAL;
  140. if nargin == 4
  141. t.CompareOptions = {varargin{2}, 1}; % by default, stay within one percent
  142. elseif nargin == 5 && isnumeric(varargin{3})
  143. t.CompareOptions = {varargin{2}, varargin{3}}; % by default, stay within one percent
  144. else
  145. error('Test:ArgumentError', 'When using the -approxequal option, there should be a fourth required option and a fifth optional argument')
  146. end
  147. case '-range'
  148. t.Mode = t.RANGE;
  149. if nargin == 4 && length(varargin{2}) == 2 && isnumeric(varargin{2})
  150. t.CompareOptions = varargin(2);
  151. else
  152. error('Test:ArgumentError', 'When using the -range option, there should be a fourth required option of length 2')
  153. end
  154. case '-regexpmatch'
  155. t.Mode = t.REGEXP;
  156. if nargin == 4 && ischar(varargin{2})
  157. t.CompareOptions = varargin(2);
  158. else
  159. error('Test:ArgumentError', 'When using the -regexpmatch option, there should be a required string option');
  160. end
  161. otherwise
  162. error('Test:ArgumentError','Unexpected third argument to Test')
  163. end
  164. elseif nargin >= 3 && not(ischar(varargin{1}))
  165. error('Test:ArgumentError', 'Expected string as third argument')
  166. end
  167. end
  168. function Execute(t)
  169. curDir = pwd;
  170. cd(t.Dir);
  171. % localtime = tic;
  172. % if ischar(t.Function)
  173. % [t.Output, t.Return] = evalc('feval(t.Function)');
  174. % else
  175. % finfo = functions(t.Function);
  176. % [t.Output, t.Return] = evalc('feval(t.Function)');
  177. % end
  178. % t.Time = toc(localtime);
  179. [t.Output, t.Return, t.Time, errored] = executeFunction(t.Function);
  180. if ~errored
  181. % check the output
  182. switch t.Mode
  183. case t.NOTERROR
  184. t.Result = t.PASSED;
  185. case t.BOOLEAN
  186. if t.Return
  187. t.Result = t.PASSED;
  188. else
  189. t.Result = t.FAILED;
  190. end
  191. case t.EQUAL
  192. if equiv(t.Return, t.CompareOptions{1})
  193. t.Result = t.PASSED;
  194. else
  195. t.Result = t.FAILED;
  196. if isnumeric(t.Return)
  197. t.Message = ['Returned ''' num2str(t.Return) ''' instead'];
  198. elseif isstruct(t.Return)
  199. t.Message = ['Returned a different structure instead'];
  200. else
  201. t.Message = ['Returned a different quantity instead'];
  202. end
  203. end
  204. case t.APPROXEQUAL
  205. if approx_equiv(t.CompareOptions{1}, t.Return, t.CompareOptions{2})
  206. t.Result = t.PASSED;
  207. else
  208. t.Result = t.FAILED;
  209. %t.Message = sprintf('Returned ''%s'' instead, which is not +/- %g%% of %g', num2str(t.Return), t.CompareOptions(2), t.CompareOptions(1));
  210. if isnumeric(t.Return)
  211. t.Message = ['Returned ''' num2str(t.Return) ''' instead'];
  212. elseif isstruct(t.Return)
  213. t.Message = ['Returned a different structure instead'];
  214. else
  215. t.Message = ['Returned a different quantity instead'];
  216. %error('Test:ExecuteError:ApproxEqual', 'Return value ''%s'' not numeric or does not have length of one', num2str(t.Return));
  217. end
  218. end
  219. case t.ALLEQUAL
  220. if all_equiv(t.Return)
  221. t.Result = t.PASSED;
  222. else
  223. t.Result = t.FAILED;
  224. t.Message = ['Not all parallel structures are identical'];
  225. end
  226. case t.RANGE
  227. if isnumeric(t.Return) && length(t.Return) == 1
  228. l = t.CompareOptions{1};
  229. h = t.CompareOptions{2};
  230. if t.Return <= h && t.Return >= l
  231. t.Result = t.PASSED;
  232. else
  233. t.Result = t.FAILED;
  234. t.Message = sprintf('Returned ''%s'' instead, which is not within [%g,%g]', num2str(t.Return), t.CompareOptions{1}, t.CompareOptions{2});
  235. end
  236. else
  237. t.Message = ['Unexpected return value ''' num2str(t.Return) ''''];
  238. error('Test:ExecuteError:Range', 'Return value ''%s'' not numeric or does not have length of one', num2str(t.Return));
  239. end
  240. case t.REGEXP
  241. matches = regexp(t.Output, ...
  242. t.CompareOptions{1});
  243. if isempty(matches)
  244. t.Result = t.FAILED;
  245. t.Message = sprintf('Regular expression ''%s'' not found in command output', t.CompareOptions{1});
  246. else
  247. t.Result = t.PASSED;
  248. end
  249. otherwise
  250. error('Test:ExecuteError', 'Unknown mode encountered');
  251. end
  252. else % an error occured
  253. switch t.Mode
  254. case t.REGEXP % in REGEXP mode, we don't care if an
  255. % error occured, since we are often
  256. % looking for error messages
  257. matches = regexp(t.Output, ...
  258. t.CompareOptions{1});
  259. if isempty(matches)
  260. t.Result = t.FAILED;
  261. t.Message = sprintf('Regular expression ''%s'' not found in command output', t.CompareOptions{1});
  262. else
  263. t.Result = t.PASSED;
  264. end
  265. otherwise
  266. if t.Mode == t.NOTERROR
  267. t.Result = t.FAILED;
  268. t.Message = t.Return.message;
  269. else
  270. t.Result = t.ERROR;
  271. t.Message = t.Return.message;
  272. end
  273. end
  274. end
  275. % Check if expected fail
  276. if t.ExpectFail
  277. if t.Result == t.PASSED
  278. t.Result = t.FAILED;
  279. t.Message = [t.Message '(Test passed, but expected to FAIL)'];
  280. elseif t.Result == t.FAILED
  281. t.Result = t.PASSED;
  282. t.Message = [t.Message ' (Expected to FAIL)'];
  283. end
  284. end
  285. % Location string
  286. location = ['(' t.FunctionFile ':' num2str(t.FunctionLine) ')'];
  287. % Show result of this test
  288. if t.Result == t.PASSED
  289. status = 'Passed';
  290. elseif t.Result == t.FAILED
  291. status = ['FAILED <---- ' location];
  292. else
  293. status = ['ERRORED <---- ' location];
  294. end
  295. s = sprintf('%40s:\t%s', t.Name, status);
  296. disp(s)
  297. cd(curDir);
  298. end
  299. function Skip(t)
  300. t.Result = t.SKIPPED;
  301. end
  302. function s = tostr(t)
  303. if isempty(t.Message)
  304. s = sprintf('Test ''%s'': %s', t.Name, result2str(t.Result));
  305. else
  306. s = sprintf('Test ''%s'': %s (%s)', t.Name, result2str(t.Result), t.Message);
  307. end
  308. if t.Result ~= t.NOT_EXECUTED
  309. s = sprintf('%s [time=%gs]', s, t.Time);
  310. end
  311. end
  312. % override the display function
  313. function disp(t)
  314. disp(tostr(t));
  315. end
  316. function root = toXML (t, varargin)
  317. if 1 == nargin
  318. xml = com.mathworks.xml.XMLUtils.createDocument('testcase');
  319. root = xml.getDocumentElement;
  320. else
  321. xml = varargin{1};
  322. parent = varargin{2};
  323. root = parent.appendChild(xml.createElement('testcase'));
  324. end
  325. root.setAttribute('time', num2str(t.Time));
  326. root.setAttribute('name', num2str(t.Name));
  327. output = root.appendChild(xml.createElement('system-out'));
  328. output.appendChild(xml.createTextNode(t.Output));
  329. switch(t.Result)
  330. case t.FAILED
  331. message = root.appendChild(xml.createElement('failure'));
  332. message.setAttribute('message', t.Message);
  333. message.appendChild(xml.createTextNode(t.Output));
  334. case t.ERROR
  335. message = root.appendChild(xml.createElement('error'));
  336. message.setAttribute('message', t.Message);
  337. message.appendChild(xml.createTextNode(t.Output));
  338. end
  339. end
  340. % write JUnit XML output
  341. function writeXML(t, file)
  342. xmlwrite(file, t.toXML);
  343. end
  344. end % methods
  345. end % end class Test
  346. % Run the funname and return all the data/output
  347. function [output, data, time, errored] = executeFunction (funname)
  348. [output, data, time, errored] = evalc('executeFunctionHelper(funname)');
  349. end
  350. function [data, time, errored] = executeFunctionHelper(funname)
  351. t = tic;
  352. try
  353. data = feval(funname);
  354. errored = false;
  355. catch me
  356. data = me;
  357. errored = true;
  358. end
  359. time = toc(t);
  360. end
  361. % read the output data file
  362. function str = readOutputFile(filename)
  363. if exist(filename, 'file')
  364. strcell = textread(filename, '%s', 'whitespace', '');
  365. if ~isempty(strcell)
  366. str = strcell{1};
  367. else
  368. str = '';
  369. end
  370. delete(filename);
  371. else
  372. str = '';
  373. end
  374. end
  375. % Additional helper functions
  376. function r = result2str(result)
  377. switch result
  378. case 0
  379. r = 'Not Executed';
  380. case 1
  381. r = 'Passed';
  382. case 2
  383. r = 'Failed';
  384. case 3
  385. r = 'Error occurred';
  386. case 4
  387. r = 'Skipped';
  388. otherwise
  389. r = 'Unknown';
  390. end
  391. end