PageRenderTime 47ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 0ms

/IronPython_Main/Runtime/Tests/AstTest/Program.cs

#
C# | 520 lines | 388 code | 75 blank | 57 comment | 91 complexity | d408259395ba2d97b6fb99bd9fecce03 MD5 | raw file
Possible License(s): GPL-2.0, MPL-2.0-no-copyleft-exception, CPL-1.0, CC-BY-SA-3.0, BSD-3-Clause, ISC, AGPL-3.0, LGPL-2.1, Apache-2.0
  1. /* ****************************************************************************
  2. *
  3. * Copyright (c) Microsoft Corporation.
  4. *
  5. * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
  6. * copy of the license can be found in the License.html file at the root of this distribution. If
  7. * you cannot locate the Apache License, Version 2.0, please send an email to
  8. * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
  9. * by the terms of the Apache License, Version 2.0.
  10. *
  11. * You must not remove this notice, or any other, from this software.
  12. *
  13. *
  14. * ***************************************************************************/
  15. #if !CLR2
  16. using System.Linq.Expressions;
  17. #else
  18. using Microsoft.Scripting.Ast;
  19. #endif
  20. using System;
  21. using System.Collections.Generic;
  22. using System.Diagnostics;
  23. using System.Reflection;
  24. using System.Security;
  25. using System.Security.Permissions;
  26. using ETUtils;
  27. using Utils = AstTest.Utils;
  28. using EU = ETUtils.ExpressionUtils;
  29. namespace AstTest {
  30. //
  31. // Quick note on the meaning of "Positive" "Negative" and "Slow" test
  32. // prefixes:
  33. //
  34. // Positive: this test throws no exceptions while running. We want these
  35. // to run first
  36. //
  37. // Negative: this test intentionally throws an exception while running.
  38. // These are run after positive tests so we don't see intentional
  39. // exceptions when debugging positive test failures.
  40. //
  41. // Slow: this test takes a while to run because it compiles lots of
  42. // trees. For this reason, we want it to run last.
  43. //
  44. // Disabled: the test is disabled. It can still be run manually
  45. //
  46. public static partial class Scenarios {
  47. public static bool FullTrust;
  48. #if SILVERLIGHT
  49. public class ArgumentData {
  50. #else
  51. public class ArgumentData : MarshalByRefObject {
  52. #endif
  53. public Boolean CompileToMethod = false;
  54. public Boolean VerifyAssembly = false;
  55. public Boolean FullTrust = false;
  56. public Boolean NotExact = false;
  57. public Boolean Help = false;
  58. public List<string> Rewrite = new List<string>();
  59. public List<string> Testcases = new List<string>();
  60. public void Initialize(string[] args) {
  61. foreach (string s in args) {
  62. String temp;
  63. #if SILVERLIGHT3
  64. temp = s.ToLower();
  65. #else
  66. temp = s.ToLowerInvariant();
  67. #endif
  68. if (temp.StartsWith("/") || temp.StartsWith("-")) {
  69. //switch
  70. //clear / or -
  71. temp = temp.Substring(1);
  72. if (temp.Equals("ne") || temp.Equals("notexact")) {
  73. NotExact = true;
  74. }
  75. if (temp.Equals("ctm") || temp.Equals("compiletomethod")) {
  76. CompileToMethod = true;
  77. }
  78. if (temp.Equals("va") || temp.Equals("verifyassembly")) {
  79. CompileToMethod = true;
  80. VerifyAssembly = true;
  81. FullTrust = true;
  82. }
  83. if (temp.Equals("f") || temp.Equals("fulltrust")) {
  84. FullTrust = true;
  85. }
  86. if (temp.Equals("?") || temp.Equals("help")) {
  87. Help = true;
  88. }
  89. if (temp.StartsWith("r") || temp.StartsWith("rewrite")) {
  90. //get rewriters
  91. Rewrite.AddRange(Utils.GetArgumentValues(temp));
  92. }
  93. } else {
  94. //testcase
  95. Testcases.Add(temp);
  96. }
  97. }
  98. }
  99. }
  100. #if SILVERLIGHT
  101. public sealed class TestRunner {
  102. #else
  103. public sealed class TestRunner : MarshalByRefObject {
  104. #endif
  105. private string[] _arguments;
  106. public string[] Arguments {
  107. get {
  108. return _arguments;
  109. }
  110. set {
  111. _arguments = value;
  112. Options.Initialize(value);
  113. }
  114. }
  115. public readonly ArgumentData Options = new ArgumentData();
  116. public int ExitCode { get; private set; }
  117. public int TestsRan { get; private set; }
  118. public TrustKind TrustMode { get; set; }
  119. public readonly List<Assembly> PreLoadedAssemblies = new List<Assembly>();
  120. public enum TrustKind {
  121. FullTrust,
  122. FullTrustOnly,
  123. PartialTrust,
  124. PartialTrustOnly
  125. }
  126. private Exception RunScenario(MethodInfo test, String Rewriter, bool CompileToMethod) {
  127. var attr = ETUtils.TestAttribute.GetAttribute(test);
  128. Exception error = null;
  129. try {
  130. Expression body = null;
  131. if (test.GetParameters().Length == 0) { // legacy LINQ tests, NoPia tests
  132. test.Invoke(null, new object[] {});
  133. } else {
  134. #if SILVERLIGHT3
  135. throw new Exception("Should not be here"); //@RYANOOB
  136. #else
  137. var V = new EU.TestValidator(Rewriter, Options.CompileToMethod, Options.VerifyAssembly);
  138. body = (Expression)test.Invoke(null, new object[] { V }); // Validator will compile and invoke resulting tree
  139. #endif
  140. }
  141. // check ToString and DebugView don't blow up
  142. try {
  143. if (body != null) {
  144. ValidateExpression(body);
  145. }
  146. } catch (TargetInvocationException e) {
  147. return e.InnerException;
  148. }
  149. return null;
  150. } catch (Exception e) {
  151. if (e is TargetInvocationException) {
  152. error = e.InnerException;
  153. } else {
  154. error = e;
  155. }
  156. // Rewriters may cause invalid trees, we'll ignore those failures so hopefully rewriters only fail with legitimate bugs
  157. // A test that legitimately fails due to a NotSupportedException will throw a TestFailedException wrapping the NotSupportedException.
  158. if (error is NotSupportedException && Rewriter != "") {
  159. error = null;
  160. }
  161. }
  162. return error;
  163. }
  164. private static Exception ValidateExpression(Expression expr) {
  165. //verifies that Expression.ToString() and DebugView do not blow up.
  166. try {
  167. expr.ToString();
  168. string s = expr.DebugView();
  169. } catch (Exception e) {
  170. return e;
  171. }
  172. return null;
  173. }
  174. private sealed class TestComparer : IComparer<string> {
  175. private readonly string[] _prefixes;
  176. internal TestComparer(string[] prefixes) {
  177. _prefixes = prefixes;
  178. }
  179. public int Compare(string x, string y) {
  180. int px = MatchPrefix(x, _prefixes);
  181. int py = MatchPrefix(y, _prefixes);
  182. if (px != py) {
  183. return px - py;
  184. }
  185. return string.Compare(x, y, StringComparison.InvariantCultureIgnoreCase);
  186. }
  187. }
  188. private static List<Type> GetTestTypes(List<Assembly> Assemblies)
  189. {
  190. var Types = new List<Type>();
  191. #if !SILVERLIGHT3
  192. Types.AddRange(typeof(ETScenarios.Program).Assembly.GetTypes());
  193. #endif
  194. Types.AddRange(typeof(ExpressionCompiler.Scenario0.Test).Assembly.GetTypes());
  195. foreach(Assembly a in Assemblies){
  196. Types.AddRange(a.GetTypes());
  197. }
  198. return Types;
  199. }
  200. const int MaxName = 64;
  201. public void RunTests() {
  202. bool RunAll = !Utils.ArgumentsHaveTestcaseNames(Arguments);
  203. bool NotExact = Options.NotExact;
  204. bool CompileToMethod = Options.CompileToMethod;
  205. var Rewriters = new List<String>();
  206. Rewriters.AddRange(Options.Rewrite);
  207. if (Rewriters.Count == 0) Rewriters.Add(""); //no rewriter case.
  208. #if SILVERLIGHT
  209. var tests = new Dictionary<string, MethodInfo>();
  210. #else
  211. var tests = new SortedList<string, MethodInfo>(new TestComparer(Arguments));
  212. #endif
  213. foreach (MethodInfo m in typeof(Scenarios).GetMethods(BindingFlags.Public | BindingFlags.Static)) {
  214. string name = GetTestName(m);
  215. if (((RunAll || Utils.InsensitiveStringInArray(name, Arguments) || NotExact) && MatchPrefix(name, Arguments) >= 0 && MatchPrefix(name, new[] { "Positive", "Negative" }) >= 0) || (RunAll && MatchPrefix(name, new[] { "Positive", "Negative" }) >= 0)) {
  216. tests.Add(name, m);
  217. }
  218. }
  219. var warnTypes = new List<Type>();
  220. foreach (Type t in GetTestTypes(PreLoadedAssemblies)) {
  221. if (!t.IsPublic) {
  222. foreach (MethodInfo m in t.GetMethods(BindingFlags.Public | BindingFlags.Static)) {
  223. if (ETUtils.TestAttribute.GetAttribute(m) != null) {
  224. warnTypes.Add(t);
  225. break;
  226. }
  227. }
  228. continue;
  229. }
  230. foreach (MethodInfo m in t.GetMethods(BindingFlags.Public | BindingFlags.Static)) {
  231. var attr = ETUtils.TestAttribute.GetAttribute(m);
  232. if (attr != null && attr.State == ETUtils.TestState.Enabled) {
  233. string name = GetTestName(m);
  234. if (RunAll || Utils.InsensitiveStringInArray(name, Arguments) || (NotExact && MatchPrefix(name, Arguments) >= 0)) {
  235. tests.Add(name, m);
  236. }
  237. }
  238. }
  239. }
  240. var errors = new List<KeyValuePair<string,Exception>>();
  241. int TestCount = 0;
  242. foreach (var pair in tests) {
  243. var Attr = ETUtils.TestAttribute.GetAttribute(pair.Value);
  244. if (Attr != null && Array.IndexOf(Attr.KeyWords, "PartialTrustOnly") >= 0 && (TrustMode == TestRunner.TrustKind.FullTrust || TrustMode == TestRunner.TrustKind.FullTrustOnly ))
  245. continue;
  246. if (Attr != null && Array.IndexOf(Attr.KeyWords, "FullTrustOnly") >= 0 && (TrustMode == TestRunner.TrustKind.PartialTrust || TrustMode == TestRunner.TrustKind.PartialTrustOnly))
  247. continue;
  248. // If mode is FullTrustOnly or PartialTrustOnly, we only run the tests that have that key
  249. if (TrustMode == TestRunner.TrustKind.FullTrustOnly && ! (Attr != null && Array.IndexOf(Attr.KeyWords, "FullTrustOnly") >= 0))
  250. continue;
  251. if (TrustMode == TestRunner.TrustKind.PartialTrustOnly && !(Attr != null && Array.IndexOf(Attr.KeyWords, "PartialTrustOnly") >= 0))
  252. continue;
  253. string name = pair.Key;
  254. MethodInfo test = pair.Value;
  255. Exception error;
  256. foreach (string compl in Rewriters) {
  257. String name1 = name + (compl != "" ? "+" + compl : "");
  258. name1 = FormatName(name1);
  259. Console.Write("Testing " + name1);
  260. error = RunScenario(test, compl, CompileToMethod);
  261. TestCount++;
  262. LogPassFail(name1, error, errors);
  263. }
  264. }
  265. if (Options.VerifyAssembly)
  266. {
  267. try
  268. {
  269. Utils.VerifyAssembly();
  270. }
  271. catch (Exception ex)
  272. {
  273. LogPassFail("Peverify:", ex, errors);
  274. }
  275. }
  276. foreach (var error in errors) {
  277. Console.WriteLine();
  278. Console.WriteLine("test " + error.Key + " threw:");
  279. Console.WriteLine(error.Value);
  280. }
  281. foreach (var type in warnTypes) {
  282. Console.WriteLine();
  283. Console.WriteLine("ERROR: class {0} not public, no tests ran", type);
  284. }
  285. TestsRan = TestCount;
  286. ExitCode = errors.Count + warnTypes.Count;
  287. }
  288. private static string FormatName(string name) {
  289. if (name.Length > MaxName) {
  290. name = name.Substring(0, MaxName - 3) + "...";
  291. }
  292. return name;
  293. }
  294. private static void LogPassFail(String name, Exception error, List<KeyValuePair<string, Exception>> errors) {
  295. Console.Write(new String(' ', 1 + MaxName - name.Length));
  296. if (error == null) {
  297. Console.WriteLine("PASS");
  298. } else {
  299. Console.WriteLine("FAIL");
  300. errors.Add(new KeyValuePair<string, Exception>(name, error));
  301. }
  302. }
  303. private static int MatchPrefix(string name, string[] prefix) {
  304. for (int i = 0, n = prefix.Length; i < n; i++) {
  305. if (name.StartsWith(prefix[i], StringComparison.InvariantCultureIgnoreCase)) {
  306. return i;
  307. }
  308. }
  309. return -1;
  310. }
  311. private static string GetTestName(MethodInfo test) {
  312. string name = test.Name;
  313. if (test.DeclaringType != typeof(Scenarios)) {
  314. if (ETUtils.TestAttribute.IsTest(test)){
  315. return ETUtils.TestAttribute.GetAttribute(test).Description;
  316. }
  317. if (name.StartsWith(test.DeclaringType.Name)) {
  318. name = name.Substring(test.DeclaringType.Name.Length);
  319. }
  320. name = test.DeclaringType.FullName + "." + name;
  321. }
  322. return name;
  323. }
  324. }
  325. static void PrintArguments() {
  326. //Arguments: [/NE|/NotExact] [/F|/FullTrust] [/Rewrite:TestRewritter1,...] TestName TestName
  327. Console.WriteLine("Executes Expression Tree tests.");
  328. Console.WriteLine("Arguments:");
  329. Console.WriteLine("\t[/NE|/NotExact] [/F|/FullTrust] [/Rewrite:TestRewritter1,...] TestName TestName");
  330. Console.WriteLine("\t/NE or /NotExact - treat test names as prefixes. Will search all tests that start with testname. By default searches only for the exact name.");
  331. Console.WriteLine("\t/F or /FullTrust - Runs tests under full trust. By Default tests are run under partial trust. Fulltrust is always used if a code coverage build is detected.");
  332. Console.WriteLine("\t/CompileToMethod or /CTM - uses Lambda.CompileToMethod to compile trees instead of Lambda.Compile.");
  333. Console.WriteLine("\t/VerifyAssembly or /VA - Verifies that the IL generated for the expressions is valid. Implies /FullTrust and /CompileToMethod.");
  334. Console.WriteLine("\t/Rewrite:Rewrittername1,Rewrittername2 - modify tests by running each of the specified rewritters. A comma at the end of the list will also run the tests without modification.");
  335. Console.WriteLine("\tTestName - Runs only tests that match the name. For tests in AstTest's scenario class, the method name. For tests in ETScenarios, the description. If /NotExact is specified, all tests starting with TestName will be run.");
  336. }
  337. public static int Main(string[] args) {
  338. //If we're running under code coverage we require full permissions.
  339. //Some tests will not run properly, but that's a sacrifice we need to make.
  340. #if SILVERLIGHT
  341. FullTrust = true;
  342. #else
  343. //This environment variable is now set for these runs.
  344. if (!String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("THISISCOVERAGE")))
  345. {
  346. Console.WriteLine("Detected code coverage run - Running all scenarios in full trust");
  347. FullTrust = true;
  348. }
  349. #endif
  350. var runner = new TestRunner { Arguments = args };
  351. if (runner.Options.Help) {
  352. PrintArguments();
  353. return 0;
  354. }
  355. if (args.Length > 0) {
  356. // Explicit option to run in full trust
  357. // Useful when running under the debugger, for visualizers, etc.
  358. if (runner.Options.FullTrust) {
  359. FullTrust = true;
  360. }
  361. }
  362. #if !SILVERLIGHT
  363. var s = new Stopwatch();
  364. #endif
  365. int TestsRan = 0;
  366. int LoaderExitCode = 0;
  367. #if !SILVERLIGHT
  368. try {
  369. if (Environment.Version.Major >= 4) {
  370. MoveNoPiaFile("NoPiaTests.Dll");
  371. MoveNoPiaFile("NoPiaPia.dll");
  372. MoveNoPiaFile("NoPiaHelperClass.dll");
  373. MoveNoPiaFile("NoPiaHelper2.dll");
  374. var NoPiaStream = System.IO.File.OpenRead("NoPiaTests.Dll");
  375. Byte[] NoPia = (byte[])Array.CreateInstance(typeof(byte), NoPiaStream.Length);
  376. NoPiaStream.Read(NoPia, 0, (int)NoPiaStream.Length);
  377. runner.PreLoadedAssemblies.Add(Assembly.Load(NoPia));
  378. }
  379. } catch (Exception ex) {
  380. Console.WriteLine("Failed to load Assemblies " + ex.Message);
  381. throw;
  382. }
  383. s.Start();
  384. //Currently, only two trust modes are availlable:
  385. // (default) Partial trust - runs all possible tests under partial trust, runs full trust only tests under full trust
  386. // Fulltrust - runs all possible tests as full trust, doesn't run partial trust requiring tests
  387. //In the future we might have a partial trust only mode for environments where asttest can't be started under full trust
  388. //The reason why TrustMode has four settings is to allow us not to runt he same test multiple times. (under full and partial trust)
  389. if (!FullTrust) {
  390. //PartialTrust run
  391. var setup = AppDomain.CurrentDomain.SetupInformation;
  392. var permissions = new PermissionSet(PermissionState.None);
  393. permissions.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));
  394. permissions.AddPermission(new ReflectionPermission(ReflectionPermissionFlag.RestrictedMemberAccess));
  395. var domain = AppDomain.CreateDomain("Tests", null, setup, permissions);
  396. // Create the test runner in the remote domain
  397. var remoteRunner = (TestRunner)domain.CreateInstanceAndUnwrap(
  398. typeof(TestRunner).Assembly.FullName, typeof(TestRunner).FullName
  399. );
  400. remoteRunner.TrustMode = TestRunner.TrustKind.PartialTrust;
  401. remoteRunner.Arguments = args;
  402. remoteRunner.RunTests();
  403. LoaderExitCode = remoteRunner.ExitCode;
  404. TestsRan += remoteRunner.TestsRan;
  405. runner.TrustMode = TestRunner.TrustKind.FullTrustOnly;
  406. runner.RunTests();
  407. LoaderExitCode += runner.ExitCode;
  408. TestsRan += runner.TestsRan;
  409. } else {
  410. //Fulltrust run
  411. runner.TrustMode = TestRunner.TrustKind.FullTrust;
  412. #else
  413. #if SILVERLIGHT3
  414. //Silverlight 3 can only run the legacy tests, which are all marked FullTrustOnly despite the fact that they work
  415. //on SL3. The proper fix here would be to change the attribute on those tests if they really suppor partial trust
  416. //which would have the added benefit of running them on SL4 (non quirks mode), which we haven't done yet.
  417. //But that's a bigger change + verification pass, and I'm just trying to consolidate my source changes at the moment.
  418. runner.TrustMode = TestRunner.TrustKind.FullTrust; //.PartialTrust; //@RYANOOB - tests that are known to work only in "full trust" probably don't work in Silverlight
  419. #else
  420. runner.TrustMode = TestRunner.TrustKind.PartialTrust; //@RYANOOB - tests that are known to work only in "full trust" probably don't work in Silverlight
  421. #endif
  422. #endif
  423. runner.RunTests();
  424. LoaderExitCode = runner.ExitCode;
  425. TestsRan += runner.TestsRan;
  426. #if !SILVERLIGHT
  427. }
  428. s.Stop();
  429. #endif
  430. string resultStr = String.Format("{0}Ran {1} tests{2}{3}", Environment.NewLine, TestsRan, Environment.NewLine,
  431. LoaderExitCode == 0 ? "All scenarios passed." : "There were " + LoaderExitCode + " failures.");
  432. #if SILVERLIGHT
  433. System.Windows.MessageBox.Show(resultStr);
  434. #else
  435. Console.WriteLine(resultStr);
  436. #endif
  437. return LoaderExitCode ;
  438. }
  439. private static void MoveNoPiaFile(string file)
  440. {
  441. System.IO.File.Copy(System.IO.Path.Combine("NoPia", file), file, true);
  442. System.IO.File.SetAttributes(file, System.IO.FileAttributes.Normal);
  443. }
  444. }
  445. }