PageRenderTime 57ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/Languages/Ruby/IronRuby.Tests/Driver.cs

http://github.com/IronLanguages/main
C# | 635 lines | 507 code | 106 blank | 22 comment | 88 complexity | d1e1b1d859532f48077c95dfc7e4a3ee MD5 | raw file
Possible License(s): CPL-1.0, BSD-3-Clause, ISC, GPL-2.0, MPL-2.0-no-copyleft-exception
  1. /* ****************************************************************************
  2. *
  3. * Copyright (c) Microsoft Corporation.
  4. *
  5. * This source code is subject to terms and conditions of the Apache License, Version 2.0. A
  6. * copy of the license can be found in the License.html file at the root of this distribution. If
  7. * you cannot locate the Apache License, Version 2.0, please send an email to
  8. * ironruby@microsoft.com. By using this source code in any fashion, you are agreeing to be bound
  9. * by the terms of the Apache License, Version 2.0.
  10. *
  11. * You must not remove this notice, or any other, from this software.
  12. *
  13. *
  14. * ***************************************************************************/
  15. using System;
  16. using System.Collections;
  17. using System.Collections.Generic;
  18. using System.IO;
  19. using System.Security;
  20. using System.Security.Permissions;
  21. using System.Security.Policy;
  22. using Microsoft.Scripting.Generation;
  23. using Microsoft.Scripting.Hosting;
  24. using Microsoft.Scripting.Utils;
  25. using IronRuby.Runtime;
  26. using Microsoft.Scripting;
  27. using System.Diagnostics;
  28. using System.Reflection;
  29. using System.Text;
  30. using System.Globalization;
  31. using System.Threading;
  32. using Microsoft.Scripting.Hosting.Providers;
  33. namespace IronRuby.Tests {
  34. public class TestCase {
  35. public Action TestMethod { get; set; }
  36. public OptionsAttribute Options { get; set; }
  37. public string Name { get; set; }
  38. }
  39. [AttributeUsage(AttributeTargets.Method)]
  40. public sealed class RunAttribute : Attribute {
  41. }
  42. [AttributeUsage(AttributeTargets.Method)]
  43. [Serializable]
  44. public sealed class OptionsAttribute : Attribute {
  45. public bool PrivateBinding { get; set; }
  46. public bool NoRuntime { get; set; }
  47. public Type Pal { get; set; }
  48. }
  49. public class TestRuntime {
  50. private readonly Driver/*!*/ _driver;
  51. private readonly string/*!*/ _testName;
  52. private readonly ScriptRuntime _runtime;
  53. private readonly ScriptEngine _engine;
  54. private readonly RubyContext _context;
  55. public string/*!*/ TestName { get { return _testName; } }
  56. public ScriptEngine Engine { get { return _engine; } }
  57. public ScriptRuntime ScriptRuntime { get { return _runtime; } }
  58. public RubyContext Context { get { return _context; } }
  59. public TestRuntime(Driver/*!*/ driver, TestCase/*!*/ testCase) {
  60. _driver = driver;
  61. _testName = testCase.Name;
  62. if (testCase.Options.NoRuntime) {
  63. return;
  64. }
  65. if (_driver.SaveToAssemblies) {
  66. Environment.SetEnvironmentVariable("DLR_AssembliesFileName", _testName);
  67. }
  68. var runtimeSetup = ScriptRuntimeSetup.ReadConfiguration();
  69. var languageSetup = runtimeSetup.AddRubySetup();
  70. runtimeSetup.DebugMode = _driver.IsDebug;
  71. runtimeSetup.PrivateBinding = testCase.Options.PrivateBinding;
  72. runtimeSetup.HostType = typeof(TestHost);
  73. runtimeSetup.HostArguments = new object[] { testCase.Options };
  74. languageSetup.Options["ApplicationBase"] = _driver.BaseDirectory;
  75. languageSetup.Options["NoAdaptiveCompilation"] = _driver.NoAdaptiveCompilation;
  76. languageSetup.Options["CompilationThreshold"] = _driver.CompilationThreshold;
  77. languageSetup.Options["Verbosity"] = 2;
  78. _runtime = Ruby.CreateRuntime(runtimeSetup);
  79. _engine = Ruby.GetEngine(_runtime);
  80. _context = (RubyContext)HostingHelpers.GetLanguageContext(_engine);
  81. }
  82. }
  83. public class TestHost : ScriptHost {
  84. private readonly OptionsAttribute/*!*/ _options;
  85. private readonly PlatformAdaptationLayer/*!*/ _pal;
  86. public TestHost(OptionsAttribute/*!*/ options) {
  87. _options = options;
  88. _pal = options.Pal != null ?
  89. (PlatformAdaptationLayer)Activator.CreateInstance(options.Pal) :
  90. PlatformAdaptationLayer.Default;
  91. }
  92. public override PlatformAdaptationLayer PlatformAdaptationLayer {
  93. get { return _pal; }
  94. }
  95. }
  96. public class Driver {
  97. private Tests _tests;
  98. private readonly List<MutableTuple<string, StackFrame, string, object>>/*!*/ _failedAssertions = new List<MutableTuple<string, StackFrame, string, object>>();
  99. private readonly List<MutableTuple<string, Exception>>/*!*/ _unexpectedExceptions = new List<MutableTuple<string, Exception>>();
  100. private static TestRuntime _testRuntime;
  101. private static bool _excludeSelectedCases;
  102. private static bool _verbose;
  103. private static bool _isDebug;
  104. private static bool _saveToAssemblies;
  105. private static bool _runTokenizerDriver;
  106. private static bool _displayList;
  107. private static bool _partialTrust;
  108. private static bool _noAdaptiveCompilation;
  109. private static int _compilationThreshold;
  110. private static bool _runPython = true;
  111. private readonly string/*!*/ _baseDirectory;
  112. public Driver(string/*!*/ baseDirectory) {
  113. _baseDirectory = baseDirectory;
  114. }
  115. public TestRuntime TestRuntime {
  116. get { return _testRuntime; }
  117. }
  118. public List<MutableTuple<string, StackFrame, string, object>>/*!*/ FailedAssertions {
  119. get { return _failedAssertions; }
  120. }
  121. public List<MutableTuple<string, Exception>>/*!*/ UnexpectedExceptions {
  122. get { return _unexpectedExceptions; }
  123. }
  124. public bool Verbose {
  125. get { return _verbose; }
  126. }
  127. public bool IsDebug {
  128. get { return _isDebug; }
  129. }
  130. public bool SaveToAssemblies {
  131. get { return _saveToAssemblies; }
  132. }
  133. public bool PartialTrust {
  134. get { return _partialTrust; }
  135. }
  136. public bool NoAdaptiveCompilation {
  137. get { return _noAdaptiveCompilation; }
  138. }
  139. public int CompilationThreshold {
  140. get { return _compilationThreshold; }
  141. }
  142. public bool RunPython {
  143. get { return _runPython; }
  144. }
  145. public string BaseDirectory {
  146. get { return _baseDirectory; }
  147. }
  148. private static bool ParseArguments(List<string>/*!*/ args) {
  149. if (args.Contains("/help") || args.Contains("-?") || args.Contains("/?") || args.Contains("-help")) {
  150. Console.WriteLine("Verbose : /verbose");
  151. Console.WriteLine("Partial trust : /partial");
  152. Console.WriteLine("No adaptive compilation : /noadaptive");
  153. Console.WriteLine("Synchronous compilation : /sync0 (-X:CompilationThreshold 0)");
  154. Console.WriteLine("Synchronous compilation : /sync1 (-X:CompilationThreshold 1)");
  155. Console.WriteLine("Interpret only : /interpret (-X:CompilationThreshold Int32.MaxValue)");
  156. Console.WriteLine("Save to assemblies : /save");
  157. Console.WriteLine("Debug Mode : /debug");
  158. Console.WriteLine("Disable Python interop tests : /py-");
  159. Console.WriteLine("Run Specific Tests : [/exclude] [test_to_run ...]");
  160. Console.WriteLine("List Tests : /list");
  161. Console.WriteLine("Tokenizer baseline : /tokenizer <target-dir> <sources-file>");
  162. Console.WriteLine("Productions dump : /tokenizer /prod <target-dir> <sources-file>");
  163. Console.WriteLine("Benchmark : /tokenizer /bm <target-dir> <sources-file>");
  164. }
  165. if (args.Contains("/list")) {
  166. _displayList = true;
  167. return true;
  168. }
  169. if (args.Contains("/verbose")) {
  170. args.Remove("/verbose");
  171. _verbose = true;
  172. }
  173. if (args.Contains("/debug")) {
  174. args.Remove("/debug");
  175. _isDebug = true;
  176. }
  177. if (args.Contains("-D")) {
  178. args.Remove("-D");
  179. _isDebug = true;
  180. }
  181. if (args.Contains("/save")) {
  182. args.Remove("/save");
  183. _saveToAssemblies = true;
  184. }
  185. if (args.Contains("/partial")) {
  186. args.Remove("/partial");
  187. _partialTrust = true;
  188. }
  189. if (args.Contains("-X:NoAdaptiveCompilation")) {
  190. args.Remove("-X:NoAdaptiveCompilation");
  191. _noAdaptiveCompilation = true;
  192. }
  193. if (args.Contains("/noadaptive")) {
  194. args.Remove("/noadaptive");
  195. _noAdaptiveCompilation = true;
  196. }
  197. if (args.Contains("/sync0")) {
  198. args.Remove("/sync0");
  199. _compilationThreshold = 0;
  200. }
  201. if (args.Contains("/sync1")) {
  202. args.Remove("/sync1");
  203. _compilationThreshold = 1;
  204. }
  205. if (args.Contains("/interpret")) {
  206. args.Remove("/interpret");
  207. _compilationThreshold = Int32.MaxValue;
  208. }
  209. if (args.Contains("/py-")) {
  210. args.Remove("/py-");
  211. _runPython = false;
  212. }
  213. if (args.Contains("/py")) {
  214. args.Remove("/py");
  215. _runPython = true;
  216. }
  217. if (args.Contains("/exclude")) {
  218. _excludeSelectedCases = true;
  219. args.Remove("/exclude");
  220. }
  221. if (args.Contains("/tokenizer")) {
  222. args.Remove("/tokenizer");
  223. _runTokenizerDriver = true;
  224. }
  225. return true;
  226. }
  227. public static void Main(string[]/*!*/ arguments) {
  228. List<string> args = new List<string>(arguments);
  229. string culture = Environment.GetEnvironmentVariable("IR_CULTURE");
  230. if (args.Contains("/partial")) {
  231. Console.WriteLine("Running in partial trust");
  232. PermissionSet ps = CreatePermissionSet();
  233. AppDomainSetup setup = new AppDomainSetup();
  234. setup.ApplicationBase = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
  235. AppDomain domain = AppDomain.CreateDomain("Tests", null, setup, ps);
  236. Loader loader = new Loader(args, setup.ApplicationBase);
  237. domain.DoCallBack(new CrossAppDomainDelegate(loader.Run));
  238. Environment.ExitCode = loader.ExitCode;
  239. } else {
  240. if (!String.IsNullOrEmpty(culture)) {
  241. Thread.CurrentThread.CurrentCulture = new CultureInfo(culture, false);
  242. }
  243. Environment.ExitCode = Run(args, AppDomain.CurrentDomain.BaseDirectory);
  244. }
  245. }
  246. public sealed class Loader : MarshalByRefObject {
  247. public int ExitCode;
  248. public readonly List<string>/*!*/ Args;
  249. public readonly string/*!*/ BaseDirectory;
  250. public Loader(List<string>/*!*/ args, string/*!*/ baseDirectory) {
  251. Args = args;
  252. BaseDirectory = baseDirectory;
  253. }
  254. public void Run() {
  255. ExitCode = Driver.Run(Args, BaseDirectory);
  256. }
  257. }
  258. private static PermissionSet/*!*/ CreatePermissionSet() {
  259. #if CLR2
  260. string name = "Internet";
  261. bool foundName = false;
  262. PermissionSet setIntersection = new PermissionSet(PermissionState.Unrestricted);
  263. // iterate over each policy level
  264. IEnumerator e = SecurityManager.PolicyHierarchy();
  265. while (e.MoveNext()) {
  266. PolicyLevel level = (PolicyLevel)e.Current;
  267. PermissionSet levelSet = level.GetNamedPermissionSet(name);
  268. if (levelSet != null) {
  269. foundName = true;
  270. setIntersection = setIntersection.Intersect(levelSet);
  271. }
  272. }
  273. if (setIntersection == null || !foundName) {
  274. setIntersection = new PermissionSet(PermissionState.None);
  275. } else {
  276. setIntersection = new NamedPermissionSet(name, setIntersection);
  277. }
  278. return setIntersection;
  279. #else
  280. // this functionality is not available on Mono (AddHostEvidence is undefined), use dynamic to resolve it at runtime
  281. dynamic e = new Evidence();
  282. e.AddHostEvidence(new Zone(SecurityZone.Internet));
  283. return SecurityManager.GetStandardSandbox((Evidence)e);
  284. #endif
  285. }
  286. public static int Run(List<string>/*!*/ args, string/*!*/ baseDirectory) {
  287. if (Thread.CurrentThread.CurrentCulture.ToString() != "en-US") {
  288. Console.WriteLine("Current culture: {0}", Thread.CurrentThread.CurrentCulture);
  289. }
  290. if (!ParseArguments(args)) {
  291. return -3;
  292. }
  293. int status = 0;
  294. if (_runTokenizerDriver) {
  295. TokenizerTestDriver driver = new TokenizerTestDriver((RubyContext)HostingHelpers.GetLanguageContext(Ruby.CreateEngine()));
  296. if (!driver.ParseArgs(args)) {
  297. return -3;
  298. }
  299. status = driver.RunTests();
  300. } else {
  301. InitializeDomain();
  302. Driver driver = new Driver(baseDirectory);
  303. if (Manual.TestCode.Trim().Length == 0) {
  304. status = driver.RunUnitTests(args);
  305. } else {
  306. driver.RunManualTest();
  307. // for case the test is forgotten, this would fail the test suite:
  308. status = -2;
  309. }
  310. }
  311. // return failure on bad filter (any real failures throw)
  312. return status;
  313. }
  314. private static void InitializeDomain() {
  315. if (_saveToAssemblies) {
  316. string _dumpDir = Path.Combine(Path.GetTempPath(), "RubyTests");
  317. if (Directory.Exists(_dumpDir)) {
  318. Array.ForEach(Directory.GetFiles(_dumpDir), delegate(string file) {
  319. try { File.Delete(Path.Combine(_dumpDir, file)); } catch { /* nop */ }
  320. });
  321. } else {
  322. Directory.CreateDirectory(_dumpDir);
  323. }
  324. Console.WriteLine("Generating binaries to {0}", _dumpDir);
  325. Snippets.SetSaveAssemblies(true, _dumpDir);
  326. }
  327. }
  328. private void RunManualTest() {
  329. Console.WriteLine("Running hardcoded test case");
  330. if (Manual.ParseOnly) {
  331. _testRuntime = new TestRuntime(this, new TestCase { Name = "<manual>" });
  332. Tests.GetRubyTokens(_testRuntime.Context, new LoggingErrorSink(false), Manual.TestCode, !Manual.DumpReductions, Manual.DumpReductions);
  333. } else {
  334. try {
  335. for (int i = 0; i < Manual.RequiredFiles.Length; i += 2) {
  336. File.CreateText(Manual.RequiredFiles[i]).WriteLine(Manual.RequiredFiles[i + 1]);
  337. }
  338. _tests = new Tests(this);
  339. RunTestCase(new TestCase() {
  340. Name = "$manual$",
  341. TestMethod = () => _tests.CompilerTest(Manual.TestCode),
  342. });
  343. } finally {
  344. for (int i = 0; i < Manual.RequiredFiles.Length; i += 2) {
  345. try {
  346. File.Delete(Manual.RequiredFiles[i]);
  347. } catch {
  348. // nop
  349. }
  350. }
  351. }
  352. }
  353. }
  354. private int RunUnitTests(List<string>/*!*/ largs) {
  355. _tests = new Tests(this);
  356. if (_displayList) {
  357. for (int i = 0; i < _tests.TestMethods.Length; i++) {
  358. Console.WriteLine(_tests.TestMethods[i].Method.Name);
  359. }
  360. return -1;
  361. }
  362. // check whether there is a preselected case:
  363. IList<TestCase> selectedCases = new List<TestCase>();
  364. foreach (var m in _tests.TestMethods) {
  365. if (m.Method.IsDefined(typeof(RunAttribute), false)) {
  366. AddTestCases(selectedCases, m);
  367. }
  368. }
  369. if (selectedCases.Count == 0 && largs.Count > 0) {
  370. foreach (var m in _tests.TestMethods) {
  371. bool caseIsSpecified = largs.Contains(m.Method.Name);
  372. if ((caseIsSpecified && !_excludeSelectedCases) || (!caseIsSpecified && _excludeSelectedCases)) {
  373. AddTestCases(selectedCases, m);
  374. }
  375. }
  376. } else if (selectedCases.Count > 0 && largs.Count > 0) {
  377. Console.WriteLine("Arguments overrided by Run attribute.");
  378. } else if (selectedCases.Count == 0 && largs.Count == 0) {
  379. foreach (var m in _tests.TestMethods) {
  380. AddTestCases(selectedCases, m);
  381. }
  382. }
  383. foreach (TestCase testCase in selectedCases) {
  384. RunTestCase(testCase);
  385. }
  386. var failedCases = new List<string>();
  387. if (_failedAssertions.Count > 0) {
  388. for (int i = 0; i < _failedAssertions.Count; i++) {
  389. string test = _failedAssertions[i].Item000;
  390. StackFrame frame = _failedAssertions[i].Item001;
  391. string message = _failedAssertions[i].Item002;
  392. failedCases.Add(test);
  393. Console.Error.WriteLine();
  394. if (_partialTrust) {
  395. ColorWrite(ConsoleColor.Red, "{0}) {1}", failedCases.Count, test);
  396. } else {
  397. ColorWrite(ConsoleColor.Red, "{0}) {1} {2} : {3}", failedCases.Count, test, frame.GetFileName(), frame.GetFileLineNumber());
  398. }
  399. Console.Error.WriteLine(message);
  400. }
  401. }
  402. if (_unexpectedExceptions.Count > 0) {
  403. for (int i = 0; i < _unexpectedExceptions.Count; i++) {
  404. string test = _unexpectedExceptions[i].Item000;
  405. Exception exception = _unexpectedExceptions[i].Item001;
  406. Console.Error.WriteLine();
  407. ColorWrite(ConsoleColor.Red, "{0}) {1} (unexpected exception)", failedCases.Count, test);
  408. Console.Error.WriteLine(exception);
  409. failedCases.Add(test);
  410. }
  411. }
  412. if (failedCases.Count == 0) {
  413. ColorWrite(ConsoleColor.Green, "PASSED");
  414. } else {
  415. Console.WriteLine();
  416. // TODO:
  417. if (!_partialTrust) {
  418. Console.Write("Repro: {0}", Environment.CommandLine);
  419. } else {
  420. Console.Write("Repro: IronRuby.Tests.exe /partial{0}{1}",
  421. _noAdaptiveCompilation ? " /noadaptive" : "",
  422. _isDebug ? " /debug" : "");
  423. }
  424. if (largs.Count == 0) {
  425. Console.Write(" {0}", String.Join(" ", failedCases.ToArray()));
  426. }
  427. Console.WriteLine();
  428. }
  429. return failedCases.Count;
  430. }
  431. private void AddTestCases(IList<TestCase>/*!*/ cases, Action/*!*/ testMethod) {
  432. var attrs = testMethod.Method.GetCustomAttributes(typeof(OptionsAttribute), false);
  433. if (attrs.Length > 0) {
  434. foreach (OptionsAttribute options in attrs) {
  435. cases.Add(new TestCase {
  436. Name = testMethod.Method.Name,
  437. TestMethod = testMethod,
  438. Options = options,
  439. });
  440. }
  441. } else {
  442. cases.Add(new TestCase {
  443. Name = testMethod.Method.Name,
  444. TestMethod = testMethod,
  445. Options = new OptionsAttribute(),
  446. });
  447. }
  448. }
  449. private void RunTestCase(TestCase/*!*/ testCase) {
  450. _testRuntime = new TestRuntime(this, testCase);
  451. if (_verbose) {
  452. Console.WriteLine("Executing {0}", testCase.Name);
  453. } else {
  454. Console.Write('.');
  455. }
  456. try {
  457. testCase.TestMethod();
  458. } catch (Exception e) {
  459. PrintTestCaseFailed();
  460. _unexpectedExceptions.Add(new MutableTuple<string, Exception>(testCase.Name, e));
  461. } finally {
  462. Snippets.SaveAndVerifyAssemblies();
  463. }
  464. }
  465. private void PrintTestCaseFailed() {
  466. ColorWrite(ConsoleColor.Red, "\n> FAILED: {0}", _testRuntime.TestName);
  467. }
  468. internal static void ColorWrite(ConsoleColor color, string/*!*/ str, params object[] args) {
  469. var oldColor = Console.ForegroundColor;
  470. Console.ForegroundColor = color;
  471. Console.WriteLine(str, args);
  472. Console.ForegroundColor = oldColor;
  473. }
  474. [DebuggerHiddenAttribute]
  475. internal void AssertionFailed(string/*!*/ msg) {
  476. var trace = new StackTrace(true);
  477. StackFrame frame = null;
  478. for (int i = 0; i < trace.FrameCount; i++) {
  479. frame = trace.GetFrame(i);
  480. var method = frame.GetMethod();
  481. if (!method.IsDefined(typeof(DebuggerHiddenAttribute), true)) {
  482. break;
  483. }
  484. }
  485. Debug.Assert(frame != null);
  486. _failedAssertions.Add(new MutableTuple<string, StackFrame, string, object>(_testRuntime.TestName, frame, msg, null));
  487. PrintTestCaseFailed();
  488. }
  489. internal string/*!*/ MakeTempDir() {
  490. string dir = Path.Combine(Path.GetTempPath(), _testRuntime.TestName);
  491. Directory.CreateDirectory(dir);
  492. return dir;
  493. }
  494. internal TempFile/*!*/ MakeTempFile(string/*!*/ globalVariableName, string/*!*/ suffix, string/*!*/ content) {
  495. var fileName = Driver.MakeTempFile(suffix, content);
  496. TestRuntime.Context.DefineGlobalVariable(globalVariableName, fileName);
  497. return new TempFile(fileName);
  498. }
  499. internal static string/*!*/ MakeTempFile(string/*!*/ suffix, string/*!*/ content) {
  500. var dir = Path.GetTempPath();
  501. int pid = Process.GetCurrentProcess().Id;
  502. while (true) {
  503. string path = Path.Combine(dir, "IR_" + pid + "_" + DateTime.Now.Ticks.ToString("X") + suffix);
  504. if (!File.Exists(path)) {
  505. try {
  506. using (var file = File.Open(path, FileMode.CreateNew)) {
  507. var writer = new StreamWriter(file);
  508. writer.Write(content);
  509. writer.Close();
  510. }
  511. return path;
  512. } catch (IOException) {
  513. // nop
  514. }
  515. }
  516. }
  517. }
  518. internal sealed class TempFile : IDisposable {
  519. private readonly string/*!*/ _file;
  520. public TempFile(string/*!*/ file) {
  521. Assert.NotNull(file);
  522. _file = file;
  523. }
  524. public void Dispose() {
  525. File.Delete(_file);
  526. }
  527. }
  528. }
  529. }