PageRenderTime 41ms CodeModel.GetById 1ms app.highlight 31ms RepoModel.GetById 1ms app.codeStats 0ms

/IronPython_Main/Runtime/Tests/AstTest/Program.cs

#
C# | 520 lines | 388 code | 75 blank | 57 comment | 91 complexity | d408259395ba2d97b6fb99bd9fecce03 MD5 | raw file
  1/* ****************************************************************************
  2 *
  3 * Copyright (c) Microsoft Corporation. 
  4 *
  5 * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
  6 * copy of the license can be found in the License.html file at the root of this distribution. If 
  7 * you cannot locate the  Apache License, Version 2.0, please send an email to 
  8 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
  9 * by the terms of the Apache License, Version 2.0.
 10 *
 11 * You must not remove this notice, or any other, from this software.
 12 *
 13 *
 14 * ***************************************************************************/
 15
 16#if !CLR2
 17using System.Linq.Expressions;
 18#else
 19using Microsoft.Scripting.Ast;
 20#endif
 21
 22using System;
 23using System.Collections.Generic;
 24using System.Diagnostics;
 25using System.Reflection;
 26using System.Security;
 27using System.Security.Permissions;
 28using ETUtils;
 29using Utils = AstTest.Utils;
 30using EU = ETUtils.ExpressionUtils;
 31
 32namespace AstTest {
 33
 34    //
 35    // Quick note on the meaning of "Positive" "Negative" and "Slow" test
 36    // prefixes:
 37    //
 38    //   Positive: this test throws no exceptions while running. We want these
 39    //     to run first
 40    //
 41    //   Negative: this test intentionally throws an exception while running.
 42    //     These are run after positive tests so we don't see intentional
 43    //     exceptions when debugging positive test failures.
 44    //
 45    //   Slow: this test takes a while to run because it compiles lots of
 46    //     trees. For this reason, we want it to run last.
 47    //
 48    //   Disabled: the test is disabled. It can still be run manually
 49    //
 50
 51    public static partial class Scenarios {
 52        public static bool FullTrust;
 53
 54
 55#if SILVERLIGHT
 56        public class ArgumentData {
 57#else
 58        public class ArgumentData : MarshalByRefObject {
 59#endif
 60            public Boolean CompileToMethod = false;
 61            public Boolean VerifyAssembly = false;
 62            public Boolean FullTrust = false;
 63            public Boolean NotExact = false;
 64            public Boolean Help = false;
 65            public List<string> Rewrite = new List<string>();
 66            public List<string> Testcases = new List<string>();
 67
 68            public void Initialize(string[] args) {
 69                foreach (string s in args) {
 70                    String temp;
 71#if SILVERLIGHT3
 72                    temp = s.ToLower();
 73#else
 74                    temp = s.ToLowerInvariant();
 75#endif
 76                    if (temp.StartsWith("/") || temp.StartsWith("-")) {
 77                        //switch
 78
 79                        //clear / or -
 80                        temp = temp.Substring(1);
 81
 82                        if (temp.Equals("ne") || temp.Equals("notexact")) {
 83                            NotExact = true;
 84                        }
 85
 86                        if (temp.Equals("ctm") || temp.Equals("compiletomethod")) {
 87                            CompileToMethod = true;
 88                        }
 89
 90                        if (temp.Equals("va") || temp.Equals("verifyassembly")) {
 91                            CompileToMethod = true;
 92                            VerifyAssembly = true;
 93                            FullTrust = true;
 94                        }
 95
 96                        if (temp.Equals("f") || temp.Equals("fulltrust")) {
 97                            FullTrust = true;
 98                        }
 99
100                        if (temp.Equals("?") || temp.Equals("help")) {
101                            Help = true;
102                        }
103
104                        if (temp.StartsWith("r") || temp.StartsWith("rewrite")) {
105                            //get rewriters
106                            Rewrite.AddRange(Utils.GetArgumentValues(temp));
107                        }
108
109                    } else {
110                        //testcase
111                        Testcases.Add(temp);
112                    }
113                }
114            }
115        }
116
117#if SILVERLIGHT
118        public sealed class TestRunner {
119#else
120        public sealed class TestRunner : MarshalByRefObject {
121#endif
122            private string[] _arguments;
123            public string[] Arguments {
124                get {
125                    return _arguments;
126                }
127                set {
128                    _arguments = value;
129                    Options.Initialize(value);
130                }
131            }
132            public readonly ArgumentData Options = new ArgumentData();
133            public int ExitCode { get; private set; }
134            public int TestsRan { get; private set; }
135            public TrustKind TrustMode { get; set; }
136            public readonly List<Assembly> PreLoadedAssemblies = new List<Assembly>();
137
138            public enum TrustKind {
139                FullTrust,
140                FullTrustOnly,
141                PartialTrust,
142                PartialTrustOnly
143            }
144
145            private Exception RunScenario(MethodInfo test, String Rewriter, bool CompileToMethod) {
146                var attr = ETUtils.TestAttribute.GetAttribute(test);
147
148                Exception error = null;
149                try {
150                    Expression body = null;
151                    if (test.GetParameters().Length == 0) { // legacy LINQ tests, NoPia tests
152                        test.Invoke(null, new object[] {});
153                    } else {
154#if SILVERLIGHT3
155                        throw new Exception("Should not be here"); //@RYANOOB
156#else
157                        var V = new EU.TestValidator(Rewriter, Options.CompileToMethod, Options.VerifyAssembly);
158                        body = (Expression)test.Invoke(null, new object[] { V }); // Validator will compile and invoke resulting tree
159#endif
160                    }
161
162                    // check ToString and DebugView don't blow up
163                    try {
164                        if (body != null) {
165                            ValidateExpression(body);
166                        }
167                    } catch (TargetInvocationException e) {
168                        return e.InnerException;
169                    }
170
171                    return null;
172                } catch (Exception e) {
173                    if (e is TargetInvocationException) {
174                        error = e.InnerException;
175                    } else {
176                        error = e;
177                    }
178                    // Rewriters may cause invalid trees, we'll ignore those failures so hopefully rewriters only fail with legitimate bugs
179                    // A test that legitimately fails due to a NotSupportedException will throw a TestFailedException wrapping the NotSupportedException.
180                    if (error is NotSupportedException && Rewriter != "") {
181                        error = null;
182                    }
183                }
184
185                return error;
186            }
187
188            private static Exception ValidateExpression(Expression expr) {
189                //verifies that Expression.ToString() and DebugView do not blow up.
190                try {
191                    expr.ToString();
192                    string s = expr.DebugView();
193                } catch (Exception e) {
194                    return e;
195                }
196                return null;
197            }
198            
199            private sealed class TestComparer : IComparer<string> {
200                private readonly string[] _prefixes;
201
202                internal TestComparer(string[] prefixes) {
203                    _prefixes = prefixes;
204                }
205
206                public int Compare(string x, string y) {
207                    int px = MatchPrefix(x, _prefixes);
208                    int py = MatchPrefix(y, _prefixes);
209                    if (px != py) {
210                        return px - py;
211                    }
212                    return string.Compare(x, y, StringComparison.InvariantCultureIgnoreCase);
213                }
214            }
215
216            private static List<Type> GetTestTypes(List<Assembly> Assemblies)
217            {
218                var Types = new List<Type>();
219#if !SILVERLIGHT3
220                Types.AddRange(typeof(ETScenarios.Program).Assembly.GetTypes());
221#endif
222                Types.AddRange(typeof(ExpressionCompiler.Scenario0.Test).Assembly.GetTypes());
223
224                foreach(Assembly a in Assemblies){
225                    Types.AddRange(a.GetTypes());
226                }
227                return Types;
228            }
229
230            const int MaxName = 64;
231
232            public void RunTests() {
233                bool RunAll = !Utils.ArgumentsHaveTestcaseNames(Arguments);
234
235                bool NotExact = Options.NotExact;
236                bool CompileToMethod = Options.CompileToMethod;
237
238                var Rewriters = new List<String>();
239                Rewriters.AddRange(Options.Rewrite);
240
241                if (Rewriters.Count == 0) Rewriters.Add(""); //no rewriter case.
242
243#if SILVERLIGHT
244                var tests = new Dictionary<string, MethodInfo>();
245#else
246                var tests = new SortedList<string, MethodInfo>(new TestComparer(Arguments));
247#endif
248
249                foreach (MethodInfo m in typeof(Scenarios).GetMethods(BindingFlags.Public | BindingFlags.Static)) {
250                    string name = GetTestName(m);
251                    if (((RunAll || Utils.InsensitiveStringInArray(name, Arguments) || NotExact) && MatchPrefix(name, Arguments) >= 0 && MatchPrefix(name, new[] { "Positive", "Negative" }) >= 0) || (RunAll && MatchPrefix(name, new[] { "Positive", "Negative" }) >= 0)) {
252                        tests.Add(name, m);
253                    }
254                }
255
256                var warnTypes = new List<Type>();
257                foreach (Type t in GetTestTypes(PreLoadedAssemblies)) {
258                    if (!t.IsPublic) {
259                        foreach (MethodInfo m in t.GetMethods(BindingFlags.Public | BindingFlags.Static)) {
260                            if (ETUtils.TestAttribute.GetAttribute(m) != null) {
261                                warnTypes.Add(t);
262                                break;
263                            }
264                        }
265                        continue;
266                    }
267                    foreach (MethodInfo m in t.GetMethods(BindingFlags.Public | BindingFlags.Static)) {
268                        var attr = ETUtils.TestAttribute.GetAttribute(m);
269                        if (attr != null && attr.State == ETUtils.TestState.Enabled) {
270                            string name = GetTestName(m);
271                            if (RunAll || Utils.InsensitiveStringInArray(name, Arguments) || (NotExact && MatchPrefix(name, Arguments) >= 0)) {
272                                tests.Add(name, m);
273                            }
274                        }
275                    }
276                }
277
278                var errors = new List<KeyValuePair<string,Exception>>();
279                int TestCount = 0;
280
281                foreach (var pair in tests) {
282
283                    var Attr = ETUtils.TestAttribute.GetAttribute(pair.Value);
284                    if (Attr != null && Array.IndexOf(Attr.KeyWords, "PartialTrustOnly") >= 0 && (TrustMode == TestRunner.TrustKind.FullTrust || TrustMode == TestRunner.TrustKind.FullTrustOnly ))
285                        continue;
286
287                    if (Attr != null && Array.IndexOf(Attr.KeyWords, "FullTrustOnly") >= 0 && (TrustMode == TestRunner.TrustKind.PartialTrust || TrustMode == TestRunner.TrustKind.PartialTrustOnly))
288                        continue;
289
290                    // If mode is FullTrustOnly or PartialTrustOnly, we only run the tests that have that key
291                    if (TrustMode == TestRunner.TrustKind.FullTrustOnly && ! (Attr != null && Array.IndexOf(Attr.KeyWords, "FullTrustOnly") >= 0))
292                        continue;
293
294                    if (TrustMode == TestRunner.TrustKind.PartialTrustOnly && !(Attr != null && Array.IndexOf(Attr.KeyWords, "PartialTrustOnly") >= 0))
295                        continue;
296                    
297                    string name = pair.Key;
298                    MethodInfo test = pair.Value;
299
300                    Exception error;
301                    foreach (string compl in Rewriters) {
302                        String name1 = name + (compl != "" ? "+" + compl : "");
303                        name1 = FormatName(name1);
304
305                        Console.Write("Testing " + name1);
306
307                        error = RunScenario(test, compl, CompileToMethod);
308                        TestCount++;
309
310                        LogPassFail(name1, error, errors);
311                    }
312                }
313
314                if (Options.VerifyAssembly)
315                {
316                    try
317                    {
318                        Utils.VerifyAssembly();
319                    }
320                    catch (Exception ex)
321                    {
322                        LogPassFail("Peverify:", ex, errors);
323                    }
324                }
325
326                foreach (var error in errors) {
327                    Console.WriteLine();
328                    Console.WriteLine("test " + error.Key + " threw:");
329                    Console.WriteLine(error.Value);
330                }
331
332                foreach (var type in warnTypes) {
333                    Console.WriteLine();
334                    Console.WriteLine("ERROR: class {0} not public, no tests ran", type);
335                }
336
337                TestsRan = TestCount;
338                ExitCode = errors.Count + warnTypes.Count;
339            }
340
341            private static string FormatName(string name) {
342                if (name.Length > MaxName) {
343                    name = name.Substring(0, MaxName - 3) + "...";
344                }
345                return name;
346            }
347
348            private static void LogPassFail(String name, Exception error, List<KeyValuePair<string, Exception>> errors) {
349                Console.Write(new String(' ', 1 + MaxName - name.Length));
350                if (error == null) {
351                    Console.WriteLine("PASS");
352                } else {
353                    Console.WriteLine("FAIL");
354                    errors.Add(new KeyValuePair<string, Exception>(name, error));
355                }
356            }
357
358            private static int MatchPrefix(string name, string[] prefix) {
359                for (int i = 0, n = prefix.Length; i < n; i++) {
360                    if (name.StartsWith(prefix[i], StringComparison.InvariantCultureIgnoreCase)) {
361                        return i;
362                    }
363                }
364                return -1;
365            }
366
367            private static string GetTestName(MethodInfo test) {
368                string name = test.Name;
369                if (test.DeclaringType != typeof(Scenarios)) {
370                    if (ETUtils.TestAttribute.IsTest(test)){
371                        return ETUtils.TestAttribute.GetAttribute(test).Description;
372                    }
373                    if (name.StartsWith(test.DeclaringType.Name)) {
374                        name = name.Substring(test.DeclaringType.Name.Length);
375                    }
376                    name = test.DeclaringType.FullName + "." + name;
377                }
378                return name;
379            }
380
381        }
382
383        static void PrintArguments() {
384            //Arguments: [/NE|/NotExact] [/F|/FullTrust] [/Rewrite:TestRewritter1,...] TestName TestName
385            Console.WriteLine("Executes Expression Tree tests.");
386            Console.WriteLine("Arguments:");
387            Console.WriteLine("\t[/NE|/NotExact] [/F|/FullTrust] [/Rewrite:TestRewritter1,...] TestName TestName");
388            Console.WriteLine("\t/NE or /NotExact - treat test names as prefixes. Will search all tests that start with testname. By default searches only for the exact name.");
389            Console.WriteLine("\t/F or /FullTrust - Runs tests under full trust. By Default tests are run under partial trust. Fulltrust is always used if a code coverage build is detected.");
390            Console.WriteLine("\t/CompileToMethod or /CTM - uses Lambda.CompileToMethod to compile trees instead of Lambda.Compile.");
391            Console.WriteLine("\t/VerifyAssembly or /VA - Verifies that the IL generated for the expressions is valid. Implies /FullTrust and /CompileToMethod.");
392            Console.WriteLine("\t/Rewrite:Rewrittername1,Rewrittername2 - modify tests by running each of the specified rewritters. A comma at the end of the list will also run the tests without modification.");
393            Console.WriteLine("\tTestName - Runs only tests that match the name. For tests in AstTest's scenario class, the method name. For tests in ETScenarios, the description. If /NotExact is specified, all tests starting with TestName will be run.");
394        }
395
396        public static int Main(string[] args) {
397            //If we're running under code coverage we require full permissions.
398            //Some tests will not run properly, but that's a sacrifice we need to make.
399
400#if SILVERLIGHT
401            FullTrust = true;
402#else
403            //This environment variable is now set for these runs.
404            if (!String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("THISISCOVERAGE")))
405            {
406                Console.WriteLine("Detected code coverage run - Running all scenarios in full trust");
407                FullTrust = true;
408            }
409#endif
410
411            var runner = new TestRunner { Arguments = args };
412
413            if (runner.Options.Help) {
414                PrintArguments();
415                return 0;
416            }
417
418            if (args.Length > 0) {
419                // Explicit option to run in full trust
420                // Useful when running under the debugger, for visualizers, etc.
421                if (runner.Options.FullTrust) {
422                    FullTrust = true;
423                }
424            }
425
426#if !SILVERLIGHT
427            var s = new Stopwatch();
428#endif
429
430            int TestsRan = 0;
431            int LoaderExitCode = 0;
432#if !SILVERLIGHT
433            try {
434                if (Environment.Version.Major >= 4) {
435                    MoveNoPiaFile("NoPiaTests.Dll");
436                    MoveNoPiaFile("NoPiaPia.dll");
437                    MoveNoPiaFile("NoPiaHelperClass.dll");
438                    MoveNoPiaFile("NoPiaHelper2.dll");
439
440                    var NoPiaStream = System.IO.File.OpenRead("NoPiaTests.Dll");
441                    Byte[] NoPia = (byte[])Array.CreateInstance(typeof(byte), NoPiaStream.Length);
442                    NoPiaStream.Read(NoPia, 0, (int)NoPiaStream.Length);
443
444                    runner.PreLoadedAssemblies.Add(Assembly.Load(NoPia));
445                }
446            } catch (Exception ex) {
447                Console.WriteLine("Failed to load Assemblies " + ex.Message);
448                throw;
449            }
450
451
452            s.Start();
453
454            //Currently, only two trust modes are availlable:
455            // (default) Partial trust - runs all possible tests under partial trust, runs full trust only tests under full trust 
456            // Fulltrust - runs all possible tests as full trust, doesn't run partial trust requiring tests
457            //In the future we might have a partial trust only mode for environments where asttest can't be started under full trust
458            //The reason why TrustMode has four settings is to allow us not to runt he same test multiple times. (under full and partial trust)
459            if (!FullTrust) {
460                //PartialTrust run
461
462                var setup = AppDomain.CurrentDomain.SetupInformation;
463                var permissions = new PermissionSet(PermissionState.None);
464                permissions.AddPermission(new SecurityPermission(SecurityPermissionFlag.Execution));
465                permissions.AddPermission(new ReflectionPermission(ReflectionPermissionFlag.RestrictedMemberAccess));
466                var domain = AppDomain.CreateDomain("Tests", null, setup, permissions);
467
468                // Create the test runner in the remote domain
469                var remoteRunner = (TestRunner)domain.CreateInstanceAndUnwrap(
470                    typeof(TestRunner).Assembly.FullName, typeof(TestRunner).FullName
471                );
472                remoteRunner.TrustMode = TestRunner.TrustKind.PartialTrust;
473                remoteRunner.Arguments = args;
474                remoteRunner.RunTests();
475                LoaderExitCode = remoteRunner.ExitCode;
476                TestsRan += remoteRunner.TestsRan;
477
478                runner.TrustMode = TestRunner.TrustKind.FullTrustOnly;
479                runner.RunTests();
480                LoaderExitCode += runner.ExitCode;
481                TestsRan += runner.TestsRan;
482            } else {
483                //Fulltrust run
484                runner.TrustMode = TestRunner.TrustKind.FullTrust;
485#else
486#if SILVERLIGHT3
487		//Silverlight 3 can only run the legacy tests, which are all marked FullTrustOnly despite the fact that they work
488		//on SL3.  The proper fix here would be to change the attribute on those tests if they really suppor partial trust
489		//which would have the added benefit of running them on SL4 (non quirks mode), which we haven't done yet.
490		//But that's a bigger change + verification pass, and I'm just trying to consolidate my source changes at the moment.
491                runner.TrustMode = TestRunner.TrustKind.FullTrust; //.PartialTrust; //@RYANOOB - tests that are known to work only in "full trust" probably don't work in Silverlight
492#else
493                runner.TrustMode = TestRunner.TrustKind.PartialTrust; //@RYANOOB - tests that are known to work only in "full trust" probably don't work in Silverlight
494#endif
495#endif
496                runner.RunTests();
497                LoaderExitCode = runner.ExitCode;
498                TestsRan += runner.TestsRan;
499#if !SILVERLIGHT
500            }
501            s.Stop();
502#endif
503
504            string resultStr = String.Format("{0}Ran {1} tests{2}{3}", Environment.NewLine, TestsRan, Environment.NewLine,
505                LoaderExitCode == 0 ? "All scenarios passed." : "There were " + LoaderExitCode + " failures.");
506#if SILVERLIGHT
507            System.Windows.MessageBox.Show(resultStr);
508#else
509            Console.WriteLine(resultStr);
510#endif
511            return LoaderExitCode ;
512        }
513
514        private static void MoveNoPiaFile(string file)
515        {
516            System.IO.File.Copy(System.IO.Path.Combine("NoPia", file), file, true);
517            System.IO.File.SetAttributes(file, System.IO.FileAttributes.Normal);
518        }
519    }
520}