/src/Testsuite.hs

  1{-# LANGUAGE UnicodeSyntax, RecordWildCards #-}
  2
  3import qualified Request
  4import qualified RequestEval
  5import qualified Cxx.Show
  6
  7import Control.Exception ()
  8import Control.Monad (unless, forM_, foldM)
  9import System.Environment (getArgs)
 10import System.IO (utf8, stdout, hSetEncoding)
 11import Text.Regex (matchRegex, mkRegex)
 12import Data.List (sort, isPrefixOf)
 13
 14-- Ascii art:
 15
 16esc :: Char
 17esc = '\x1b'
 18
 19colored :: Integer → String → String
 20colored c s = [esc] ++ "[" ++ show c ++ "m" ++ s ++ [esc] ++ "[0m"
 21
 22red, green, yellow, cyan :: String → String
 23red = colored 31
 24green = colored 32
 25yellow = colored 33
 26cyan = colored 36
 27
 28box :: String → String
 29box s = '+' : bar ++ "+\n| " ++ s ++ " |\n+" ++ bar ++ "+"
 30  where bar = replicate (length s + 2) '-'
 31
 32-- Infrastructure:
 33
 34class Show t ⇒ TestPred t where testPred :: t → String → Bool
 35
 36newtype ExactMatch = ExactMatch String deriving Show
 37instance TestPred ExactMatch where testPred (ExactMatch s) = (== s)
 38
 39newtype PrefixMatch = PrefixMatch String deriving Show
 40instance TestPred PrefixMatch where testPred (PrefixMatch s) = isPrefixOf s
 41
 42newtype RegexMatch = RegexMatch String deriving Show
 43instance TestPred RegexMatch where testPred (RegexMatch r) s = matchRegex (mkRegex r) s /= Nothing
 44
 45data NoOutput = NoOutput deriving Show
 46instance TestPred NoOutput where testPred NoOutput = (== "")
 47
 48data Test = Test { test_name, test_request :: String, test_pred :: String → Bool, test_pred_name :: String }
 49
 50test :: TestPred p ⇒ String → String → p → Test
 51test n r p = Test n r (testPred p) (show p)
 52
 53utest :: TestPred p ⇒ String → p → Test
 54utest = test "unnamed test"
 55
 56main :: IO ()
 57main = do
 58
 59  hSetEncoding stdout utf8
 60
 61  evalRequest ← RequestEval.evaluator
 62
 63  putStrLn
 64    "\nNote: In several tests, output is expected to include an error (sometimes on a separate line), so seeing an error in a test's output does not mean the test failed. A test failed if its output is colored red.\n"
 65
 66  let default_test_sets = ["resources", "misc", "diagnostics", "utilities"]
 67
 68  args ← getArgs
 69  forM_ (case args of [] → default_test_sets; _ → args) $ \set → do
 70    putStrLn $ box $ "Test set: " ++ set
 71    foldM (\context (Test n r p pn) → do
 72        putStrLn $ "\nTest: " ++ yellow n
 73        putStrLn $ "Request: " ++ cyan r
 74        Request.Response{..} ← evalRequest r context []
 75        let success = p response_output
 76        putStrLn $ "Output: " ++ (if success then green else red) (if response_output == "" then "<none>" else response_output)
 77        unless success $ putStrLn $ "Expected: " ++ pn
 78        return $ maybe context (`Request.modify_history` context) response_history_modification
 79      ) (Request.Context Cxx.Show.noHighlighting False []) (tests set)
 80    putStrLn ""
 81
 82  putStrLn "Done running tests."
 83
 84-- Actual test:
 85
 86tests :: String → [Test]
 87
 88tests "resources" =
 89  [ test "Program timeout" "{ for(;;) ; }" $ ExactMatch "CPU time limit exceeded"
 90  , test "Stack overflow" "<< f(3); int f(int i) { if (i % 10000 == 0) cout << '+' << flush; return -f(++i); }" $
 91    RegexMatch "\\++ Undefined behavior detected."
 92  , test "Open FDs" "{ for(int i = 0; i != 1024; ++i) assert(i == 1 || i == 2 || close(i) == -1); }" NoOutput
 93  , test "File creation" "{ ofstream f (\"bla\"); assert(errno == EACCES); }" NoOutput
 94  , test "Working directory" "<< get_current_dir_name()" $ ExactMatch "/geordi/run"
 95  , test "File I/O" "{ { ofstream f (__FILE__); f << \"foo\"; } cout << ifstream(__FILE__).rdbuf(); }" $
 96    ExactMatch "foo"
 97  , test "Memory limit" "{ int i = 0; while (new (nothrow) char [1 << 20]) ++i; assert(i < 600); }" NoOutput
 98  , test "Fd limit" "{ int i = 0; while (open(__FILE__, 0) != -1) ++i; assert(errno == EMFILE); assert(i < 50); }  extern \"C\" int open (char const *, int);" NoOutput
 99  , test "File size limit" "{ ofstream f (__FILE__); string meg (1 << 20, 'x'); for (;;) { f << meg << flush; cout << '+' << flush; } }" $
100    RegexMatch "\\+{1,50} File size limit exceeded$"
101  , test "System call interception" "{ wait(nullptr); } #include <sys/wait.h>" $ PrefixMatch "Operation not permitted: wait"
102  , test "Signal" "{ int x = 0; cout << 3 / x; }" $ ExactMatch "Floating point exception"
103  , test "Recursive exec()"
104    "int main (int const argc, char const * const * argv) { string s; if (argc >= 2) s = argv[1]; s += 'x'; if (s.size() % 100 == 0) cout << '+' << flush; execl(\"/geordi/run/t\", \"t\", s.c_str(), 0); }" $
105    RegexMatch "\\++( Alarm clock| CPU time limit exceeded)?$"
106  ]
107
108tests "resume" =
109  [ test "" "<< i;int i=2;" $ ExactMatch "2"
110  , test "" "-r <<i+i" $ ExactMatch "4"
111  , test "" "int i=3; int /**/ main (){} int j=4;" $ ExactMatch ""
112  , test "" "-r <<i+j" $ ExactMatch "7"
113  ]
114
115tests "misc" =
116  [ test "Simple output" "<< 3" $ ExactMatch "3"
117  , let quine = "{string t,u,y(1,34);stringstream i(\"{string t,u,y(1,34);stringstream i(!);getline(i,t,'!')>>u;cout<<t<<y<<i.str()<<y<<u;}\");getline(i,t,'!')>>u;cout<<t<<y<<i.str()<<y<<u;}" in test "Quine" quine $ ExactMatch quine
118  , test "UTF-8 handling" "<< 'b' << char(215) << char(144) << 'c' << char(-5) << 'd'" $
119    ExactMatch "bאc�d"
120  , let s = "dicekjhbagfl" in
121    test "Nontrivial program (Brainfuck interpreter)" ("{b(program);}char program[]=\">>,[>>,]<<[[-<+<]>[>[>>]<[.[-]<[[>>+<<-]<]>>]>]<<]\",input[]=\"" ++ s ++ "\", *i=input,m[512]={},*p=m;void b(char*c){for(;*c&&*c!=']';++c){(*((p+=*c=='>')-=*c=='<')+=*c=='+') -=*c=='-';*c=='.'&&cout<<*p;if(*c==',')*p=*i++;if(*c=='['){for(++c;*p;)b(c);for(int d=0;*c!=']'||d--;++c)d+=*c=='[';}}}") $ ExactMatch (sort s)
122  , test "srand()/time()" "{ srand(time(0)); }" NoOutput
123  , test "line breaks" "#define X \"\\\\\" \\ #define Y X \\ int main() { cout \\ << Y Y; }" $ ExactMatch "\\\\"
124  , test "-v" "-v" $ PrefixMatch "GCC "
125  , test "Clang -v" "-v --clang" $ PrefixMatch "Clang "
126  , test "getopt" "-monkey chicken" $ ExactMatch "error: No such option: -m"
127  , test "operator new/delete overriding" "{ printf(\"| \"); list<int> v(5); } void * operator new(size_t const s) { printf(\"%lu \", (unsigned long)s); return malloc(s); } void operator delete(void * const p) throw() { free(p); }" $ RegexMatch "[^-]*\\| [[:digit:] ]+"
128  , test "multiple TUs" "{ cout << __FILE__, f<int>(); } template<class> string f(); extern template string f<int>(); \\\\ template<class> string f() {return __FILE__;} template string f<int>();" $ ExactMatch "0, 1"
129  ]
130
131tests "diagnostics" =
132  [ test "-fstack-protector-all" "-w { char buf [10]; fill(buf, buf+30, 'x'); }" $
133    PrefixMatch "*** stack smashing detected ***"
134  , test "-mcheck diagnostic" "{ int * p = new int [3]; p[3] = 6; delete[] p; }" $
135    PrefixMatch "memory clobbered past end of allocated block\n"
136  , test "Ditto" "{ int * p = new int [3]; p[-1] = 6; delete[] p; }" $
137    PrefixMatch "memory clobbered before allocated block\n"
138  , test "Checking global allocation/deallocation operators" "{ delete new int[3]; }" $
139    PrefixMatch "error: tried to apply non-array operator delete to pointer returned by new[]. Aborted"
140  , test "Ditto." "{ int * const p = new int; delete p; new int; delete p; }" $ ExactMatch "error: tried to delete already deleted pointer. Aborted"
141  , test "Custom terminate() handler" "{ throw std::logic_error(\"It is not logical, Captain.\"); }" $
142    ExactMatch "terminated by logic_error: It is not logical, Captain."
143  -- todo: , test "libstdc++ debug mode" "<< *vector<int>().begin()" $ PrefixMatch "error: attempt to dereference a past-the-end iterator"
144  , test "Fatal warnings" "{} int f() {}" $ PrefixMatch "warning: "
145  ]
146
147tests "tracked" =
148  [ test "Operation on destructed B" "{ using tracked::B; B x(0), y(x); x.B::~B(); x.f(); }" $ ExactMatch "B0*(0) B1*(B0) B0~ error: tried to call B::f() on destructed B0. Aborted"
149  , test "Re-destruction" "{ using tracked::B; B b; b.~B(); }" $ ExactMatch "B0* B0~ B0~ error: tried to re-destruct destructed B0. Aborted"
150  , test "Stack leak" "{ using tracked::B; { union { double d; char c[sizeof(B)]; }; new (c) B; } B b; }" $ ExactMatch "B0* B1* B1~ leaked: B0. Aborted"
151  , test "Stack overwrite" "{ using tracked::B; B b; new (&b) B; }" $ ExactMatch "B0* error: leaked: B0. Aborted"
152  , test "new[]/delete[]." "{ boost::shared_array<tracked::B> a(new tracked::D[2]); }" $ ExactMatch "new(D[]) B0* D0* B1* D1* D1~ B1~ D0~ B0~ delete[D0, D1]"
153  , test "Operation on non-existent object" "{ tracked::B * p = 0; p->f(); }" $ ExactMatch "error: tried to call B::f() on non-existent object. Aborted"
154  , test "Read from dead object." "-w << f(); using tracked::B; B const & f() { return B(); }" $ ExactMatch "B0* B0~ error: tried to read destructed B0. Aborted"
155  , test "Initialization" "{ tracked::B b = 1, c(1); }" $ ExactMatch "B0*(1) B1*(B0) B0~ B2*(1) B2~ B1~"
156  , test "--trace" "--trace {X x;}struct X{X(){}~X(){}}y;" $ ExactMatch "X::X(){}main{X::X(){}X::~X(){}}X::~X(){}"
157  ]
158
159tests "utilities" =
160  [ test "TYPE" "{ int i = 4; cout << TYPE(++i); }" $ ExactMatch "lvalue int"
161  , test "Range printing" "{ vector<int> v{3, 5, 9, 4, 1}; cout << v; }" $ ExactMatch "{3, 5, 9, 4, 1}"
162  , test "Demangled printable typeid" "<< typeid(int)" $ ExactMatch "int"
163  , test "Custom assert()/abort()" "{ assert(4 > 9); }" $ ExactMatch "Assertion `4 > 9' fails."
164  , test "bin IO manipulator" "<< showbase << uppercase << bin << setfill('_') << internal << setw(10) << 53" $ ExactMatch "0B__110101"
165  , test "BARK" "{ Y y; y.f(0); y.Xi::f(0); } template <typename> struct X {}; template <typename T> struct X<T const> { virtual void f(void(*)()) { BARK; } }; typedef X<std::string const> Xi; struct Y: Xi { void f(void(*)()) { BARK; } };" $ ExactMatch "Y::f(void (*)()) X<const string>::f(void (*)())"
166  , test "SHOW" "{ cout << \"hm\"; SHOW(3 * 2), SHOW(9 - 3); cout << \"ok\"; }" $ ExactMatch "hm 3 * 2 = 6, 9 - 3 = 6 ok"
167    -- (Tests parsep handling.)
168  ]
169
170tests "errorfilters" =
171  [ test "Type cleanup" "{ wistringstream is; !is.str(); }" $
172    ExactMatch "error: no match for 'operator!' in '!wistringstream::str() const()'"
173  , test "Ditto" "<< TYPE(&vector<queue<istream_iterator<int> > >::foo)" $
174    ExactMatch "error: 'foo' is not a member of 'vector<queue<istream_iterator<int>>>'"
175  , test "Preprocessor error" "<< 08" $ ExactMatch "error: invalid digit \"8\" in octal constant"
176  , test "[with ...]-replacement" "<< 1 == 1" $ ExactMatch "error: no match for 'operator==' in 'cout.ostream::operator<<(1) == 1'"
177  , test "Ditto" "template <typename T> void f(); template <> void f<int>() {} template <> void f<int>() {}" $ ExactMatch "error: redefinition of 'void f() [with T = int]'"
178  ]
179
180tests "uncategorized" =
181  [ utest "-version" $ ExactMatch "error: No such option: -e." -- .. rather than "No such option: -v", which we got when certain options (like -v) were treated separately from evaluation options.
182  ]
183
184tests s = error $ "no such test set: " ++ s