/tests/makewebstatistics.d
D | 700 lines | 675 code | 24 blank | 1 comment | 12 complexity | 77a432f36f4f62c5ed958adc91603933 MD5 | raw file
Possible License(s): BSD-3-Clause, AGPL-1.0
- // Based on DSTRESS code by Thomas K端hne
- module findregressions;
- private import std.string;
- private import std.conv;
- private import std.stdio;
- private import std.stream;
- private import std.file;
- private import std.c.stdlib;
- private import std.date;
- private import std.path;
- enum Result{
- UNTESTED = 0,
- PASS = 1 << 2,
- XFAIL = 2 << 2,
- XPASS = 3 << 2,
- FAIL = 4 << 2,
- ERROR = 5 << 2,
- BASE_MASK = 7 << 2,
- EXT_MASK = 3,
- BAD_MSG = 1,
- BAD_GDB = 2,
-
- MAX = BAD_GDB + BASE_MASK
- }
- char[] toString(Result r){
- switch(r & Result.BASE_MASK){
- case Result.PASS: return "PASS";
- case Result.XPASS: return "XPASS";
- case Result.FAIL: return "FAIL";
- case Result.XFAIL: return "XFAIL";
- case Result.ERROR: return "ERROR";
- case Result.UNTESTED: return "UNTESTED";
- default:
- break;
- }
- throw new Exception(format("unhandled Result value %s", cast(int)r));
- }
- char[] dateString(){
- static char[] date;
- if(date is null){
- auto time = getUTCtime();
- auto year = YearFromTime(time);
- auto month = MonthFromTime(time);
- auto day = DateFromTime(time);
- date = format("%d-%02d-%02d", year, month+1, day);
- }
- return date;
- }
- char[][] unique(char[][] a){
- char[][] b = a.sort;
- char[][] back;
- back ~= b[0];
- size_t ii=0;
- for(size_t i=0; i<b.length; i++){
- if(back[ii]!=b[i]){
- back~=b[i];
- ii++;
- }
- }
- return back;
- }
- private{
- version(Windows){
- import std.c.windows.windows;
- extern(Windows) BOOL GetFileTime(HANDLE hFile, LPFILETIME lpCreationTime, LPFILETIME lpLastAccessTime, LPFILETIME lpLastWriteTime);
- }else version(linux){
- import std.c.linux.linux;
- version = Unix;
- }else version(Unix){
- import std.c.unix.unix;
- }else{
- static assert(0);
- }
- alias ulong FStime;
- FStime getFStime(char[] fileName){
- version(Windows){
- HANDLE h;
-
- if (useWfuncs){
- wchar* namez = std.utf.toUTF16z(fileName);
- h = CreateFileW(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
- FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
- }else{
- char* namez = toMBSz(fileName);
- h = CreateFileA(namez,GENERIC_WRITE,0,null,OPEN_ALWAYS,
- FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,cast(HANDLE)null);
- }
- if (h == INVALID_HANDLE_VALUE)
- goto err;
- FILETIME creationTime;
- FILETIME accessTime;
- FILETIME writeTime;
-
- BOOL b = GetFileTime(h, &creationTime, &accessTime, &writeTime);
- if(b==1){
- long modA = writeTime.dwLowDateTime;
- long modB = writeTime.dwHighDateTime;
- return modA | (modB << (writeTime.dwHighDateTime.sizeof*8));
- }
- err:
- CloseHandle(h);
- throw new Exception("failed to query file modification : "~fileName);
- }else version(Unix){
- char* namez = toStringz(fileName);
- struct_stat statbuf;
-
- if(stat(namez, &statbuf)){
- throw new FileException(fileName, getErrno());
- }
- return statbuf.st_mtime;
- }else{
- static assert(0);
- }
- }
- }
- char[] cleanFileName(char[] file){
- char[] back;
- bool hadSep;
- foreach(char c; file){
- if(c == '/' || c == '\\'){
- if(!hadSep){
- back ~= '/';
- hadSep = true;
- }
- }else{
- back ~= c;
- hadSep = false;
- }
- }
- size_t start = 0;
- while(back[start] <= ' ' && start < back.length){
- start++;
- }
- size_t end = back.length-1;
- while(back[end] <= ' ' && end >= start){
- end--;
- }
- back = back[start .. end+1];
- return back;
- }
- class Test{
- char[] name;
- char[] file;
- Result r;
- this(char[] file){
- this.file = file;
- int start = rfind(file, "/");
- if(start<0){
- start = 0;
- }else{
- start += 1;
- }
-
- int end = rfind(file, ".");
- if(end < start){
- end = file.length;
- }
- name = file[start .. end];
- }
- }
- class Log{
- Test[char[]] tests;
- char[] id;
- int[Result] counts;
- this(char[] id, char[] file){
- this.id = id;
- counts = [
- Result.PASS: 0,
- Result.FAIL: 0,
- Result.XPASS: 0,
- Result.XFAIL: 0,
- Result.ERROR: 0 ];
- writefln("parsing: %s", file);
- FStime logTime = getFStime(file);
- Stream source = new BufferedFile(file, FileMode.In);
- while(!source.eof()){
- add(source.readLine());
- }
- dropBogusResults(logTime, "dstress");
- }
- void dropBogusResults(FStime recordTime, char[] testRoot){
- uint totalCount = tests.length;
-
- char[][] sourcesTests = tests.keys;
- foreach(char[] source; sourcesTests){
- if(find(source, "complex/") < 0){
- try{
- FStime caseTime = getFStime(testRoot~std.path.sep~source);
- if(caseTime > recordTime){
- debug(drop) fwritefln(stderr, "dropped: %s", source);
- counts[tests[source].r & Result.BASE_MASK]--;
- tests.remove(source);
- continue;
- }
- }catch(Exception e){
- debug(drop) fwritefln(stderr, "dropped: %s", source);
- counts[tests[source].r & Result.BASE_MASK]--;
- tests.remove(source);
- continue;
- }
- }
- // asm-filter
- int i = find(source, "asm_p");
- if(i >= 0){
- counts[tests[source].r & Result.BASE_MASK]--;
- tests.remove(source);
- continue;
- }
- }
- tests.rehash;
-
- writefln("dropped %s outdated tests (%s remaining)", totalCount - tests.length, tests.length);
- }
-
- bool add(char[] line){
- const char[] SUB = "Torture-Sub-";
- const char[] TORTURE = "Torture:";
- line = strip(line);
- int id = -1;
- Result r = Result.UNTESTED;
- if(line.length > SUB.length && line[0 .. SUB.length] == SUB){
- line = line[SUB.length .. $];
- id = 0;
- while(line[id] >= '0' && line[id] <= '9'){
- id++;
- }
- int start = id;
- id = std.conv.toUint(line[0 .. id]);
- while(line[start] != '-'){
- start++;
- }
- line = line[start+1 .. $];
- }
- char[][] token = split(line);
- if(token.length < 2){
- return false;
- }
- char[] file = strip(token[1]);
- switch(token[0]){
- case "PASS:":
- r = Result.PASS; break;
- case "FAIL:":
- r = Result.FAIL; break;
- case "XPASS:":
- r = Result.XPASS; break;
- case "XFAIL:":
- r = Result.XFAIL; break;
- case "ERROR:":
- r = Result.ERROR; break;
- default:{
- if(token[0] == TORTURE){
- throw new Exception("not yet handled: "~line);
- }else if(id > -1){
- throw new Exception(format("bug in SUB line: (%s) %s", id, line));
- }
- }
- }
- if(r != Result.UNTESTED){
- if(std.string.find(line, "bad error message") > -1){
- r |= Result.BAD_MSG;
- }
- if(std.string.find(line, "bad debugger message") > -1){
- r |= Result.BAD_MSG;
- }
-
- file = cleanFileName(file);
-
- if(id >= 0){
- // update sub
- id--;
-
- Test* test = file in tests;
- if(test is null){
- Test t = new Test(file);
- tests[file] = t;
- t.r = r;
- counts[r & Result.BASE_MASK]++;
- }else{
- if(test.r != Result.UNTESTED){
- test.r = Result.UNTESTED;
- }
- test.r = r;
- }
- }
- return true;
- }
- return false;
- }
- }
- char[] basedir = "web";
- bool regenerate = false;
- int main(char[][] args){
- if(args.length < 3 || (args[1] == "--regenerate" && args.length < 4)){
- fwritefln(stderr, "%s [--regenerate] <reference-log> <log> <log> ...", args[0]);
- fwritefln(stderr, "bash example: %s reference/dmd-something $(ls reference/ldc*)", args[0]);
- return 1;
- }
- char[] reference;
- char[][] files;
- if(args[1] == "--regenerate") {
- regenerate = true;
- reference = args[2];
- files = args[3..$] ~ reference;
- } else {
- reference = args[1];
- files = args[2..$] ~ reference;
- }
- // make sure base path exists
- if(std.file.exists(basedir) && !std.file.isdir(basedir))
- throw new Exception(basedir ~ " is not a directory!");
- else if(!std.file.exists(basedir))
- std.file.mkdir(basedir);
-
-
- Log[char[]] logs;
- // emit per-log data
- foreach(char[] file; files)
- generateLogStatistics(file, logs);
-
- // differences between logs
- foreach(int i, char[] file; files[1 .. $])
- generateChangeStatistics(files[1+i], files[1+i-1], logs);
- // differences between reference and logs
- foreach(char[] file; files[0..$-1])
- generateChangeStatistics(file, reference, logs);
- // collect all the stats.base files into a large table
- BufferedFile index = new BufferedFile(std.path.join(basedir, "index.html"), FileMode.OutNew);
- scope(exit) index.close();
- index.writefln(`
- <!DOCTYPE html>
- <html>
- <head>
- <title>DStress results for x86-32 Linux</title>
- <style type="text/css">
- body {
- font-family: Arial, Helvetica, sans-serif;
- font-size: 0.8em;
- }
- a {
- text-decoration: none;
- }
- a:hover {
- border-bottom: 1px dotted blue;
- }
- table {
- border-collapse: collapse;
- }
- tr {
- border-bottom: 1px solid #CCC;
- }
- tr.odd {
- background: #e0e0e0;
- }
- tr.head {
- border-bottom: none;
- }
- td,th {
- padding: 2px 10px 2px 10px;
- }
- .result:hover {
- background: #C3DFFF;
- }
- .pass,.xfail,.xpass,.fail,.xpass,.error,.generic {
- text-align: center;
- }
- .generic {
- background: #EEE;
- color: gray;
- }
- .pass {
- background: #98FF90;
- color: green;
- }
- tr:hover .pass {
- background: #83E67B;
- }
- .xfail {
- background: #BDFFB8;
- color: #0CAE00;
- }
- tr:hover .xfail {
- background: #98FF90;
- }
- .fail {
- background: #FF6E7A;
- color: maroon;
- }
- .xpass {
- background: #FF949D;
- color: maroon;
- }
- .error {
- background: #FFB3B9;
- color: maroon;
- }
- .borderleft {
- border-left: 1px solid #CCC;
- }
- </style>
- </head>
- <body>
- <h1>DStress results for x86-32 Linux</h1>
- <h2>Legend</h2>
- <table id="legend">
- <tr>
- <th>Color</th>
- <th>Description</th>
- </tr>
- <tr class="result">
- <td class="pass">PASS</td>
- <td>Test passed and was expected to pass</td>
- </tr>
- <tr class="result">
- <td class="xfail">XFAIL</td>
- <td>Test failed and expected to fail</td>
- </tr>
- <tr class="result">
- <td class="fail">FAIL</td>
- <td>Test failed but was expected to pass</td>
- </tr>
- <tr class="result">
- <td class="xpass">XPASS</td>
- <td>Test passed but was expected to fail</td>
- </tr>
- <tr class="result">
- <td class="error">ERROR</td>
- <td>The compiler, linker or the test segfaulted</td>
- </tr>
- <tr class="result">
- <td class="generic">+</td>
- <td>Changes from FAIL, XPASS or ERROR to PASS or XFAIL</td>
- </tr>
- <tr class="result">
- <td class="generic">-</td>
- <td>Changes from PASS or XFAIL to FAIL, XPASS or ERROR</td>
- </tr>
- <tr class="result">
- <td class="generic">chg</td>
- <td>Changed within the good or bad group without crossing over</td>
- </tr>
- </table>
-
- <h2>Results</h2>
- <table>
- <tr class="head">
- <th></th>
- <th colspan="5" class="borderleft">Test results</th>
- <th colspan="3" class="borderleft">Diff to previous</th>
- <th colspan="3" class="borderleft">Diff to ` ~ std.path.getBaseName(reference) ~ `</th>
- </tr>
- <tr>
- <th>Name</th>
- <th class="borderleft">PASS</th>
- <th>XFAIL</th>
- <th>FAIL</th>
- <th>XPASS</th>
- <th>ERROR</th>
- <th class="borderleft">+</th>
- <th>-</th>
- <th>chg</th>
- <th class="borderleft">+</th>
- <th>-</th>
- <th>chg</th>
- </tr>
- `);
- for(int i = files.length - 1; i >= 0; --i) {
- auto file = files[i];
- index.writefln(`<tr class="` ~ (i%2 ? `result` : `odd result`) ~ `">`);
- char[] id = std.path.getBaseName(file);
- char[] statsname = std.path.join(std.path.join(basedir, id), "stats.base");
- index.writef(cast(char[])std.file.read(statsname));
- if(i != 0) {
- char[] newid = std.path.getBaseName(files[i-1]);
- statsname = std.path.join(std.path.join(basedir, newid ~ "-to-" ~ id), "stats.base");
- index.writef(cast(char[])std.file.read(statsname));
- } else {
- index.writefln(`<td class="borderleft"></td><td></td><td></td>`);
- }
- if(i != files.length - 1) {
- char[] refid = std.path.getBaseName(reference);
- statsname = std.path.join(std.path.join(basedir, refid ~ "-to-" ~ id), "stats.base");
- index.writef(cast(char[])std.file.read(statsname));
- } else {
- index.writefln(`<td class="borderleft"></td><td></td><td></td>`);
- }
- index.writefln(`</tr>`);
- }
- index.writefln(`</table></body></html>`);
-
- return 0;
- }
- void generateLogStatistics(char[] file, ref Log[char[]] logs)
- {
- char[] id = std.path.getBaseName(file);
- char[] dirname = std.path.join(basedir, id);
- if(std.file.exists(dirname)) {
- if(std.file.isdir(dirname)) {
- if(!regenerate) {
- writefln("Directory ", dirname, " already exists, skipping...");
- return;
- }
- }
- else
- throw new Exception(dirname ~ " is not a directory!");
- }
- else
- std.file.mkdir(dirname);
- // parse etc.
- Log log = new Log(id, file);
- logs[id] = log;
- // write status
- {
- BufferedFile makeFile(char[] name) {
- return new BufferedFile(std.path.join(dirname, name), FileMode.OutNew);
- }
- BufferedFile[Result] resultsfile = [
- Result.PASS: makeFile("pass.html"),
- Result.FAIL: makeFile("fail.html"),
- Result.XPASS: makeFile("xpass.html"),
- Result.XFAIL: makeFile("xfail.html"),
- Result.ERROR: makeFile("error.html") ];
-
- scope(exit) {
- foreach(file; resultsfile)
- file.close();
- }
-
- foreach(file; resultsfile)
- file.writefln(`<html><body>`);
-
- foreach(tkey; log.tests.keys.sort) {
- auto test = log.tests[tkey];
- auto result = test.r & Result.BASE_MASK;
- resultsfile[result].writefln(test.name, " in ", test.file, "<br>");
- }
-
- foreach(file; resultsfile)
- file.writefln(`</body></html>`);
- }
- BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew);
- scope(exit) stats.close();
- stats.writefln(`<td>`, id, `</td>`);
- stats.writefln(`<td class="pass borderleft"><a href="`, std.path.join(log.id, "pass.html"), `">`, log.counts[Result.PASS], `</a></td>`);
- stats.writefln(`<td class="xfail"><a href="`, std.path.join(log.id, "xfail.html"), `">`, log.counts[Result.XFAIL], `</a></td>`);
- stats.writefln(`<td class="fail"><a href="`, std.path.join(log.id, "fail.html"), `">`, log.counts[Result.FAIL], `</a></td>`);
- stats.writefln(`<td class="xpass"><a href="`, std.path.join(log.id, "xpass.html"), `">`, log.counts[Result.XPASS], `</a></td>`);
- stats.writefln(`<td class="error"><a href="`, std.path.join(log.id, "error.html"), `">`, log.counts[Result.ERROR], `</a></td>`);
- }
- void generateChangeStatistics(char[] file1, char[] file2, ref Log[char[]] logs)
- {
- char[] newid = std.path.getBaseName(file1);
- char[] oldid = std.path.getBaseName(file2);
- char[] dirname = std.path.join(basedir, oldid ~ "-to-" ~ newid);
- if(std.file.exists(dirname)) {
- if(std.file.isdir(dirname)) {
- if(!regenerate) {
- writefln("Directory ", dirname, " already exists, skipping...");
- return;
- }
- }
- else
- throw new Exception(dirname ~ " is not a directory!");
- }
- else
- std.file.mkdir(dirname);
- // parse etc.
- Log newLog, oldLog;
- Log getOrParse(char[] id, char[] file) {
- if(id in logs)
- return logs[id];
- else {
- Log tmp = new Log(id, file);
- logs[id] = tmp;
- return tmp;
- }
- }
- newLog = getOrParse(newid, file1);
- oldLog = getOrParse(oldid, file2);
- int nRegressions, nImprovements, nChanges;
- {
- auto regressionsFile = new BufferedFile(std.path.join(dirname, "regressions.html"), FileMode.OutNew);
- scope(exit) regressionsFile.close();
- regressionsFile.writefln(`<html><body>`);
- auto improvementsFile = new BufferedFile(std.path.join(dirname, "improvements.html"), FileMode.OutNew);
- scope(exit) improvementsFile.close();
- improvementsFile.writefln(`<html><body>`);
- auto changesFile = new BufferedFile(std.path.join(dirname, "changes.html"), FileMode.OutNew);
- scope(exit) changesFile.close();
- changesFile.writefln(`<html><body>`);
- BufferedFile targetFile;
-
- foreach(file; newLog.tests.keys.sort){
- Test* t = file in newLog.tests;
- Test* oldT = file in oldLog.tests;
-
- if(oldT !is null){
- if(oldT.r == t.r)
- continue;
- else if(t.r >= Result.XPASS && oldT.r && oldT.r <= Result.XFAIL){
- targetFile = regressionsFile;
- nRegressions++;
- }
- else if(t.r && t.r <= Result.XFAIL && oldT.r >= Result.XPASS){
- targetFile = improvementsFile;
- nImprovements++;
- }
- else {
- targetFile = changesFile;
- nChanges++;
- }
- targetFile.writefln(toString(oldT.r), " -> ", toString(t.r), " : ", t.name, " in ", t.file, "<br>");
- }
- }
- regressionsFile.writefln(`</body></html>`);
- improvementsFile.writefln(`</body></html>`);
- changesFile.writefln(`</body></html>`);
- }
- BufferedFile stats = new BufferedFile(std.path.join(dirname, "stats.base"), FileMode.OutNew);
- scope(exit) stats.close();
- auto dir = oldid ~ "-to-" ~ newid;
- stats.writefln(`<td class="borderleft"><a href="`, std.path.join(dir, "improvements.html"), `">`, nImprovements, `</a></td>`);
- stats.writefln(`<td><a href="`, std.path.join(dir, "regressions.html"), `">`, nRegressions, `</a></td>`);
- stats.writefln(`<td><a href="`, std.path.join(dir, "changes.html"), `">`, nChanges, `</a></td>`);
- }