/ojc-core/component-common/xmlbeans/xbean-src/2.3.0/xmlbeans/test/perf/src/org/apache/xmlbeans/test/performance/utils/RunComparator.java
Java | 250 lines | 163 code | 42 blank | 45 comment | 33 complexity | 8d87f2ce030d78744dcf90ac5a4819bc MD5 | raw file
- /* Copyright 2004 The Apache Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- package org.apache.xmlbeans.test.performance.utils;
-
- import org.apache.xmlbeans.XmlError;
- import org.apache.xmlbeans.XmlOptions;
- import org.openuri.perf.Custom;
- import org.openuri.perf.Result;
- import org.openuri.perf.ResultSetDocument;
-
- import java.io.File;
- import java.io.FileNotFoundException;
- import java.util.ArrayList;
- import java.util.HashMap;
- import java.util.Iterator;
- import java.util.List;
-
-
- public class RunComparator
- {
- // This class compares the xml results file of 2 perf runs and checks differences in the numbers for each test case
- // An optional 'tolerance' % deviation value can be provided which will result in only the cases that are beyond this
- // value (+/-) getting reported
-
- private ResultSetDocument resultsDocOne; // XmlObject for the first result set
- private ResultSetDocument resultsDocTwo; // and the second
- private double tolerance; // % deviation in test run numbers
-
- // The computation of deviation for a given test scase is done as follows:
- // (run1 - run2)/run1 as a %. Negative value implies regression, +ve implies improvement
-
-
- public RunComparator(String runResultsXmlFilePath1, String runResultsXmlFilePath2, double tolerance) throws Exception
- {
- System.out.println("XMLBeans RunComparator - comparing Perf Runs...");
-
- if ((runResultsXmlFilePath1 != null) && (runResultsXmlFilePath2 != null))
- {
- try
- {
- File resultsFileOne = new File(runResultsXmlFilePath1);
- File resultsFileTwo = new File(runResultsXmlFilePath2);
-
- if (tolerance > 100 || tolerance < 0)
- {
- throw(new Exception("Invalid input value for 'tolerance'"));
- }
- this.tolerance = tolerance;
-
- // generate XmlObjects and validate against schema
- XmlOptions validateOptions = new XmlOptions();
- List errors = new ArrayList();
- validateOptions.setErrorListener(errors);
-
- resultsDocOne = ResultSetDocument.Factory.parse(resultsFileOne);
- if (!resultsDocOne.validate(validateOptions))
- {
- System.out.println("Result Set XML Document " + runResultsXmlFilePath1 + "is not Valid against schema!");
- for (Iterator iterator = errors.iterator(); iterator.hasNext();)
- {
- XmlError eachErr = (XmlError) iterator.next();
- //System.out.println("Validation Error:" + eachErr.getMessage() + eachErr.getLine());
- }
- //throw new Exception("Invalid Results File " + runResultsXmlFilePath1);
- }
-
- errors.clear();
- resultsDocTwo = ResultSetDocument.Factory.parse(resultsFileTwo);
- if (!resultsDocTwo.validate(validateOptions))
- {
- System.out.println("Result Set XML Document " + runResultsXmlFilePath2 + "is not Valid against schema!");
- for (Iterator iterator = errors.iterator(); iterator.hasNext();)
- {
- XmlError eachErr = (XmlError) iterator.next();
- //System.out.println("Validation Error:" + eachErr.getMessage() + eachErr.getLine());
- }
-
- //throw new Exception("Invalid Results File " + runResultsXmlFilePath2);
- }
-
- }
-
- catch (FileNotFoundException fne)
- {
- throw new Exception("Invalid results file(s) specified!");
- }
- catch (Exception e)
- {
- e.printStackTrace();
- }
- }
- else
- {
- throw new Exception("Input File Path(s) in null!");
- }
-
- } // end constructor
-
-
- private void compare()
- {
- // create a hashmap with first run results
- Result[] res1 = resultsDocOne.getResultSet().getResultArray();
-
- HashMap runOneHashMap = new HashMap();
- for (int i = 0; i < res1.length; i++)
- {
- Result eachResult = res1[i];
- //System.out.println("Name:" + eachResult.getName() + "\tnote:" + eachResult.getNote() + "\tTime:" + eachResult.getTime());
-
- Custom[] custArr = eachResult.getCustomArray();
- if ((custArr != null) || (custArr.length > 0))
- {
- for (int j=0; j < custArr.length; j++)
- {
- Custom eachCustomElem = custArr[j];
- //System.out.println("Cust:" + eachCustomElem.getName() + "\tvalue:" + eachCustomElem.getValue());
-
- String hashKey = eachResult.getName() + "_" + eachCustomElem.getName() + "_" + eachCustomElem.getValue();
- //System.out.println("Key:" + hashKey);
- //System.out.println("Val:" + eachResult.getTime());
-
- // insert each result into the hashmap
- runOneHashMap.put(hashKey, eachResult);
- }
- }
- else
- {
- // every result should have a custom element but this is not always true, the generated results are invalid sometimes
- System.out.println("Result Set 1, Custom Element Missing!:" + eachResult.getName());
- }
- }
-
- // now iterate thro second result set and compare
- Result[] res2 = resultsDocTwo.getResultSet().getResultArray();
- for (int i = 0; i < res2.length; i++)
- {
- Result eachResult2 = res2[i];
-
- Custom[] custArr2 = eachResult2.getCustomArray();
- if ((custArr2 != null) || (custArr2.length > 0))
- {
- for (int j = 0; j < custArr2.length; j++)
- {
- Custom custom = custArr2[j];
-
- String hashKeyToLookFor = eachResult2.getName() + "_" + custom.getName() + "_" + custom.getValue();
- Result resFromHash = (Result) runOneHashMap.get(hashKeyToLookFor);
-
- if(resFromHash != null)
- {
- if (resFromHash.getTime() == eachResult2.getTime())
- {
- // no-op
- //System.out.println("MATCH:" + hashKeyToLookFor );
- //System.out.println("First Result Set:" + runOneTime.longValue());
- //System.out.println("Second Result Set:" + result2.getTime());
- //System.out.println("------");
- }
- else
- {
- // compute diff here and compare with tolerence
- double diffPercent = ((double) resFromHash.getTime() - (double) eachResult2.getTime()) / (double) resFromHash.getTime();
- diffPercent *= 100;
-
- //**System.out.println("% Change= " + diff);
- String testDetails = "Test Case Name\t:" + resFromHash.getName() + "\n" +
- "Test Case Spec\t:" + resFromHash.getCustomArray(j).getName() + "=" +
- resFromHash.getCustomArray(j).getValue() + "\n" +
- "Tolerance Spec\t:" + tolerance;
-
- // regression
- if (diffPercent < 0)
- {
- diffPercent *= -1.00;
- if (diffPercent >= 0)
- {
- if (diffPercent > tolerance)
- {
- diffPercent *= -1.00;
- System.out.println("Regression Found! \n" + testDetails + "\n" + "% Deviation\t\t:" + diffPercent);
- System.out.println("------");
- }
- }
-
- }
-
- // improvement
- if (diffPercent >= 0)
- {
- if (diffPercent > tolerance)
- {
- //System.out.println("Improvement Found! \n" + testDetails + "\n" + "% Deviation\t\t:" + diffPercent);
- //System.out.println("------");
- }
- }
- }
- } // end of hashtable look up result null check
- } // end for
- }
- else
- {
- // every result should have a custom element
- System.out.println("Result Set 2, Custom Element Missing!::" + eachResult2.getName());
- }
-
-
- }
-
-
- }
-
-
- public static void main(String[] args) throws Exception
- {
- if((args.length > 3) || args.length < 2)
- {
- System.out.println("Invalid Number of arguments to RunComparator utility!");
- System.out.println("Usage: java RunComparator <path to run output file 1> <path to run output file 2> [<tolerance>]");
- System.exit(0);
- }
-
- // TODO: More checking here and add a driver ant task for this utility
- String runOne = args[0];
- String runTwo = args[1];
- double tolerance = 5.00; // default value
- if(args[2] != null)
- {
- tolerance = Double.parseDouble(args[2]);
- }
-
-
- RunComparator rc = new RunComparator(runOne, runTwo, tolerance);
- rc.compare();
- }
-
- }
-