PageRenderTime 81ms CodeModel.GetById 45ms RepoModel.GetById 0ms app.codeStats 0ms

/tpc/src/serializers/BenchmarkRunner.java

https://github.com/johanprinsloo/jvm-serializers
Java | 993 lines | 819 code | 115 blank | 59 comment | 128 complexity | d29175bd350dfd95a2d5ab2c965ff672 MD5 | raw file
  1. package serializers;
  2. import java.io.ByteArrayOutputStream;
  3. import java.io.File;
  4. import java.io.FileInputStream;
  5. import java.io.IOException;
  6. import java.io.UnsupportedEncodingException;
  7. import java.io.PrintWriter;
  8. import java.io.StringWriter;
  9. import java.net.URLEncoder;
  10. import java.util.ArrayList;
  11. import java.util.Arrays;
  12. import java.util.Collections;
  13. import java.util.Comparator;
  14. import java.util.EnumMap;
  15. import java.util.HashMap;
  16. import java.util.HashSet;
  17. import java.util.LinkedHashMap;
  18. import java.util.Map;
  19. import java.util.Set;
  20. import java.util.regex.Pattern;
  21. import java.util.zip.DeflaterOutputStream;
  22. public class BenchmarkRunner
  23. {
  24. public final static int DEFAULT_ITERATIONS = 2000;
  25. public final static int DEFAULT_TRIALS = 20;
  26. /**
  27. * Number of milliseconds to warm up for each operation type for each serializer. Let's
  28. * start with 3 seconds.
  29. */
  30. final static long DEFAULT_WARMUP_MSECS = 3000;
  31. // These tests aren't included by default. Use the "-hidden" flag to enable them.
  32. private static final HashSet<String> HIDDEN = new HashSet<String>();
  33. static {
  34. // CKS is not included because it's not really publicly released.
  35. HIDDEN.add("cks");
  36. HIDDEN.add("cks-text");
  37. }
  38. private static final String ERROR_DIVIDER = "-------------------------------------------------------------------";
  39. public static void main(String[] args)
  40. {
  41. // --------------------------------------------------
  42. // Parse command-line options.
  43. Boolean filterIsInclude = null;
  44. Set<String> filterStrings = null;
  45. Integer iterations = null;
  46. Integer trials = null;
  47. Long warmupTime = null;
  48. boolean printChart = false;
  49. boolean prewarm = false;
  50. String dataFileName = null;
  51. boolean enableHidden = false;
  52. Set<String> optionsSeen = new HashSet<String>();
  53. for (String arg : args) {
  54. String remainder;
  55. if (arg.startsWith("--")) {
  56. remainder = arg.substring(2);
  57. }
  58. else if (arg.startsWith("-")) {
  59. remainder = arg.substring(1);
  60. }
  61. else if (dataFileName == null) {
  62. dataFileName = arg;
  63. continue;
  64. }
  65. else {
  66. System.err.println("Expecting only one non-option argument (<data-file> = \"" + dataFileName + "\").");
  67. System.err.println("Found a second one: \"" + arg + "\"");
  68. System.err.println("Use \"-help\" for usage information.");
  69. System.exit(1); return;
  70. }
  71. String option, value;
  72. int eqPos = remainder.indexOf('=');
  73. if (eqPos >= 0) {
  74. option = remainder.substring(0, eqPos);
  75. value = remainder.substring(eqPos+1);
  76. } else {
  77. option = remainder;
  78. value = null;
  79. }
  80. if (!optionsSeen.add(option)) {
  81. System.err.println("Repeated option: \"" + arg + "\"");
  82. System.exit(1); return;
  83. }
  84. if (option.equals("include")) {
  85. if (value == null) {
  86. System.err.println("The \"include\" option requires a value.");
  87. System.exit(1); return;
  88. }
  89. if (filterIsInclude == null) {
  90. filterIsInclude = true;
  91. filterStrings = new HashSet<String>(Arrays.asList(value.split(",")));
  92. } else {
  93. System.err.println("Can't use 'include' and 'exclude' options at the same time.");
  94. System.exit(1); return;
  95. }
  96. }
  97. else if (option.equals("exclude")) {
  98. if (value == null) {
  99. System.err.println("The \"exclude\" option requires a value.");
  100. System.exit(1); return;
  101. }
  102. if (filterIsInclude == null) {
  103. filterIsInclude = false;
  104. filterStrings = new HashSet<String>(Arrays.asList(value.split(",")));
  105. } else {
  106. System.err.println("Can't use 'include' and 'exclude' options at the same time.");
  107. System.exit(1); return;
  108. }
  109. }
  110. else if (option.equals("iterations")) {
  111. if (value == null) {
  112. System.err.println("The \"iterations\" option requires a value.");
  113. System.exit(1); return;
  114. }
  115. assert iterations == null;
  116. try {
  117. iterations = Integer.parseInt(value);
  118. } catch (NumberFormatException ex) {
  119. System.err.println("Invalid value for \"iterations\" option: \"" + value + "\"");
  120. System.exit(1); return;
  121. }
  122. if (iterations < 1) {
  123. System.err.println("Invalid value for \"iterations\" option: \"" + value + "\"");
  124. System.exit(1); return;
  125. }
  126. }
  127. else if (option.equals("trials")) {
  128. if (value == null) {
  129. System.err.println("The \"trials\" option requires a value.");
  130. System.exit(1); return;
  131. }
  132. assert trials == null;
  133. try {
  134. trials = Integer.parseInt(value);
  135. } catch (NumberFormatException ex) {
  136. System.err.println("Invalid value for \"trials\" option: \"" + value + "\"");
  137. System.exit(1); return;
  138. }
  139. if (trials < 1) {
  140. System.err.println("Invalid value for \"trials\" option: \"" + value + "\"");
  141. System.exit(1); return;
  142. }
  143. }
  144. else if (option.equals("warmup-time")) {
  145. if (value == null) {
  146. System.err.println("The \"warmup-time\" option requires a value.");
  147. System.exit(1); return;
  148. }
  149. assert warmupTime == null;
  150. try {
  151. warmupTime = Long.parseLong(value);
  152. } catch (NumberFormatException ex) {
  153. System.err.println("Invalid value for \"warmup-time\" option: \"" + value + "\"");
  154. System.exit(1); return;
  155. }
  156. if (warmupTime < 0) {
  157. System.err.println("Invalid value for \"warmup-time\" option: \"" + value + "\"");
  158. System.exit(1); return;
  159. }
  160. }
  161. else if (option.equals("pre-warmup")) {
  162. if (value != null) {
  163. System.err.println("The \"pre-warmup\" option does not take a value: \"" + arg + "\"");
  164. System.exit(1); return;
  165. }
  166. assert !prewarm;
  167. prewarm = true;
  168. }
  169. else if (option.equals("chart")) {
  170. if (value != null) {
  171. System.err.println("The \"chart\" option does not take a value: \"" + arg + "\"");
  172. System.exit(1); return;
  173. }
  174. assert !printChart;
  175. printChart = true;
  176. }
  177. else if (option.equals("hidden")) {
  178. if (value != null) {
  179. System.err.println("The \"hidden\" option does not take a value: \"" + arg + "\"");
  180. System.exit(1); return;
  181. }
  182. assert !enableHidden;
  183. enableHidden = true;
  184. }
  185. else if (option.equals("help")) {
  186. if (value != null) {
  187. System.err.println("The \"help\" option does not take a value: \"" + arg + "\"");
  188. System.exit(1); return;
  189. }
  190. if (args.length != 1) {
  191. System.err.println("The \"help\" option cannot be combined with any other option.");
  192. System.exit(1); return;
  193. }
  194. System.out.println();
  195. System.out.println("Usage: run [options] <data-file>");
  196. System.out.println();
  197. System.out.println("Options:");
  198. System.out.println(" -iterations=n [default=" + DEFAULT_ITERATIONS + "]");
  199. System.out.println(" -trials=n [default=" + DEFAULT_TRIALS + "]");
  200. System.out.println(" -warmup-time=millis [default=" + DEFAULT_WARMUP_MSECS + "]");
  201. System.out.println(" -pre-warmup (warm all serializers before the first measurement)");
  202. System.out.println(" -chart (generate a Google Chart URL for the results)");
  203. System.out.println(" -include=impl1,impl2,impl3,...");
  204. System.out.println(" -exclude=impl1,impl2,impl3,...");
  205. System.out.println(" -hidden (enable \"hidden\" serializers)");
  206. System.out.println(" -help");
  207. System.out.println();
  208. System.out.println("Example: run -chart -include=protobuf,thrift data/media.1.cks");
  209. System.out.println();
  210. System.exit(0); return;
  211. }
  212. else {
  213. System.err.println("Unknown option: \"" + arg + "\"");
  214. System.err.println("Use \"-help\" for usage information.");
  215. System.exit(1); return;
  216. }
  217. }
  218. if (iterations == null) iterations = DEFAULT_ITERATIONS;
  219. if (trials == null) trials = DEFAULT_TRIALS;
  220. if (warmupTime == null) warmupTime = DEFAULT_WARMUP_MSECS;
  221. if (dataFileName == null) {
  222. System.err.println("Missing <data-file> argument.");
  223. System.err.println("Use \"-help\" for usage information.");
  224. System.exit(1); return;
  225. }
  226. // --------------------------------------------------
  227. // Load serializers.
  228. TestGroups groups = new TestGroups();
  229. // Binary Formats; language-specific ones
  230. JavaBuiltIn.register(groups);
  231. JavaManual.register(groups);
  232. Scala.register(groups);
  233. // hessian and kryo are Java object serializations
  234. Hessian.register(groups);
  235. Kryo.register(groups);
  236. // Binary formats, generic: protobuf, thrift, avro, kryo, CKS
  237. Protobuf.register(groups);
  238. ActiveMQProtobuf.register(groups);
  239. Protostuff.register(groups);
  240. Thrift.register(groups);
  241. AvroSpecific.register(groups);
  242. AvroGeneric.register(groups);
  243. CksBinary.register(groups);
  244. // JSON
  245. JsonJacksonManual.register(groups);
  246. JsonJacksonDatabind.register(groups);
  247. JsonTwoLattes.register(groups);
  248. ProtostuffJson.register(groups);
  249. ProtobufJson.register(groups);
  250. GsonManual.register(groups);
  251. Gson.register(groups);
  252. // Then JSON-like
  253. // CKS text is textual JSON-like format
  254. CksText.register(groups);
  255. // then binary variants
  256. // BSON is binary JSON-like format
  257. BsonJackson.register(groups);
  258. BsonJacksonDatabind.register(groups);
  259. MongoDB.register(groups);
  260. // Smile is 1-to-1 binary representation of JSON
  261. JacksonSmileManual.register(groups);
  262. JacksonSmileDatabind.register(groups);
  263. ProtostuffSmile.register(groups);
  264. // XML-based formats.
  265. Stax.register(groups);
  266. XStream.register(groups);
  267. JacksonXmlDatabind.register(groups);
  268. JavolutionXml.register(groups);
  269. // --------------------------------------------------
  270. // Load data value.
  271. Object dataValue;
  272. TestGroup<?> group;
  273. {
  274. File dataFile = new File(dataFileName);
  275. if (!dataFile.exists()) {
  276. System.out.println("Couldn't find data file \"" + dataFile.getPath() + "\"");
  277. System.exit(1); return;
  278. }
  279. String[] parts = dataFile.getName().split("\\.");
  280. if (parts.length < 3) {
  281. System.out.println("Data file \"" + dataFile.getName() + "\" should be of the form \"<type>.<name>.<extension>\"");
  282. System.exit(1); return;
  283. }
  284. String dataType = parts[0];
  285. String extension = parts[parts.length-1];
  286. group = groups.groupMap.get(dataType);
  287. if (group == null) {
  288. System.out.println("Data file \"" + dataFileName + "\" can't be loaded.");
  289. System.out.println("Don't know about data type \"" + dataType + "\"");
  290. System.exit(1); return;
  291. }
  292. TestGroup.Entry<?,Object> loader = group.extensionMap.get(parts[parts.length-1]);
  293. if (loader == null) {
  294. System.out.println("Data file \"" + dataFileName + "\" can't be loaded.");
  295. System.out.println("No deserializer registered for data type \"" + dataType + "\" and file extension \"." + extension + "\"");
  296. System.exit(1); return;
  297. }
  298. Object deserialized;
  299. try {
  300. byte[] fileBytes = readFile(new File(dataFileName)); // Load entire file into a byte array.
  301. deserialized = loader.serializer.deserialize(fileBytes);
  302. }
  303. catch (Exception ex) {
  304. System.err.println("Error loading data from file \"" + dataFileName + "\".");
  305. System.err.println(ex.getMessage());
  306. System.exit(1); return;
  307. }
  308. dataValue = loader.transformer.reverse(deserialized);
  309. }
  310. @SuppressWarnings("unchecked")
  311. TestGroup<Object> group_ = (TestGroup<Object>) group;
  312. // --------------------------------------------------
  313. Set<String> matched = new HashSet<String>();
  314. Iterable<TestGroup.Entry<Object,Object>> available;
  315. if (enableHidden) {
  316. // Use all of them.
  317. available = group_.entries;
  318. } else {
  319. // Remove the hidden ones.
  320. ArrayList<TestGroup.Entry<Object,Object>> unhidden = new ArrayList<TestGroup.Entry<Object,Object>>();
  321. for (TestGroup.Entry<?,Object> entry_ : group.entries) {
  322. @SuppressWarnings("unchecked")
  323. TestGroup.Entry<Object,Object> entry = (TestGroup.Entry<Object,Object>) entry_;
  324. String name = entry.serializer.getName();
  325. if (!HIDDEN.contains(name)) unhidden.add(entry);
  326. }
  327. available = unhidden;
  328. }
  329. Iterable<TestGroup.Entry<Object,Object>> matchingEntries;
  330. if (filterStrings == null) {
  331. matchingEntries = available;
  332. }
  333. else {
  334. ArrayList<TestGroup.Entry<Object,Object>> al = new ArrayList<TestGroup.Entry<Object,Object>>();
  335. matchingEntries = al;
  336. for (TestGroup.Entry<?,Object> entry_ : available) {
  337. @SuppressWarnings("unchecked")
  338. TestGroup.Entry<Object,Object> entry = (TestGroup.Entry<Object,Object>) entry_;
  339. String name = entry.serializer.getName();
  340. // See if any of the filters match.
  341. boolean found = false;
  342. for (String s : filterStrings) {
  343. boolean thisOneMatches = match(s, name);
  344. if (thisOneMatches) {
  345. matched.add(s);
  346. found = true;
  347. }
  348. }
  349. if (found == filterIsInclude) {
  350. al.add(entry);
  351. }
  352. }
  353. Set<String> unmatched = new HashSet<String>(filterStrings);
  354. unmatched.removeAll(matched);
  355. for (String s : unmatched) {
  356. System.err.println("Warning: there is no implementation name matching the pattern \"" + s + "\"");
  357. if (!enableHidden) {
  358. for (String hiddenName : HIDDEN) {
  359. if (match(s, hiddenName)) {
  360. System.err.println("(The \"" + hiddenName + "\", serializer is hidden by default.");
  361. System.err.println(" Use the \"-hidden\" option to enable hidden serializers)");
  362. break;
  363. }
  364. }
  365. }
  366. }
  367. }
  368. EnumMap<measurements, Map<String, Double>> values;
  369. StringWriter errors = new StringWriter();
  370. PrintWriter errorsPW = new PrintWriter(errors);
  371. try {
  372. values = start(errorsPW, iterations, trials, warmupTime, prewarm, matchingEntries, dataValue);
  373. }
  374. catch (Exception ex) {
  375. ex.printStackTrace(System.err);
  376. System.exit(1); return;
  377. }
  378. if (printChart) {
  379. printImages(values);
  380. }
  381. // Print errors after chart. That way you can't miss it.
  382. String errorsString = errors.toString();
  383. if (errorsString.length() > 0) {
  384. System.out.println(ERROR_DIVIDER);
  385. System.out.println("Errors occurred during benchmarking:");
  386. System.out.print(errorsString);
  387. System.exit(1); return;
  388. }
  389. }
  390. private static boolean match(String pattern, String name)
  391. {
  392. StringBuilder regex = new StringBuilder();
  393. while (pattern.length() > 0) {
  394. int starPos = pattern.indexOf('*');
  395. if (starPos < 0) {
  396. regex.append(Pattern.quote(pattern));
  397. break;
  398. }
  399. else {
  400. String beforeStar = pattern.substring(0, starPos);
  401. String afterStar = pattern.substring(starPos + 1);
  402. regex.append(Pattern.quote(beforeStar));
  403. regex.append(".*");
  404. pattern = afterStar;
  405. }
  406. }
  407. return Pattern.matches(regex.toString(), name);
  408. }
  409. // ------------------------------------------------------------------------------------
  410. private static byte[] readFile(File file)
  411. throws IOException
  412. {
  413. FileInputStream fin = new FileInputStream(file);
  414. try {
  415. ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
  416. byte[] data = new byte[1024];
  417. while (true) {
  418. int numBytes = fin.read(data);
  419. if (numBytes < 0) break;
  420. baos.write(data, 0, numBytes);
  421. }
  422. return baos.toByteArray();
  423. }
  424. finally {
  425. fin.close();
  426. }
  427. }
  428. // ------------------------------------------------------------------------------------
  429. private static double iterationTime(long delta, int iterations)
  430. {
  431. return (double) delta / (double) (iterations);
  432. }
  433. private static final TestCase Create = new TestCase()
  434. {
  435. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  436. {
  437. long start = System.nanoTime();
  438. for (int i = 0; i < iterations; i++)
  439. {
  440. transformer.forward(value);
  441. }
  442. return iterationTime(System.nanoTime() - start, iterations);
  443. }
  444. };
  445. private static final TestCase Serialize = new TestCase()
  446. {
  447. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  448. {
  449. long start = System.nanoTime();
  450. for (int i = 0; i < iterations; i++)
  451. {
  452. Object obj = transformer.forward(value);
  453. serializer.serialize(obj);
  454. }
  455. return iterationTime(System.nanoTime() - start, iterations);
  456. }
  457. };
  458. private static final TestCase SerializeSameObject = new TestCase()
  459. {
  460. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  461. {
  462. // let's reuse same instance to reduce overhead
  463. Object obj = transformer.forward(value);
  464. long start = System.nanoTime();
  465. for (int i = 0; i < iterations; i++)
  466. {
  467. serializer.serialize(obj);
  468. //if (i % 1000 == 0)
  469. // doGc();
  470. }
  471. return iterationTime(System.nanoTime() - start, iterations);
  472. }
  473. };
  474. private static final TestCase Deserialize = new TestCase()
  475. {
  476. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  477. {
  478. byte[] array = serializer.serialize(transformer.forward(value));
  479. long start = System.nanoTime();
  480. for (int i = 0; i < iterations; i++)
  481. {
  482. serializer.deserialize(array);
  483. }
  484. return iterationTime(System.nanoTime() - start, iterations);
  485. }
  486. };
  487. private static final TestCase DeserializeAndCheck = new TestCase()
  488. {
  489. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  490. {
  491. byte[] array = serializer.serialize(transformer.forward(value));
  492. long start = System.nanoTime();
  493. for (int i = 0; i < iterations; i++)
  494. {
  495. Object obj = serializer.deserialize(array);
  496. transformer.reverse(obj);
  497. }
  498. return iterationTime(System.nanoTime() - start, iterations);
  499. }
  500. };
  501. private static final TestCase DeserializeAndCheckShallow = new TestCase()
  502. {
  503. public <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception
  504. {
  505. byte[] array = serializer.serialize(transformer.forward(value));
  506. long start = System.nanoTime();
  507. for (int i = 0; i < iterations; i++)
  508. {
  509. Object obj = serializer.deserialize(array);
  510. transformer.shallowReverse(obj);
  511. }
  512. return iterationTime(System.nanoTime() - start, iterations);
  513. }
  514. };
  515. /**
  516. * JVM is not required to honor GC requests, but adding bit of sleep around request is
  517. * most likely to give it a chance to do it.
  518. */
  519. private static void doGc()
  520. {
  521. try {
  522. Thread.sleep(50L);
  523. } catch (InterruptedException ie) {
  524. System.err.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
  525. }
  526. System.gc();
  527. try { // longer sleep afterwards (not needed by GC, but may help with scheduling)
  528. Thread.sleep(200L);
  529. } catch (InterruptedException ie) {
  530. System.err.println("Interrupted while sleeping in serializers.BenchmarkRunner.doGc()");
  531. }
  532. }
  533. // ------------------------------------------------------------------------------------
  534. private static abstract class TestCase
  535. {
  536. public abstract <J> double run(Transformer<J,Object> transformer, Serializer<Object> serializer, J value, int iterations) throws Exception;
  537. }
  538. private static final class TestCaseRunner<J>
  539. {
  540. private final Transformer<J,Object> transformer;
  541. private final Serializer<Object> serializer;
  542. private final J value;
  543. public TestCaseRunner(Transformer<J,Object> transformer, Serializer<Object> serializer, J value)
  544. {
  545. this.transformer = transformer;
  546. this.serializer = serializer;
  547. this.value = value;
  548. }
  549. public double run(TestCase tc, int iterations) throws Exception
  550. {
  551. return tc.run(transformer, serializer, value, iterations);
  552. }
  553. public double runTakeMin(int trials, TestCase tc, int iterations) throws Exception
  554. {
  555. double minTime = Double.MAX_VALUE;
  556. for (int i = 0; i < trials; i++) {
  557. double time = tc.run(transformer, serializer, value, iterations);
  558. minTime = Math.min(minTime, time);
  559. }
  560. return minTime;
  561. }
  562. }
  563. enum measurements
  564. {
  565. timeCreate("create (nanos)"), timeSerializeDifferentObjects("ser (nanos)"), timeSerializeSameObject("ser+same (nanos)"),
  566. timeDeserializeNoFieldAccess("deser (nanos)"), timeDeserializeAndCheck("deser+deep (nanos)"), timeDeserializeAndCheckShallow("deser+shal (nanos)"),
  567. totalTime("total (nanos)"), length("size (bytes)"), lengthDeflate("size+dfl (bytes)"),
  568. ;
  569. public final String displayName;
  570. measurements(String displayName)
  571. {
  572. this.displayName = displayName;
  573. }
  574. }
  575. private static <J> EnumMap<measurements, Map<String, Double>>
  576. start(PrintWriter errors, int iterations, int trials, long warmupTime, boolean prewarm, Iterable<TestGroup.Entry<J,Object>> groups, J value) throws Exception
  577. {
  578. // Check correctness first.
  579. System.out.println("Checking correctness...");
  580. for (TestGroup.Entry<J,Object> entry : groups)
  581. {
  582. checkCorrectness(errors, entry.transformer, entry.serializer, value);
  583. }
  584. System.out.println("[done]");
  585. // Pre-warm.
  586. if (prewarm) {
  587. System.out.print("Pre-warmup...");
  588. for (TestGroup.Entry<J,Object> entry : groups)
  589. {
  590. TestCaseRunner<J> runner = new TestCaseRunner<J>(entry.transformer, entry.serializer, value);
  591. String name = entry.serializer.getName();
  592. System.out.print(" " + name);
  593. warmCreation(runner, warmupTime);
  594. warmSerialization(runner, warmupTime);
  595. warmDeserialization(runner, warmupTime);
  596. }
  597. System.out.println();
  598. System.out.println("[done]");
  599. }
  600. System.out.printf("%-32s %6s %7s %7s %7s %7s %7s %7s %6s %5s\n",
  601. "",
  602. "create",
  603. "ser",
  604. "+same",
  605. "deser",
  606. "+shal",
  607. "+deep",
  608. "total",
  609. "size",
  610. "+dfl");
  611. EnumMap<measurements, Map<String, Double>> values = new EnumMap<measurements, Map<String, Double>>(measurements.class);
  612. for (measurements m : measurements.values())
  613. values.put(m, new HashMap<String, Double>());
  614. // Actual tests.
  615. for (TestGroup.Entry<J,Object> entry : groups)
  616. {
  617. TestCaseRunner<J> runner = new TestCaseRunner<J>(entry.transformer, entry.serializer, value);
  618. String name = entry.serializer.getName();
  619. try {
  620. /*
  621. * Should only warm things for the serializer that we test next: HotSpot JIT will
  622. * otherwise spent most of its time optimizing slower ones... Use
  623. * -XX:CompileThreshold=1 to hint the JIT to start immediately
  624. *
  625. * Actually: 1 is often not a good value -- threshold is the number
  626. * of samples needed to trigger inlining, and there's no point in
  627. * inlining everything. Default value is in thousands, so lowering
  628. * it to, say, 1000 is usually better.
  629. */
  630. warmCreation(runner, warmupTime);
  631. doGc();
  632. double timeCreate = runner.runTakeMin(trials, Create, iterations * 100); // do more iteration for object creation because of its short time
  633. warmSerialization(runner, warmupTime);
  634. doGc();
  635. double timeSerializeDifferentObjects = runner.runTakeMin(trials, Serialize, iterations);
  636. doGc();
  637. double timeSerializeSameObject = runner.runTakeMin(trials, SerializeSameObject, iterations);
  638. warmDeserialization(runner, warmupTime);
  639. doGc();
  640. double timeDeserializeNoFieldAccess = runner.runTakeMin(trials, Deserialize, iterations);
  641. doGc();
  642. double timeDeserializeAndCheckShallow = runner.runTakeMin(trials, DeserializeAndCheckShallow, iterations);
  643. doGc();
  644. double timeDeserializeAndCheck = runner.runTakeMin(trials, DeserializeAndCheck, iterations);
  645. double totalTime = timeSerializeDifferentObjects + timeDeserializeAndCheck;
  646. byte[] array = entry.serializer.serialize(entry.transformer.forward(value));
  647. byte[] compressDeflate = compressDeflate(array);
  648. System.out.printf("%-32s %6.0f %7.0f %7.0f %7.0f %7.0f %7.0f %7.0f %6d %5d\n",
  649. name,
  650. timeCreate,
  651. timeSerializeDifferentObjects,
  652. timeSerializeSameObject,
  653. timeDeserializeNoFieldAccess,
  654. timeDeserializeAndCheckShallow,
  655. timeDeserializeAndCheck,
  656. totalTime,
  657. array.length,
  658. compressDeflate.length);
  659. addValue(values, name, timeCreate, timeSerializeDifferentObjects, timeSerializeSameObject,
  660. timeDeserializeNoFieldAccess, timeDeserializeAndCheckShallow, timeDeserializeAndCheck, totalTime,
  661. array.length, compressDeflate.length);
  662. }
  663. catch (Exception ex) {
  664. System.out.println("ERROR: \"" + name + "\" crashed during benchmarking.");
  665. errors.println(ERROR_DIVIDER);
  666. errors.println("\"" + name + "\" crashed during benchmarking.");
  667. ex.printStackTrace(errors);
  668. }
  669. }
  670. return values;
  671. }
  672. private static byte[] compressDeflate(byte[] data)
  673. {
  674. try {
  675. ByteArrayOutputStream bout = new ByteArrayOutputStream(500);
  676. DeflaterOutputStream compresser = new DeflaterOutputStream(bout);
  677. compresser.write(data, 0, data.length);
  678. compresser.finish();
  679. compresser.flush();
  680. return bout.toByteArray();
  681. }
  682. catch (IOException ex) {
  683. AssertionError ae = new AssertionError("IOException while writing to ByteArrayOutputStream!");
  684. ae.initCause(ex);
  685. throw ae;
  686. }
  687. }
  688. /**
  689. * Method that tries to validate correctness of serializer, using
  690. * round-trip (construct, serializer, deserialize; compare objects
  691. * after steps 1 and 3).
  692. */
  693. private static <J> void checkCorrectness(PrintWriter errors, Transformer<J,Object> transformer, Serializer<Object> serializer, J value)
  694. throws Exception
  695. {
  696. Object specialInput;
  697. String name = serializer.getName();
  698. try {
  699. specialInput = transformer.forward(value);
  700. }
  701. catch (Exception ex) {
  702. System.out.println("ERROR: \"" + name + "\" crashed during forward transformation.");
  703. errors.println(ERROR_DIVIDER);
  704. errors.println("\"" + name + "\" crashed during forward transformation.");
  705. ex.printStackTrace(errors);
  706. return;
  707. }
  708. byte[] array;
  709. try {
  710. array = serializer.serialize(specialInput);
  711. }
  712. catch (Exception ex) {
  713. System.out.println("ERROR: \"" + name + "\" crashed during serialization.");
  714. errors.println(ERROR_DIVIDER);
  715. errors.println("\"" + name + "\" crashed during serialization.");
  716. ex.printStackTrace(errors);
  717. return;
  718. }
  719. Object specialOutput;
  720. try {
  721. specialOutput = serializer.deserialize(array);
  722. }
  723. catch (Exception ex) {
  724. System.out.println("ERROR: \"" + name + "\" crashed during deserialization.");
  725. errors.println(ERROR_DIVIDER);
  726. errors.println("\"" + name + "\" crashed during deserialization.");
  727. ex.printStackTrace(errors);
  728. return;
  729. }
  730. J output;
  731. try {
  732. output = transformer.reverse(specialOutput);
  733. }
  734. catch (Exception ex) {
  735. System.out.println("ERROR: \"" + name + "\" crashed during reverse transformation.");
  736. errors.println(ERROR_DIVIDER);
  737. errors.println("\"" + name + "\" crashed during reverse transformation.");
  738. ex.printStackTrace(errors);
  739. return;
  740. }
  741. if (!value.equals(output)) {
  742. System.out.println("ERROR: \"" + name + "\" failed round-trip check.");
  743. errors.println(ERROR_DIVIDER);
  744. errors.println("\"" + name + "\" failed round-trip check.");
  745. errors.println("ORIGINAL: " + value);
  746. errors.println("ROUNDTRIP: " + output);
  747. }
  748. }
  749. private static void printImages(EnumMap<measurements, Map<String, Double>> values)
  750. {
  751. for (measurements m : values.keySet()) {
  752. Map<String, Double> map = values.get(m);
  753. ArrayList<Map.Entry<String,Double>> list = new ArrayList<Map.Entry<String,Double>>(map.entrySet());
  754. Collections.sort(list, new Comparator<Map.Entry<String,Double>>() {
  755. public int compare (Map.Entry<String,Double> o1, Map.Entry<String,Double> o2) {
  756. double diff = o1.getValue() - o2.getValue();
  757. return diff > 0 ? 1 : (diff < 0 ? -1 : 0);
  758. }
  759. });
  760. LinkedHashMap<String, Double> sortedMap = new LinkedHashMap<String, Double>();
  761. for (Map.Entry<String, Double> entry : list) {
  762. if( !entry.getValue().isNaN() ) {
  763. sortedMap.put(entry.getKey(), entry.getValue());
  764. }
  765. }
  766. if (!sortedMap.isEmpty()) printImage(sortedMap, m);
  767. }
  768. }
  769. private static String urlEncode(String s)
  770. {
  771. try {
  772. return URLEncoder.encode(s, "UTF-8");
  773. }
  774. catch (UnsupportedEncodingException e) {
  775. throw new RuntimeException(e);
  776. }
  777. }
  778. private static void printImage(Map<String, Double> map, measurements m)
  779. {
  780. StringBuilder valSb = new StringBuilder();
  781. String names = "";
  782. double max = Double.MIN_NORMAL;
  783. for (Map.Entry<String, Double> entry : map.entrySet())
  784. {
  785. double value = entry.getValue();
  786. valSb.append((int) value).append(',');
  787. max = Math.max(max, entry.getValue());
  788. names = urlEncode(entry.getKey()) + '|' + names;
  789. }
  790. int headerSize = 30;
  791. int maxPixels = 300 * 1000; // Limit set by Google's Chart API.
  792. int width = 700;
  793. int maxHeight = maxPixels / width;
  794. int barThickness = 10;
  795. int barSpacing = 10;
  796. int height;
  797. // Reduce bar thickness and spacing until we can fit in the maximum height.
  798. while (true) {
  799. height = headerSize + map.size()*(barThickness + barSpacing);
  800. if (height <= maxHeight) break;
  801. barSpacing--;
  802. if (barSpacing == 1) break;
  803. height = headerSize + map.size()*(barThickness + barSpacing);
  804. if (height <= maxHeight) break;
  805. barThickness--;
  806. if (barThickness == 1) break;
  807. }
  808. boolean truncated = false;
  809. if (height > maxHeight) {
  810. truncated = true;
  811. height = maxHeight;
  812. }
  813. double scale = max * 1.1;
  814. System.out.println("<img src='https://chart.googleapis.com/chart?chtt="
  815. + urlEncode(m.displayName)
  816. + "&chf=c||lg||0||FFFFFF||1||76A4FB||0|bg||s||EFEFEF&chs="+width+"x"+height+"&chd=t:"
  817. + valSb.toString().substring(0, valSb.length() - 1)
  818. + "&chds=0,"+ scale
  819. + "&chxt=y"
  820. + "&chxl=0:|" + names.substring(0, names.length() - 1)
  821. + "&chm=N *f*,000000,0,-1,10&lklk&chdlp=t&chco=660000|660033|660066|660099|6600CC|6600FF|663300|663333|663366|663399|6633CC|6633FF|666600|666633|666666&cht=bhg&chbh=" + barThickness + ",0," + barSpacing + "&nonsense=aaa.png'/>");
  822. if (truncated) {
  823. System.err.println("WARNING: Not enough room to fit all bars in chart.");
  824. }
  825. }
  826. private static void addValue(
  827. EnumMap<measurements, Map<String, Double>> values,
  828. String name,
  829. double timeCreate,
  830. double timeSerializeDifferentObjects,
  831. double timeSerializeSameObject,
  832. double timeDeserializeNoFieldAccess,
  833. double timeDeserializeAndCheckShallow,
  834. double timeDeserializeAndCheck,
  835. double totalTime,
  836. double length, double lengthDeflate)
  837. {
  838. values.get(measurements.timeSerializeDifferentObjects).put(name, timeSerializeDifferentObjects);
  839. values.get(measurements.timeSerializeSameObject).put(name, timeSerializeSameObject);
  840. values.get(measurements.timeDeserializeNoFieldAccess).put(name, timeDeserializeNoFieldAccess);
  841. values.get(measurements.timeDeserializeAndCheckShallow).put(name, timeDeserializeAndCheckShallow);
  842. values.get(measurements.timeDeserializeAndCheck).put(name, timeDeserializeAndCheck);
  843. values.get(measurements.totalTime).put(name, totalTime);
  844. values.get(measurements.length).put(name, length);
  845. values.get(measurements.lengthDeflate).put(name, lengthDeflate);
  846. values.get(measurements.timeCreate).put(name, timeCreate);
  847. }
  848. private static <J> void warmCreation(TestCaseRunner<J> runner, long warmupTime) throws Exception
  849. {
  850. // Instead of fixed counts, let's try to prime by running for N seconds
  851. long endTime = System.currentTimeMillis() + warmupTime;
  852. do
  853. {
  854. runner.run(Create, 10);
  855. }
  856. while (System.currentTimeMillis() < endTime);
  857. }
  858. private static <J> void warmSerialization(TestCaseRunner<J> runner, long warmupTime) throws Exception
  859. {
  860. // Instead of fixed counts, let's try to prime by running for N seconds
  861. long endTime = System.currentTimeMillis() + warmupTime;
  862. do
  863. {
  864. runner.run(Serialize, 10);
  865. }
  866. while (System.currentTimeMillis() < endTime);
  867. }
  868. private static <J> void warmDeserialization(TestCaseRunner<J> runner, long warmupTime) throws Exception
  869. {
  870. // Instead of fixed counts, let's try to prime by running for N seconds
  871. long endTime = System.currentTimeMillis() + warmupTime;
  872. do
  873. {
  874. runner.run(DeserializeAndCheck, 10);
  875. }
  876. while (System.currentTimeMillis() < endTime);
  877. }
  878. }