/java/test/com/google/appengine/tools/mapreduce/impl/IntermediateInputTest.java

http://appengine-mapreduce.googlecode.com/ · Java · 144 lines · 118 code · 21 blank · 5 comment · 3 complexity · 71f8bbe27f275c8e2b23c9817fcf5f80 MD5 · raw file

  1. // Copyright 2012 Google Inc. All Rights Reserved.
  2. package com.google.appengine.tools.mapreduce.impl;
  3. import static java.util.Arrays.asList;
  4. import com.google.appengine.api.files.AppEngineFile;
  5. import com.google.appengine.api.files.FileServicePb.KeyValues;
  6. import com.google.appengine.api.files.RecordReadChannel;
  7. import com.google.appengine.repackaged.com.google.protobuf.ByteString;
  8. import com.google.appengine.tools.development.testing.LocalFileServiceTestConfig;
  9. import com.google.appengine.tools.development.testing.LocalServiceTestHelper;
  10. import com.google.appengine.tools.mapreduce.KeyValue;
  11. import com.google.appengine.tools.mapreduce.Marshaller;
  12. import com.google.appengine.tools.mapreduce.Marshallers;
  13. import com.google.appengine.tools.mapreduce.ReducerInput;
  14. import com.google.appengine.tools.mapreduce.impl.IntermediateInput.Reader;
  15. import com.google.common.collect.ImmutableList;
  16. import com.google.common.collect.Lists;
  17. import com.google.common.primitives.Ints;
  18. import junit.framework.TestCase;
  19. import java.io.EOFException;
  20. import java.io.IOException;
  21. import java.nio.ByteBuffer;
  22. import java.util.Iterator;
  23. import java.util.List;
  24. import java.util.NoSuchElementException;
  25. import java.util.logging.Logger;
  26. /**
  27. * @author ohler@google.com (Christian Ohler)
  28. */
  29. public class IntermediateInputTest extends TestCase {
  30. @SuppressWarnings("unused")
  31. private static final Logger log = Logger.getLogger(IntermediateInputTest.class.getName());
  32. private final LocalServiceTestHelper helper =
  33. new LocalServiceTestHelper(new LocalFileServiceTestConfig());
  34. @Override
  35. protected void setUp() throws Exception {
  36. super.setUp();
  37. helper.setUp();
  38. }
  39. @Override
  40. protected void tearDown() throws Exception {
  41. helper.tearDown();
  42. super.tearDown();
  43. }
  44. private <K, V> void assertKeyValuesMatch(KeyValue<K, List<V>> expected,
  45. KeyValue<K, ReducerInput<V>> actual) {
  46. assertEquals("key", expected.getKey(), actual.getKey());
  47. assertEquals("values for key " + expected.getKey(),
  48. expected.getValue(), Lists.newArrayList(actual.getValue()));
  49. }
  50. private <K, V> void assertYieldsValues(List<KeyValue<K, List<V>>> expected, Reader<K, V> reader) {
  51. Iterator<KeyValue<K, List<V>>> in = expected.iterator();
  52. while (in.hasNext()) {
  53. KeyValue<K, List<V>> expectedNext = in.next();
  54. KeyValue<K, ReducerInput<V>> actualNext;
  55. try {
  56. actualNext = reader.next();
  57. } catch (NoSuchElementException e) {
  58. fail("Wanted " + expectedNext + ", got NoSuchElementException");
  59. throw new AssertionError();
  60. }
  61. assertKeyValuesMatch(expectedNext, actualNext);
  62. }
  63. try {
  64. Object actualNext = reader.next();
  65. fail("Wanted end of input, got " + actualNext);
  66. } catch (NoSuchElementException e) {
  67. // ok
  68. }
  69. }
  70. private static class MockChannel implements RecordReadChannel {
  71. private final List<KeyValues> records;
  72. private int pos = 0;
  73. MockChannel(KeyValues... records) {
  74. this.records = ImmutableList.copyOf(records);
  75. log.info("records=" + this.records);
  76. }
  77. @Override public ByteBuffer readRecord() throws IOException {
  78. if (pos >= records.size()) {
  79. throw new EOFException();
  80. }
  81. log.info("readRecord() at pos " + pos + " returning " + records.get(pos));
  82. return ByteBuffer.wrap(records.get(pos++).toByteArray());
  83. }
  84. @Override public long position() throws IOException {
  85. return pos;
  86. }
  87. @Override public void position(long newPosition) throws IOException {
  88. pos = Ints.checkedCast(newPosition);
  89. }
  90. }
  91. private static final Marshaller<Integer> MARSHALLER = Marshallers.getIntegerMarshaller();
  92. private KeyValues makeProto(int key, boolean partial, int... values) {
  93. KeyValues.Builder out = KeyValues.newBuilder()
  94. .setKey(ByteString.copyFrom(MARSHALLER.toBytes(key)))
  95. .setPartial(partial);
  96. for (int value : values) {
  97. out.addValue(ByteString.copyFrom(MARSHALLER.toBytes(value)));
  98. }
  99. return out.build();
  100. }
  101. public void testFoo() throws Exception {
  102. AppEngineFile dummyFile = new AppEngineFile("/blobstore/dummy-file-name");
  103. final RecordReadChannel mockChannel = new MockChannel(
  104. makeProto(0, false, 1),
  105. makeProto(15, false, 1, 1, 4, 1),
  106. makeProto(1, true, 2),
  107. makeProto(1, false, 0, 4),
  108. makeProto(-27, true, -1, -1),
  109. makeProto(-27, false, -1));
  110. Reader<Integer, Integer> reader = new Reader<Integer, Integer>(dummyFile,
  111. MARSHALLER, MARSHALLER) {
  112. @Override RecordReadChannel openChannel() {
  113. return mockChannel;
  114. }
  115. };
  116. assertYieldsValues(
  117. ImmutableList.of(KeyValue.of(0, asList(1)),
  118. KeyValue.of(15, asList(1, 1, 4, 1)),
  119. KeyValue.of(1, asList(2, 0, 4)),
  120. KeyValue.of(-27, asList(-1, -1, -1))),
  121. reader);
  122. }
  123. }