PageRenderTime 102ms CodeModel.GetById 60ms RepoModel.GetById 8ms app.codeStats 0ms

/projects/hadoop-1.1.2/src/test/org/apache/hadoop/fs/TestLocalDirAllocator.java

https://gitlab.com/essere.lab.public/qualitas.class-corpus
Java | 211 lines | 135 code | 22 blank | 54 comment | 13 complexity | daa52a3267e9ce4dbe5c5d2c3be3fc35 MD5 | raw file
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.fs;
  19. import java.io.File;
  20. import java.io.IOException;
  21. import org.apache.hadoop.conf.Configuration;
  22. import org.apache.hadoop.util.Shell;
  23. import junit.framework.TestCase;
  24. /** This test LocalDirAllocator works correctly;
  25. * Every test case uses different buffer dirs to
  26. * enforce the AllocatorPerContext initialization.
  27. * This test does not run on Cygwin because under Cygwin
  28. * a directory can be created in a read-only directory
  29. * which breaks this test.
  30. */
  31. public class TestLocalDirAllocator extends TestCase {
  32. final static private Configuration conf = new Configuration();
  33. final static private String BUFFER_DIR_ROOT = "build/test/temp";
  34. final static private Path BUFFER_PATH_ROOT = new Path(BUFFER_DIR_ROOT);
  35. final static private File BUFFER_ROOT = new File(BUFFER_DIR_ROOT);
  36. final static private String BUFFER_DIR[] = new String[] {
  37. BUFFER_DIR_ROOT+"/tmp0", BUFFER_DIR_ROOT+"/tmp1", BUFFER_DIR_ROOT+"/tmp2",
  38. BUFFER_DIR_ROOT+"/tmp3", BUFFER_DIR_ROOT+"/tmp4", BUFFER_DIR_ROOT+"/tmp5",
  39. BUFFER_DIR_ROOT+"/tmp6"};
  40. final static private Path BUFFER_PATH[] = new Path[] {
  41. new Path(BUFFER_DIR[0]), new Path(BUFFER_DIR[1]), new Path(BUFFER_DIR[2]),
  42. new Path(BUFFER_DIR[3]), new Path(BUFFER_DIR[4]), new Path(BUFFER_DIR[5]),
  43. new Path(BUFFER_DIR[6])};
  44. final static private String CONTEXT = "dfs.client.buffer.dir";
  45. final static private String FILENAME = "block";
  46. final static private LocalDirAllocator dirAllocator =
  47. new LocalDirAllocator(CONTEXT);
  48. static LocalFileSystem localFs;
  49. final static private boolean isWindows =
  50. System.getProperty("os.name").startsWith("Windows");
  51. final static int SMALL_FILE_SIZE = 100;
  52. static {
  53. try {
  54. localFs = FileSystem.getLocal(conf);
  55. rmBufferDirs();
  56. } catch(IOException e) {
  57. System.out.println(e.getMessage());
  58. e.printStackTrace();
  59. System.exit(-1);
  60. }
  61. }
  62. private static void rmBufferDirs() throws IOException {
  63. assertTrue(!localFs.exists(BUFFER_PATH_ROOT) ||
  64. localFs.delete(BUFFER_PATH_ROOT));
  65. }
  66. private void validateTempDirCreation(int i) throws IOException {
  67. File result = createTempFile(SMALL_FILE_SIZE);
  68. assertTrue("Checking for " + BUFFER_DIR[i] + " in " + result + " - FAILED!",
  69. result.getPath().startsWith(new File(BUFFER_DIR[i], FILENAME).getPath()));
  70. }
  71. private File createTempFile() throws IOException {
  72. File result = dirAllocator.createTmpFileForWrite(FILENAME, -1, conf);
  73. result.delete();
  74. return result;
  75. }
  76. private File createTempFile(long size) throws IOException {
  77. File result = dirAllocator.createTmpFileForWrite(FILENAME, size, conf);
  78. result.delete();
  79. return result;
  80. }
  81. /** Two buffer dirs. The first dir does not exist & is on a read-only disk;
  82. * The second dir exists & is RW
  83. * @throws Exception
  84. */
  85. public void test0() throws Exception {
  86. if (isWindows) return;
  87. try {
  88. conf.set(CONTEXT, BUFFER_DIR[0]+","+BUFFER_DIR[1]);
  89. assertTrue(localFs.mkdirs(BUFFER_PATH[1]));
  90. BUFFER_ROOT.setReadOnly();
  91. validateTempDirCreation(1);
  92. validateTempDirCreation(1);
  93. } finally {
  94. Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
  95. rmBufferDirs();
  96. }
  97. }
  98. /** Two buffer dirs. The first dir exists & is on a read-only disk;
  99. * The second dir exists & is RW
  100. * @throws Exception
  101. */
  102. public void test1() throws Exception {
  103. if (isWindows) return;
  104. try {
  105. conf.set(CONTEXT, BUFFER_DIR[1]+","+BUFFER_DIR[2]);
  106. assertTrue(localFs.mkdirs(BUFFER_PATH[2]));
  107. BUFFER_ROOT.setReadOnly();
  108. validateTempDirCreation(2);
  109. validateTempDirCreation(2);
  110. } finally {
  111. Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
  112. rmBufferDirs();
  113. }
  114. }
  115. /** Two buffer dirs. Both do not exist but on a RW disk.
  116. * Check if tmp dirs are allocated in a round-robin
  117. */
  118. public void test2() throws Exception {
  119. if (isWindows) return;
  120. try {
  121. conf.set(CONTEXT, BUFFER_DIR[2]+","+BUFFER_DIR[3]);
  122. // create the first file, and then figure the round-robin sequence
  123. createTempFile(SMALL_FILE_SIZE);
  124. int firstDirIdx = (dirAllocator.getCurrentDirectoryIndex() == 0) ? 2 : 3;
  125. int secondDirIdx = (firstDirIdx == 2) ? 3 : 2;
  126. // check if tmp dirs are allocated in a round-robin manner
  127. validateTempDirCreation(firstDirIdx);
  128. validateTempDirCreation(secondDirIdx);
  129. validateTempDirCreation(firstDirIdx);
  130. } finally {
  131. rmBufferDirs();
  132. }
  133. }
  134. /** Two buffer dirs. Both exists and on a R/W disk.
  135. * Later disk1 becomes read-only.
  136. * @throws Exception
  137. */
  138. public void test3() throws Exception {
  139. if (isWindows) return;
  140. try {
  141. conf.set(CONTEXT, BUFFER_DIR[3]+","+BUFFER_DIR[4]);
  142. assertTrue(localFs.mkdirs(BUFFER_PATH[3]));
  143. assertTrue(localFs.mkdirs(BUFFER_PATH[4]));
  144. // create the first file with size, and then figure the round-robin sequence
  145. createTempFile(SMALL_FILE_SIZE);
  146. int nextDirIdx = (dirAllocator.getCurrentDirectoryIndex() == 0) ? 3 : 4;
  147. validateTempDirCreation(nextDirIdx);
  148. // change buffer directory 2 to be read only
  149. new File(BUFFER_DIR[4]).setReadOnly();
  150. validateTempDirCreation(3);
  151. validateTempDirCreation(3);
  152. } finally {
  153. rmBufferDirs();
  154. }
  155. }
  156. /**
  157. * Two buffer dirs, on read-write disk.
  158. *
  159. * Try to create a whole bunch of files.
  160. * Verify that they do indeed all get created where they should.
  161. *
  162. * Would ideally check statistical properties of distribution, but
  163. * we don't have the nerve to risk false-positives here.
  164. *
  165. * @throws Exception
  166. */
  167. static final int TRIALS = 100;
  168. public void test4() throws Exception {
  169. if (isWindows) return;
  170. try {
  171. conf.set(CONTEXT, BUFFER_DIR[5]+","+BUFFER_DIR[6]);
  172. assertTrue(localFs.mkdirs(BUFFER_PATH[5]));
  173. assertTrue(localFs.mkdirs(BUFFER_PATH[6]));
  174. int inDir5=0, inDir6=0;
  175. for(int i = 0; i < TRIALS; ++i) {
  176. File result = createTempFile();
  177. if(result.getPath().startsWith(new File(BUFFER_DIR[5], FILENAME).getPath())) {
  178. inDir5++;
  179. } else if(result.getPath().startsWith(new File(BUFFER_DIR[6], FILENAME).getPath())) {
  180. inDir6++;
  181. }
  182. result.delete();
  183. }
  184. assertTrue( inDir5 + inDir6 == TRIALS);
  185. } finally {
  186. rmBufferDirs();
  187. }
  188. }
  189. }