PageRenderTime 45ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java

#
Java | 236 lines | 149 code | 46 blank | 41 comment | 1 complexity | e2bb6361691ab34dfeb81551563b6503 MD5 | raw file
Possible License(s): Apache-2.0, BSD-3-Clause, JSON, CPL-1.0
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.hive.ql.exec.persistence;
  19. import java.io.File;
  20. import java.io.FileInputStream;
  21. import java.io.FileOutputStream;
  22. import java.io.IOException;
  23. import java.io.ObjectInputStream;
  24. import java.io.ObjectOutputStream;
  25. import java.io.Serializable;
  26. import java.lang.management.ManagementFactory;
  27. import java.lang.management.MemoryMXBean;
  28. import java.text.NumberFormat;
  29. import java.util.HashMap;
  30. import java.util.Set;
  31. import org.apache.commons.logging.Log;
  32. import org.apache.commons.logging.LogFactory;
  33. import org.apache.hadoop.hive.ql.exec.Utilities;
  34. import org.apache.hadoop.hive.ql.metadata.HiveException;
  35. import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
  36. /**
  37. * Simple wrapper for persistent Hashmap implementing only the put/get/remove/clear interface. The
  38. * main memory hash table acts as a cache and all put/get will operate on it first. If the size of
  39. * the main memory hash table exceeds a certain threshold, new elements will go into the persistent
  40. * hash table.
  41. */
  42. public class HashMapWrapper<K, V> implements Serializable {
  43. private static final long serialVersionUID = 1L;
  44. protected Log LOG = LogFactory.getLog(this.getClass().getName());
  45. // default threshold for using main memory based HashMap
  46. private static final int THRESHOLD = 1000000;
  47. private static final float LOADFACTOR = 0.75f;
  48. private static final float MEMORYUSAGE = 1;
  49. private float maxMemoryUsage;
  50. private HashMap<K, V> mHash; // main memory HashMap
  51. protected transient LogHelper console;
  52. private File dumpFile;
  53. public static MemoryMXBean memoryMXBean;
  54. private long maxMemory;
  55. private long currentMemory;
  56. private NumberFormat num;
  57. /**
  58. * Constructor.
  59. *
  60. * @param threshold
  61. * User specified threshold to store new values into persistent storage.
  62. */
  63. public HashMapWrapper(int threshold, float loadFactor, float memoryUsage) {
  64. maxMemoryUsage = memoryUsage;
  65. mHash = new HashMap<K, V>(threshold, loadFactor);
  66. memoryMXBean = ManagementFactory.getMemoryMXBean();
  67. maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
  68. LOG.info("maximum memory: " + maxMemory);
  69. num = NumberFormat.getInstance();
  70. num.setMinimumFractionDigits(2);
  71. }
  72. public HashMapWrapper(int threshold) {
  73. this(threshold, LOADFACTOR, MEMORYUSAGE);
  74. }
  75. public HashMapWrapper() {
  76. this(THRESHOLD, LOADFACTOR, MEMORYUSAGE);
  77. }
  78. public V get(K key) {
  79. return mHash.get(key);
  80. }
  81. public boolean put(K key, V value) throws HiveException {
  82. // isAbort();
  83. mHash.put(key, value);
  84. return false;
  85. }
  86. public void remove(K key) {
  87. mHash.remove(key);
  88. }
  89. /**
  90. * Flush the main memory hash table into the persistent cache file
  91. *
  92. * @return persistent cache file
  93. */
  94. public long flushMemoryCacheToPersistent(File file) throws IOException {
  95. ObjectOutputStream outputStream = null;
  96. outputStream = new ObjectOutputStream(new FileOutputStream(file));
  97. outputStream.writeObject(mHash);
  98. outputStream.flush();
  99. outputStream.close();
  100. return file.length();
  101. }
  102. public void initilizePersistentHash(String fileName) throws IOException, ClassNotFoundException {
  103. ObjectInputStream inputStream = null;
  104. inputStream = new ObjectInputStream(new FileInputStream(fileName));
  105. HashMap<K, V> hashtable = (HashMap<K, V>) inputStream.readObject();
  106. this.setMHash(hashtable);
  107. inputStream.close();
  108. }
  109. public int size() {
  110. return mHash.size();
  111. }
  112. public Set<K> keySet() {
  113. return mHash.keySet();
  114. }
  115. /**
  116. * Close the persistent hash table and clean it up.
  117. *
  118. * @throws HiveException
  119. */
  120. public void close() throws HiveException {
  121. mHash.clear();
  122. }
  123. public void clear() throws HiveException {
  124. mHash.clear();
  125. }
  126. public int getKeySize() {
  127. return mHash.size();
  128. }
  129. public boolean isAbort(long numRows,LogHelper console) {
  130. System.gc();
  131. System.gc();
  132. int size = mHash.size();
  133. long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
  134. double rate = (double) usedMemory / (double) maxMemory;
  135. console.printInfo(Utilities.now() + "\tProcessing rows:\t" + numRows + "\tHashtable size:\t"
  136. + size + "\tMemory usage:\t" + usedMemory + "\trate:\t" + num.format(rate));
  137. if (rate > (double) maxMemoryUsage) {
  138. return true;
  139. }
  140. return false;
  141. }
  142. public void setLOG(Log log) {
  143. LOG = log;
  144. }
  145. public HashMap<K, V> getMHash() {
  146. return mHash;
  147. }
  148. public void setMHash(HashMap<K, V> hash) {
  149. mHash = hash;
  150. }
  151. public LogHelper getConsole() {
  152. return console;
  153. }
  154. public void setConsole(LogHelper console) {
  155. this.console = console;
  156. }
  157. public File getDumpFile() {
  158. return dumpFile;
  159. }
  160. public void setDumpFile(File dumpFile) {
  161. this.dumpFile = dumpFile;
  162. }
  163. public static MemoryMXBean getMemoryMXBean() {
  164. return memoryMXBean;
  165. }
  166. public static void setMemoryMXBean(MemoryMXBean memoryMXBean) {
  167. HashMapWrapper.memoryMXBean = memoryMXBean;
  168. }
  169. public long getMaxMemory() {
  170. return maxMemory;
  171. }
  172. public void setMaxMemory(long maxMemory) {
  173. this.maxMemory = maxMemory;
  174. }
  175. public long getCurrentMemory() {
  176. return currentMemory;
  177. }
  178. public void setCurrentMemory(long currentMemory) {
  179. this.currentMemory = currentMemory;
  180. }
  181. public NumberFormat getNum() {
  182. return num;
  183. }
  184. public void setNum(NumberFormat num) {
  185. this.num = num;
  186. }
  187. public static int getTHRESHOLD() {
  188. return THRESHOLD;
  189. }
  190. }