PageRenderTime 49ms CodeModel.GetById 11ms app.highlight 33ms RepoModel.GetById 2ms app.codeStats 0ms

/tags/release-0.1-rc2/hive/external/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HashMapWrapper.java

#
Java | 236 lines | 149 code | 46 blank | 41 comment | 1 complexity | e2bb6361691ab34dfeb81551563b6503 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.ql.exec.persistence;
 20
 21import java.io.File;
 22import java.io.FileInputStream;
 23import java.io.FileOutputStream;
 24import java.io.IOException;
 25import java.io.ObjectInputStream;
 26import java.io.ObjectOutputStream;
 27import java.io.Serializable;
 28import java.lang.management.ManagementFactory;
 29import java.lang.management.MemoryMXBean;
 30import java.text.NumberFormat;
 31import java.util.HashMap;
 32import java.util.Set;
 33
 34import org.apache.commons.logging.Log;
 35import org.apache.commons.logging.LogFactory;
 36import org.apache.hadoop.hive.ql.exec.Utilities;
 37import org.apache.hadoop.hive.ql.metadata.HiveException;
 38import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 39
 40
 41/**
 42 * Simple wrapper for persistent Hashmap implementing only the put/get/remove/clear interface. The
 43 * main memory hash table acts as a cache and all put/get will operate on it first. If the size of
 44 * the main memory hash table exceeds a certain threshold, new elements will go into the persistent
 45 * hash table.
 46 */
 47
 48public class HashMapWrapper<K, V> implements Serializable {
 49
 50  private static final long serialVersionUID = 1L;
 51  protected Log LOG = LogFactory.getLog(this.getClass().getName());
 52
 53  // default threshold for using main memory based HashMap
 54
 55  private static final int THRESHOLD = 1000000;
 56  private static final float LOADFACTOR = 0.75f;
 57  private static final float MEMORYUSAGE = 1;
 58
 59  private float maxMemoryUsage;
 60  private HashMap<K, V> mHash; // main memory HashMap
 61  protected transient LogHelper console;
 62
 63  private File dumpFile;
 64  public static MemoryMXBean memoryMXBean;
 65  private long maxMemory;
 66  private long currentMemory;
 67  private NumberFormat num;
 68
 69  /**
 70   * Constructor.
 71   *
 72   * @param threshold
 73   *          User specified threshold to store new values into persistent storage.
 74   */
 75  public HashMapWrapper(int threshold, float loadFactor, float memoryUsage) {
 76    maxMemoryUsage = memoryUsage;
 77    mHash = new HashMap<K, V>(threshold, loadFactor);
 78    memoryMXBean = ManagementFactory.getMemoryMXBean();
 79    maxMemory = memoryMXBean.getHeapMemoryUsage().getMax();
 80    LOG.info("maximum memory: " + maxMemory);
 81    num = NumberFormat.getInstance();
 82    num.setMinimumFractionDigits(2);
 83  }
 84
 85  public HashMapWrapper(int threshold) {
 86    this(threshold, LOADFACTOR, MEMORYUSAGE);
 87  }
 88
 89  public HashMapWrapper() {
 90    this(THRESHOLD, LOADFACTOR, MEMORYUSAGE);
 91  }
 92
 93  public V get(K key) {
 94    return mHash.get(key);
 95  }
 96
 97  public boolean put(K key, V value) throws HiveException {
 98    // isAbort();
 99    mHash.put(key, value);
100    return false;
101  }
102
103
104  public void remove(K key) {
105    mHash.remove(key);
106  }
107
108  /**
109   * Flush the main memory hash table into the persistent cache file
110   *
111   * @return persistent cache file
112   */
113  public long flushMemoryCacheToPersistent(File file) throws IOException {
114    ObjectOutputStream outputStream = null;
115    outputStream = new ObjectOutputStream(new FileOutputStream(file));
116    outputStream.writeObject(mHash);
117    outputStream.flush();
118    outputStream.close();
119
120    return file.length();
121  }
122
123  public void initilizePersistentHash(String fileName) throws IOException, ClassNotFoundException {
124    ObjectInputStream inputStream = null;
125    inputStream = new ObjectInputStream(new FileInputStream(fileName));
126    HashMap<K, V> hashtable = (HashMap<K, V>) inputStream.readObject();
127    this.setMHash(hashtable);
128
129    inputStream.close();
130  }
131
132  public int size() {
133    return mHash.size();
134  }
135
136  public Set<K> keySet() {
137    return mHash.keySet();
138  }
139
140
141  /**
142   * Close the persistent hash table and clean it up.
143   *
144   * @throws HiveException
145   */
146  public void close() throws HiveException {
147    mHash.clear();
148  }
149
150  public void clear() throws HiveException {
151    mHash.clear();
152  }
153
154  public int getKeySize() {
155    return mHash.size();
156  }
157
158  public boolean isAbort(long numRows,LogHelper console) {
159    System.gc();
160    System.gc();
161    int size = mHash.size();
162    long usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed();
163    double rate = (double) usedMemory / (double) maxMemory;
164    console.printInfo(Utilities.now() + "\tProcessing rows:\t" + numRows + "\tHashtable size:\t"
165        + size + "\tMemory usage:\t" + usedMemory + "\trate:\t" + num.format(rate));
166    if (rate > (double) maxMemoryUsage) {
167      return true;
168    }
169    return false;
170  }
171
172  public void setLOG(Log log) {
173    LOG = log;
174  }
175
176  public HashMap<K, V> getMHash() {
177    return mHash;
178  }
179
180  public void setMHash(HashMap<K, V> hash) {
181    mHash = hash;
182  }
183
184  public LogHelper getConsole() {
185    return console;
186  }
187
188  public void setConsole(LogHelper console) {
189    this.console = console;
190  }
191
192  public File getDumpFile() {
193    return dumpFile;
194  }
195
196  public void setDumpFile(File dumpFile) {
197    this.dumpFile = dumpFile;
198  }
199
200  public static MemoryMXBean getMemoryMXBean() {
201    return memoryMXBean;
202  }
203
204  public static void setMemoryMXBean(MemoryMXBean memoryMXBean) {
205    HashMapWrapper.memoryMXBean = memoryMXBean;
206  }
207
208  public long getMaxMemory() {
209    return maxMemory;
210  }
211
212  public void setMaxMemory(long maxMemory) {
213    this.maxMemory = maxMemory;
214  }
215
216  public long getCurrentMemory() {
217    return currentMemory;
218  }
219
220  public void setCurrentMemory(long currentMemory) {
221    this.currentMemory = currentMemory;
222  }
223
224  public NumberFormat getNum() {
225    return num;
226  }
227
228  public void setNum(NumberFormat num) {
229    this.num = num;
230  }
231
232  public static int getTHRESHOLD() {
233    return THRESHOLD;
234  }
235
236}