PageRenderTime 71ms CodeModel.GetById 17ms app.highlight 47ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java

#
Java | 585 lines | 347 code | 62 blank | 176 comment | 55 complexity | 195f990796168e0a8827c3529c851f28 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18
 19package org.apache.hadoop.hive.hwi;
 20
 21import java.io.File;
 22import java.io.FileOutputStream;
 23import java.io.IOException;
 24import java.io.PrintStream;
 25import java.util.ArrayList;
 26import java.util.List;
 27
 28import org.apache.commons.logging.Log;
 29import org.apache.commons.logging.LogFactory;
 30import org.apache.hadoop.hive.cli.CliSessionState;
 31import org.apache.hadoop.hive.cli.OptionsProcessor;
 32import org.apache.hadoop.hive.conf.HiveConf;
 33import org.apache.hadoop.hive.ql.Driver;
 34import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
 35import org.apache.hadoop.hive.ql.processors.CommandProcessor;
 36import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
 37import org.apache.hadoop.hive.ql.session.SessionState;
 38
 39/**
 40 * HWISessionItem can be viewed as a wrapper for a Hive shell. With it the user
 41 * has a session on the web server rather then in a console window.
 42 * 
 43 */
 44public class HWISessionItem implements Runnable, Comparable<HWISessionItem> {
 45
 46  protected static final Log l4j = LogFactory.getLog(HWISessionItem.class
 47      .getName());
 48
 49  /** Represents the state a session item can be in. */
 50  public enum WebSessionItemStatus {
 51    NEW, READY, QUERY_SET, QUERY_RUNNING, DESTROY, KILL_QUERY
 52  };
 53
 54  /** The Web Interface sessionName this is used to identify the session. */
 55  private final String sessionName;
 56
 57  /**
 58   * Respresents the current status of the session. Used by components to
 59   * determine state. Operations will throw exceptions if the item is not in the
 60   * correct state.
 61   */
 62  private HWISessionItem.WebSessionItemStatus status;
 63
 64  private CliSessionState ss;
 65
 66  /** Standard out from the session will be written to this local file. */
 67  private String resultFile;
 68
 69  /** Standard error from the session will be written to this local file. */
 70  private String errorFile;
 71
 72  /**
 73   * The results from the Driver. This is used for storing the most result
 74   * results from the driver in memory.
 75   */
 76  private ArrayList<ArrayList<String>> resultBucket;
 77
 78  /** Limits the resultBucket to be no greater then this size. */
 79  private int resultBucketMaxSize;
 80
 81  /** List of queries that this item should/has operated on. */
 82  private List<String> queries;
 83
 84  /** status code results of queries. */
 85  private List<Integer> queryRet;
 86
 87  /** Reference to the configuration. */
 88  private HiveConf conf;
 89
 90  /** User privileges. */
 91  private HWIAuth auth;
 92
 93  public Thread runnable;
 94
 95  /**
 96   * Threading SessionState issues require us to capture a reference to the hive
 97   * history file and store it.
 98   */
 99  private String historyFile;
100
101  /**
102   * Creates an instance of WebSessionItem, sets status to NEW.
103   */
104  public HWISessionItem(HWIAuth auth, String sessionName) {
105    this.auth = auth;
106    this.sessionName = sessionName;
107    l4j.debug("HWISessionItem created");
108    status = WebSessionItemStatus.NEW;
109    queries = new ArrayList<String>();
110    queryRet = new ArrayList<Integer>();
111    resultBucket = new ArrayList<ArrayList<String>>();
112    resultBucketMaxSize = 1000;
113    runnable = new Thread(this);
114    runnable.start();
115
116    l4j.debug("Wait for NEW->READY transition");
117    synchronized (runnable) {
118      if (status != WebSessionItemStatus.READY) {
119        try {
120          runnable.wait();
121        } catch (Exception ex) {
122        }
123      }
124    }
125    l4j.debug("NEW->READY transition complete");
126  }
127
128  /**
129   * This is the initialization process that is carried out for each
130   * SessionItem. The goal is to emulate the startup of CLIDriver.
131   */
132  private void itemInit() {
133    l4j.debug("HWISessionItem itemInit start " + getSessionName());
134    OptionsProcessor oproc = new OptionsProcessor();
135
136    if (System.getProperty("hwi-args") != null) {
137      String[] parts = System.getProperty("hwi-args").split("\\s+");
138      if (!oproc.process_stage1(parts)) {
139      }
140    }
141
142    SessionState.initHiveLog4j();
143    conf = new HiveConf(SessionState.class);
144    ss = new CliSessionState(conf);
145    SessionState.start(ss);
146    queries.add("set hadoop.job.ugi=" + auth.getUser() + ","
147        + auth.getGroups()[0]);
148    queries.add("set user.name=" + auth.getUser());
149    /*
150     * HiveHistoryFileName will not be accessible outside this thread. We must
151     * capture this now.
152     */
153    historyFile = SessionState.get().getHiveHistory().getHistFileName();
154    l4j.debug("HWISessionItem itemInit Complete " + getSessionName());
155    status = WebSessionItemStatus.READY;
156
157    synchronized (runnable) {
158      runnable.notifyAll();
159    }
160  }
161
162  /**
163   * HWISessionItem is a Runnable instance. Calling this method will change the
164   * status to QUERY_SET and notify(). The run method detects this and then
165   * continues processing.
166   */
167  public void clientStart() throws HWIException {
168    if (status == WebSessionItemStatus.QUERY_RUNNING) {
169      throw new HWIException("Query already running");
170    }
171    status = WebSessionItemStatus.QUERY_SET;
172    synchronized (runnable) {
173      runnable.notifyAll();
174    }
175    l4j.debug(getSessionName() + " Query is set to start");
176  }
177
178  public void clientKill() throws HWIException {
179    if (status != WebSessionItemStatus.QUERY_RUNNING) {
180      throw new HWIException("Can not kill that which is not running.");
181    }
182    status = WebSessionItemStatus.KILL_QUERY;
183    l4j.debug(getSessionName() + " Query is set to KILL_QUERY");
184  }
185
186  /** This method clears the private member variables. */
187  public void clientRenew() throws HWIException {
188    throwIfRunning();
189    queries = new ArrayList<String>();
190    queryRet = new ArrayList<Integer>();
191    resultBucket = new ArrayList<ArrayList<String>>();
192    resultFile = null;
193    errorFile = null;
194    // this.conf = null;
195    // this.ss = null;
196    status = WebSessionItemStatus.NEW;
197    l4j.debug(getSessionName() + " Query is renewed to start");
198  }
199
200  /**
201   * This is a callback style function used by the HiveSessionManager. The
202   * HiveSessionManager notices this and attempts to stop the query.
203   */
204  protected void killIt() {
205    l4j.debug(getSessionName() + " Attempting kill.");
206    if (runnable != null) {
207      try {
208        runnable.join(1000);
209        l4j.debug(getSessionName() + " Thread join complete");
210      } catch (InterruptedException e) {
211        l4j.error(getSessionName() + " killing session caused exception ", e);
212      }
213    }
214  }
215
216  /**
217   * Helper function to get configuration variables.
218   * 
219   * @param wanted
220   *          a ConfVar
221   * @return Value of the configuration variable.
222   */
223  public String getHiveConfVar(HiveConf.ConfVars wanted) throws HWIException {
224    String result = null;
225    try {
226      result = ss.getConf().getVar(wanted);
227    } catch (Exception ex) {
228      throw new HWIException(ex);
229    }
230    return result;
231  }
232
233  public String getHiveConfVar(String s) throws HWIException {
234    String result = null;
235    try {
236      result = conf.get(s);
237    } catch (Exception ex) {
238      throw new HWIException(ex);
239    }
240    return result;
241  }
242
243  /*
244   * mapred.job.tracker could be host:port or just local
245   * mapred.job.tracker.http.address could be host:port or just host In some
246   * configurations http.address is set to 0.0.0.0 we are combining the two
247   * variables to provide a url to the job tracker WUI if it exists. If hadoop
248   * chose the first available port for the JobTracker HTTP port will can not
249   * determine it.
250   */
251  public String getJobTrackerURL(String jobid) throws HWIException {
252    String jt = this.getHiveConfVar("mapred.job.tracker");
253    String jth = this.getHiveConfVar("mapred.job.tracker.http.address");
254    String[] jtparts = null;
255    String[] jthttpParts = null;
256    if (jt.equalsIgnoreCase("local")) {
257      jtparts = new String[2];
258      jtparts[0] = "local";
259      jtparts[1] = "";
260    } else {
261      jtparts = jt.split(":");
262    }
263    if (jth.contains(":")) {
264      jthttpParts = jth.split(":");
265    } else {
266      jthttpParts = new String[2];
267      jthttpParts[0] = jth;
268      jthttpParts[1] = "";
269    }
270    return jtparts[0] + ":" + jthttpParts[1] + "/jobdetails.jsp?jobid=" + jobid
271        + "&refresh=30";
272  }
273
274  @Override
275  /*
276   * HWISessionItem uses a wait() notify() system. If the thread detects conf to
277   * be null, control is transfered to initItem(). A status of QUERY_SET causes
278   * control to transfer to the runQuery() method. DESTROY will cause the run
279   * loop to end permanently.
280   */
281  public void run() {
282    synchronized (runnable) {
283      while (status != HWISessionItem.WebSessionItemStatus.DESTROY) {
284        if (status == WebSessionItemStatus.NEW) {
285          itemInit();
286        }
287
288        if (status == WebSessionItemStatus.QUERY_SET) {
289          runQuery();
290        }
291
292        try {
293          runnable.wait();
294        } catch (InterruptedException e) {
295          l4j.error("in wait() state ", e);
296        }
297      } // end while
298    } // end sync
299  } // end run
300
301  /**
302   * runQuery iterates the list of queries executing each query.
303   */
304  public void runQuery() {
305    FileOutputStream fos = null;
306    if (getResultFile() != null) {
307      try {
308        fos = new FileOutputStream(new File(resultFile));
309        ss.out = new PrintStream(fos, true, "UTF-8");
310      } catch (java.io.FileNotFoundException fex) {
311        l4j.error(getSessionName() + " opening resultfile " + resultFile, fex);
312      } catch (java.io.UnsupportedEncodingException uex) {
313        l4j.error(getSessionName() + " opening resultfile " + resultFile, uex);
314      }
315    } else {
316      l4j.debug(getSessionName() + " Output file was not specified");
317    }
318    l4j.debug(getSessionName() + " state is now QUERY_RUNNING.");
319    status = WebSessionItemStatus.QUERY_RUNNING;
320
321    // expect one return per query
322    queryRet = new ArrayList<Integer>(queries.size());
323    for (int i = 0; i < queries.size(); i++) {
324      String cmd = queries.get(i);
325      String cmd_trimmed = cmd.trim();
326      String[] tokens = cmd_trimmed.split("\\s+");
327      String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim();
328
329      CommandProcessor proc = CommandProcessorFactory.get(tokens[0]);
330      if (proc != null) {
331        if (proc instanceof Driver) {
332          Driver qp = (Driver) proc;
333          queryRet.add(new Integer(qp.run(cmd).getResponseCode()));
334          ArrayList<String> res = new ArrayList<String>();
335          try {
336            while (qp.getResults(res)) {
337              ArrayList<String> resCopy = new ArrayList<String>();
338              resCopy.addAll(res);
339              resultBucket.add(resCopy);
340              if (resultBucket.size() > resultBucketMaxSize) {
341                resultBucket.remove(0);
342              }
343              for (String row : res) {
344                if (ss != null) {
345                  if (ss.out != null) {
346                    ss.out.println(row);
347                  }
348                } else {
349                  throw new RuntimeException("ss was null");
350                }
351              }
352              res.clear();
353            }
354          } catch (IOException ex) {
355            l4j.error(getSessionName() + " getting results " + getResultFile()
356                + " caused exception.", ex);
357          }
358          qp.close();
359        } else {
360          queryRet.add(new Integer(proc.run(cmd_1).getResponseCode()));
361        }
362      } else {
363        // processor was null
364        l4j.error(getSessionName()
365            + " query processor was not found for query " + cmd);
366      }
367    } // end for
368
369    // cleanup
370    try {
371      if (fos != null) {
372        fos.close();
373      }
374    } catch (IOException ex) {
375      l4j.error(getSessionName() + " closing result file " + getResultFile()
376          + " caused exception.", ex);
377    }
378    status = WebSessionItemStatus.READY;
379    l4j.debug(getSessionName() + " state is now READY");
380    synchronized (runnable) {
381      runnable.notifyAll();
382    }
383  }
384
385  /**
386   * This is a chained call to SessionState.setIsSilent(). Use this if you do
387   * not want the result file to have information status
388   */
389  public void setSSIsSilent(boolean silent) throws HWIException {
390    if (ss == null) {
391      throw new HWIException("Session State is null");
392    }
393    ss.setIsSilent(silent);
394  }
395
396  /**
397   * This is a chained call to SessionState.getIsSilent().
398   */
399  public boolean getSSIsSilent() throws HWIException {
400    if (ss == null) {
401      throw new HWIException("Session State is null");
402    }
403    return ss.getIsSilent();
404  }
405
406  /** to support sorting/Set. */
407  public int compareTo(HWISessionItem other) {
408    if (other == null) {
409      return -1;
410    }
411    return getSessionName().compareTo(other.getSessionName());
412  }
413
414  /**
415   * 
416   * @return the HiveHistoryViewer for the session
417   * @throws HWIException
418   */
419  public HiveHistoryViewer getHistoryViewer() throws HWIException {
420    if (ss == null) {
421      throw new HWIException("Session state was null");
422    }
423    /*
424     * we can not call this.ss.get().getHiveHistory().getHistFileName() directly
425     * as this call is made from a a Jetty thread and will return null
426     */
427    HiveHistoryViewer hv = new HiveHistoryViewer(historyFile);
428    return hv;
429  }
430
431  /**
432   * Uses the sessionName property to compare to sessions.
433   * 
434   * @return true if sessionNames are equal false otherwise
435   */
436  @Override
437  public boolean equals(Object other) {
438    if (other == null) {
439      return false;
440    }
441    if (!(other instanceof HWISessionItem)) {
442      return false;
443    }
444    HWISessionItem o = (HWISessionItem) other;
445    if (getSessionName().equals(o.getSessionName())) {
446      return true;
447    } else {
448      return false;
449    }
450  }
451
452  public String getResultFile() {
453    return resultFile;
454  }
455
456  public void setResultFile(String resultFile) {
457    this.resultFile = resultFile;
458  }
459
460  /**
461   * The session name is an identifier to recognize the session.
462   * 
463   * @return the session's name
464   */
465  public String getSessionName() {
466    return sessionName;
467  }
468
469  /**
470   * Used to represent to the user and other components what state the
471   * HWISessionItem is in. Certain commands can only be run when the application
472   * is in certain states.
473   * 
474   * @return the current status of the session
475   */
476  public WebSessionItemStatus getStatus() {
477    return status;
478  }
479
480  /**
481   * Currently unused.
482   * 
483   * @return a String with the full path to the error file.
484   */
485  public String getErrorFile() {
486    return errorFile;
487  }
488
489  /**
490   * Currently unused.
491   * 
492   * @param errorFile
493   *          the full path to the file for results.
494   */
495  public void setErrorFile(String errorFile) {
496    this.errorFile = errorFile;
497  }
498
499  /**
500   * @return the auth
501   */
502  public HWIAuth getAuth() {
503    return auth;
504  }
505
506  /**
507   * @param auth
508   *          the auth to set
509   */
510  protected void setAuth(HWIAuth auth) {
511    this.auth = auth;
512  }
513
514  /** Returns an unmodifiable list of queries. */
515  public List<String> getQueries() {
516    return java.util.Collections.unmodifiableList(queries);
517  }
518
519  /**
520   * Adds a new query to the execution list.
521   * 
522   * @param query
523   *          query to be added to the list
524   */
525  public void addQuery(String query) throws HWIException {
526    throwIfRunning();
527    queries.add(query);
528  }
529
530  /**
531   * Removes a query from the execution list.
532   * 
533   * @param item
534   *          the 0 based index of the item to be removed
535   */
536  public void removeQuery(int item) throws HWIException {
537    throwIfRunning();
538    queries.remove(item);
539  }
540
541  public void clearQueries() throws HWIException {
542    throwIfRunning();
543    queries.clear();
544  }
545
546  /** returns the value for resultBucketMaxSize. */
547  public int getResultBucketMaxSize() {
548    return resultBucketMaxSize;
549  }
550
551  /**
552   * sets the value for resultBucketMaxSize.
553   * 
554   * @param size
555   *          the new size
556   */
557  public void setResultBucketMaxSize(int size) {
558    resultBucketMaxSize = size;
559  }
560
561  /** gets the value for resultBucket. */
562  public ArrayList<ArrayList<String>> getResultBucket() {
563    return resultBucket;
564  }
565
566  /**
567   * The HWISessionItem stores the result of each query in an array.
568   * 
569   * @return unmodifiable list of return codes
570   */
571  public List<Integer> getQueryRet() {
572    return java.util.Collections.unmodifiableList(queryRet);
573  }
574
575  /**
576   * If the ItemStatus is QueryRunning most of the configuration is in a read
577   * only state.
578   */
579  private void throwIfRunning() throws HWIException {
580    if (status == WebSessionItemStatus.QUERY_RUNNING) {
581      throw new HWIException("Query already running");
582    }
583  }
584
585}