/tags/release-0.0.0-rc0/hive/external/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hive.hwi;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.cli.OptionsProcessor;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
import org.apache.hadoop.hive.ql.session.SessionState;

/**
 * HWISessionItem can be viewed as a wrapper for a Hive shell. With it the user
 * has a session on the web server rather then in a console window.
 * 
 */
public class HWISessionItem implements Runnable, Comparable<HWISessionItem> {

  protected static final Log l4j = LogFactory.getLog(HWISessionItem.class
      .getName());

  /** Represents the state a session item can be in. */
  public enum WebSessionItemStatus {
    NEW, READY, QUERY_SET, QUERY_RUNNING, DESTROY, KILL_QUERY
  };

  /** The Web Interface sessionName this is used to identify the session. */
  private final String sessionName;

  /**
   * Respresents the current status of the session. Used by components to
   * determine state. Operations will throw exceptions if the item is not in the
   * correct state.
   */
  private HWISessionItem.WebSessionItemStatus status;

  private CliSessionState ss;

  /** Standard out from the session will be written to this local file. */
  private String resultFile;

  /** Standard error from the session will be written to this local file. */
  private String errorFile;

  /**
   * The results from the Driver. This is used for storing the most result
   * results from the driver in memory.
   */
  private ArrayList<ArrayList<String>> resultBucket;

  /** Limits the resultBucket to be no greater then this size. */
  private int resultBucketMaxSize;

  /** List of queries that this item should/has operated on. */
  private List<String> queries;

  /** status code results of queries. */
  private List<Integer> queryRet;

  /** Reference to the configuration. */
  private HiveConf conf;

  /** User privileges. */
  private HWIAuth auth;

  public Thread runnable;

  /**
   * Threading SessionState issues require us to capture a reference to the hive
   * history file and store it.
   */
  private String historyFile;

  /**
   * Creates an instance of WebSessionItem, sets status to NEW.
   */
  public HWISessionItem(HWIAuth auth, String sessionName) {
    this.auth = auth;
    this.sessionName = sessionName;
    l4j.debug("HWISessionItem created");
    status = WebSessionItemStatus.NEW;
    queries = new ArrayList<String>();
    queryRet = new ArrayList<Integer>();
    resultBucket = new ArrayList<ArrayList<String>>();
    resultBucketMaxSize = 1000;
    runnable = new Thread(this);
    runnable.start();

    l4j.debug("Wait for NEW->READY transition");
    synchronized (runnable) {
      if (status != WebSessionItemStatus.READY) {
        try {
          runnable.wait();
        } catch (Exception ex) {
        }
      }
    }
    l4j.debug("NEW->READY transition complete");
  }

  /**
   * This is the initialization process that is carried out for each
   * SessionItem. The goal is to emulate the startup of CLIDriver.
   */
  private void itemInit() {
    l4j.debug("HWISessionItem itemInit start " + getSessionName());
    OptionsProcessor oproc = new OptionsProcessor();

    if (System.getProperty("hwi-args") != null) {
      String[] parts = System.getProperty("hwi-args").split("\\s+");
      if (!oproc.process_stage1(parts)) {
      }
    }

    SessionState.initHiveLog4j();
    conf = new HiveConf(SessionState.class);
    ss = new CliSessionState(conf);
    SessionState.start(ss);
    queries.add("set hadoop.job.ugi=" + auth.getUser() + ","
        + auth.getGroups()[0]);
    queries.add("set user.name=" + auth.getUser());
    /*
     * HiveHistoryFileName will not be accessible outside this thread. We must
     * capture this now.
     */
    historyFile = SessionState.get().getHiveHistory().getHistFileName();
    l4j.debug("HWISessionItem itemInit Complete " + getSessionName());
    status = WebSessionItemStatus.READY;

    synchronized (runnable) {
      runnable.notifyAll();
    }
  }

  /**
   * HWISessionItem is a Runnable instance. Calling this method will change the
   * status to QUERY_SET and notify(). The run method detects this and then
   * continues processing.
   */
  public void clientStart() throws HWIException {
    if (status == WebSessionItemStatus.QUERY_RUNNING) {
      throw new HWIException("Query already running");
    }
    status = WebSessionItemStatus.QUERY_SET;
    synchronized (runnable) {
      runnable.notifyAll();
    }
    l4j.debug(getSessionName() + " Query is set to start");
  }

  public void clientKill() throws HWIException {
    if (status != WebSessionItemStatus.QUERY_RUNNING) {
      throw new HWIException("Can not kill that which is not running.");
    }
    status = WebSessionItemStatus.KILL_QUERY;
    l4j.debug(getSessionName() + " Query is set to KILL_QUERY");
  }

  /** This method clears the private member variables. */
  public void clientRenew() throws HWIException {
    throwIfRunning();
    queries = new ArrayList<String>();
    queryRet = new ArrayList<Integer>();
    resultBucket = new ArrayList<ArrayList<String>>();
    resultFile = null;
    errorFile = null;
    // this.conf = null;
    // this.ss = null;
    status = WebSessionItemStatus.NEW;
    l4j.debug(getSessionName() + " Query is renewed to start");
  }

  /**
   * This is a callback style function used by the HiveSessionManager. The
   * HiveSessionManager notices this and attempts to stop the query.
   */
  protected void killIt() {
    l4j.debug(getSessionName() + " Attempting kill.");
    if (runnable != null) {
      try {
        runnable.join(1000);
        l4j.debug(getSessionName() + " Thread join complete");
      } catch (InterruptedException e) {
        l4j.error(getSessionName() + " killing session caused exception ", e);
      }
    }
  }

  /**
   * Helper function to get configuration variables.
   * 
   * @param wanted
   *          a ConfVar
   * @return Value of the configuration variable.
   */
  public String getHiveConfVar(HiveConf.ConfVars wanted) throws HWIException {
    String result = null;
    try {
      result = ss.getConf().getVar(wanted);
    } catch (Exception ex) {
      throw new HWIException(ex);
    }
    return result;
  }

  public String getHiveConfVar(String s) throws HWIException {
    String result = null;
    try {
      result = conf.get(s);
    } catch (Exception ex) {
      throw new HWIException(ex);
    }
    return result;
  }

  /*
   * mapred.job.tracker could be host:port or just local
   * mapred.job.tracker.http.address could be host:port or just host In some
   * configurations http.address is set to 0.0.0.0 we are combining the two
   * variables to provide a url to the job tracker WUI if it exists. If hadoop
   * chose the first available port for the JobTracker HTTP port will can not
   * determine it.
   */
  public String getJobTrackerURL(String jobid) throws HWIException {
    String jt = this.getHiveConfVar("mapred.job.tracker");
    String jth = this.getHiveConfVar("mapred.job.tracker.http.address");
    String[] jtparts = null;
    String[] jthttpParts = null;
    if (jt.equalsIgnoreCase("local")) {
      jtparts = new String[2];
      jtparts[0] = "local";
      jtparts[1] = "";
    } else {
      jtparts = jt.split(":");
    }
    if (jth.contains(":")) {
      jthttpParts = jth.split(":");
    } else {
      jthttpParts = new String[2];
      jthttpParts[0] = jth;
      jthttpParts[1] = "";
    }
    return jtparts[0] + ":" + jthttpParts[1] + "/jobdetails.jsp?jobid=" + jobid
        + "&refresh=30";
  }

  @Override
  /*
   * HWISessionItem uses a wait() notify() system. If the thread detects conf to
   * be null, control is transfered to initItem(). A status of QUERY_SET causes
   * control to transfer to the runQuery() method. DESTROY will cause the run
   * loop to end permanently.
   */
  public void run() {
    synchronized (runnable) {
      while (status != HWISessionItem.WebSessionItemStatus.DESTROY) {
        if (status == WebSessionItemStatus.NEW) {
          itemInit();
        }

        if (status == WebSessionItemStatus.QUERY_SET) {
          runQuery();
        }

        try {
          runnable.wait();
        } catch (InterruptedException e) {
          l4j.error("in wait() state ", e);
        }
      } // end while
    } // end sync
  } // end run

  /**
   * runQuery iterates the list of queries executing each query.
   */
  public void runQuery() {
    FileOutputStream fos = null;
    if (getResultFile() != null) {
      try {
        fos = new FileOutputStream(new File(resultFile));
        ss.out = new PrintStream(fos, true, "UTF-8");
      } catch (java.io.FileNotFoundException fex) {
        l4j.error(getSessionName() + " opening resultfile " + resultFile, fex);
      } catch (java.io.UnsupportedEncodingException uex) {
        l4j.error(getSessionName() + " opening resultfile " + resultFile, uex);
      }
    } else {
      l4j.debug(getSessionName() + " Output file was not specified");
    }
    l4j.debug(getSessionName() + " state is now QUERY_RUNNING.");
    status = WebSessionItemStatus.QUERY_RUNNING;

    // expect one return per query
    queryRet = new ArrayList<Integer>(queries.size());
    for (int i = 0; i < queries.size(); i++) {
      String cmd = queries.get(i);
      String cmd_trimmed = cmd.trim();
      String[] tokens = cmd_trimmed.split("\\s+");
      String cmd_1 = cmd_trimmed.substring(tokens[0].length()).trim();

      CommandProcessor proc = CommandProcessorFactory.get(tokens[0]);
      if (proc != null) {
        if (proc instanceof Driver) {
          Driver qp = (Driver) proc;
          queryRet.add(new Integer(qp.run(cmd).getResponseCode()));
          ArrayList<String> res = new ArrayList<String>();
          try {
            while (qp.getResults(res)) {
              ArrayList<String> resCopy = new ArrayList<String>();
              resCopy.addAll(res);
              resultBucket.add(resCopy);
              if (resultBucket.size() > resultBucketMaxSize) {
                resultBucket.remove(0);
              }
              for (String row : res) {
                if (ss != null) {
                  if (ss.out != null) {
                    ss.out.println(row);
                  }
                } else {
                  throw new RuntimeException("ss was null");
                }
              }
              res.clear();
            }
          } catch (IOException ex) {
            l4j.error(getSessionName() + " getting results " + getResultFile()
                + " caused exception.", ex);
          }
          qp.close();
        } else {
          queryRet.add(new Integer(proc.run(cmd_1).getResponseCode()));
        }
      } else {
        // processor was null
        l4j.error(getSessionName()
            + " query processor was not found for query " + cmd);
      }
    } // end for

    // cleanup
    try {
      if (fos != null) {
        fos.close();
      }
    } catch (IOException ex) {
      l4j.error(getSessionName() + " closing result file " + getResultFile()
          + " caused exception.", ex);
    }
    status = WebSessionItemStatus.READY;
    l4j.debug(getSessionName() + " state is now READY");
    synchronized (runnable) {
      runnable.notifyAll();
    }
  }

  /**
   * This is a chained call to SessionState.setIsSilent(). Use this if you do
   * not want the result file to have information status
   */
  public void setSSIsSilent(boolean silent) throws HWIException {
    if (ss == null) {
      throw new HWIException("Session State is null");
    }
    ss.setIsSilent(silent);
  }

  /**
   * This is a chained call to SessionState.getIsSilent().
   */
  public boolean getSSIsSilent() throws HWIException {
    if (ss == null) {
      throw new HWIException("Session State is null");
    }
    return ss.getIsSilent();
  }

  /** to support sorting/Set. */
  public int compareTo(HWISessionItem other) {
    if (other == null) {
      return -1;
    }
    return getSessionName().compareTo(other.getSessionName());
  }

  /**
   * 
   * @return the HiveHistoryViewer for the session
   * @throws HWIException
   */
  public HiveHistoryViewer getHistoryViewer() throws HWIException {
    if (ss == null) {
      throw new HWIException("Session state was null");
    }
    /*
     * we can not call this.ss.get().getHiveHistory().getHistFileName() directly
     * as this call is made from a a Jetty thread and will return null
     */
    HiveHistoryViewer hv = new HiveHistoryViewer(historyFile);
    return hv;
  }

  /**
   * Uses the sessionName property to compare to sessions.
   * 
   * @return true if sessionNames are equal false otherwise
   */
  @Override
  public boolean equals(Object other) {
    if (other == null) {
      return false;
    }
    if (!(other instanceof HWISessionItem)) {
      return false;
    }
    HWISessionItem o = (HWISessionItem) other;
    if (getSessionName().equals(o.getSessionName())) {
      return true;
    } else {
      return false;
    }
  }

  public String getResultFile() {
    return resultFile;
  }

  public void setResultFile(String resultFile) {
    this.resultFile = resultFile;
  }

  /**
   * The session name is an identifier to recognize the session.
   * 
   * @return the session's name
   */
  public String getSessionName() {
    return sessionName;
  }

  /**
   * Used to represent to the user and other components what state the
   * HWISessionItem is in. Certain commands can only be run when the application
   * is in certain states.
   * 
   * @return the current status of the session
   */
  public WebSessionItemStatus getStatus() {
    return status;
  }

  /**
   * Currently unused.
   * 
   * @return a String with the full path to the error file.
   */
  public String getErrorFile() {
    return errorFile;
  }

  /**
   * Currently unused.
   * 
   * @param errorFile
   *          the full path to the file for results.
   */
  public void setErrorFile(String errorFile) {
    this.errorFile = errorFile;
  }

  /**
   * @return the auth
   */
  public HWIAuth getAuth() {
    return auth;
  }

  /**
   * @param auth
   *          the auth to set
   */
  protected void setAuth(HWIAuth auth) {
    this.auth = auth;
  }

  /** Returns an unmodifiable list of queries. */
  public List<String> getQueries() {
    return java.util.Collections.unmodifiableList(queries);
  }

  /**
   * Adds a new query to the execution list.
   * 
   * @param query
   *          query to be added to the list
   */
  public void addQuery(String query) throws HWIException {
    throwIfRunning();
    queries.add(query);
  }

  /**
   * Removes a query from the execution list.
   * 
   * @param item
   *          the 0 based index of the item to be removed
   */
  public void removeQuery(int item) throws HWIException {
    throwIfRunning();
    queries.remove(item);
  }

  public void clearQueries() throws HWIException {
    throwIfRunning();
    queries.clear();
  }

  /** returns the value for resultBucketMaxSize. */
  public int getResultBucketMaxSize() {
    return resultBucketMaxSize;
  }

  /**
   * sets the value for resultBucketMaxSize.
   * 
   * @param size
   *          the new size
   */
  public void setResultBucketMaxSize(int size) {
    resultBucketMaxSize = size;
  }

  /** gets the value for resultBucket. */
  public ArrayList<ArrayList<String>> getResultBucket() {
    return resultBucket;
  }

  /**
   * The HWISessionItem stores the result of each query in an array.
   * 
   * @return unmodifiable list of return codes
   */
  public List<Integer> getQueryRet() {
    return java.util.Collections.unmodifiableList(queryRet);
  }

  /**
   * If the ItemStatus is QueryRunning most of the configuration is in a read
   * only state.
   */
  private void throwIfRunning() throws HWIException {
    if (status == WebSessionItemStatus.QUERY_RUNNING) {
      throw new HWIException("Query already running");
    }
  }

}