/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.getopt.luke;

import java.awt.Color;
import java.awt.Font;
import java.awt.GraphicsEnvironment;
import java.awt.Toolkit;
import java.awt.datatransfer.Clipboard;
import java.awt.datatransfer.ClipboardOwner;
import java.awt.datatransfer.StringSelection;
import java.awt.datatransfer.Transferable;
import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.zip.GZIPOutputStream;

import javax.swing.JFileChooser;
import javax.swing.UIManager;

import org.apache.lucene.LucenePackage;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumberTools;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.misc.SweetSpotSimilarity;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.payloads.PayloadNearQuery;
import org.apache.lucene.search.payloads.PayloadTermQuery;
import org.apache.lucene.search.similar.MoreLikeThis;
import org.apache.lucene.search.spans.SpanFirstQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.store.*;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Version;
import org.apache.lucene.xmlparser.CoreParser;
import org.apache.lucene.xmlparser.CorePlusExtensionsParser;
import org.getopt.luke.DocReconstructor.Reconstructed;
import org.getopt.luke.decoders.BinaryDecoder;
import org.getopt.luke.decoders.DateDecoder;
import org.getopt.luke.decoders.Decoder;
import org.getopt.luke.decoders.NumDoubleDecoder;
import org.getopt.luke.decoders.NumFloatDecoder;
import org.getopt.luke.decoders.NumIntDecoder;
import org.getopt.luke.decoders.NumLongDecoder;
import org.getopt.luke.decoders.OldDateFieldDecoder;
import org.getopt.luke.decoders.OldNumberToolsDecoder;
import org.getopt.luke.decoders.StringDecoder;
import org.getopt.luke.plugins.ScriptingPlugin;
import org.getopt.luke.xmlQuery.XmlQueryParserFactory;
import org.getopt.luke.xmlQuery.CorePlusExtensionsParserFactory;

import thinlet.FrameLauncher;
import thinlet.Thinlet;

/**
 * This class allows you to browse a <a href="jakarta.apache.org/lucene">Lucene
 * </a> index in several ways - by document, by term, by query, and by most
 * frequent terms.
 * 
 * @author Andrzej Bialecki
 *  
 */
public class Luke extends Thinlet implements ClipboardOwner {

  private Directory dir = null;
  String pName = null;
  private IndexReader ir = null;
  private IndexSearcher is = null;
  private boolean slowAccess = false;
  private Collection<String> fn = null;
  private String[] idxFields = null;
  private HashMap<String, FieldTermCount> termCounts = new HashMap<String, FieldTermCount>();
  private List<LukePlugin> plugins = new ArrayList<LukePlugin>();
  private Object errorDlg = null;
  private Object infoDlg = null;
  private Object statmsg = null;
  private Object slowstatus = null;
  private Object slowmsg = null;
  private Analyzer stdAnalyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
  private Analyzer analyzer = null;
  //private QueryParser qp = null;
  private boolean readOnly = false;
  private boolean ram = false;
  private boolean keepCommits = false;
  private boolean multi = false;
  private int tiiDiv = 1;
  private IndexCommit currentCommit = null;
  private Similarity similarity = null;
  private Object lastST;
  private HashMap<String, Decoder> decoders = new HashMap<String, Decoder>();
  private Decoder defDecoder = new StringDecoder();
  
  /** Default salmon theme. */
  public static final int THEME_DEFAULT     = 0;
  /** Gray theme. */
  public static final int THEME_GRAY        = 1;
  /** Sandstone theme. */
  public static final int THEME_SANDSTONE   = 2;
  /** Sky blue theme. */
  public static final int THEME_SKY         = 3;
  /** Navy blue reverse theme. */
  public static final int THEME_NAVY        = 4;
  
  /** Theme color contants. */
  public int[][] themes = {
          {0xece9d0, 0x000000, 0xf5f4f0, 0x919b9a, 0xb0b0b0, 0xeeeeee, 0xb9b9b9, 0xff8080, 0xc5c5dd}, // default
          {0xe6e6e6, 0x000000, 0xffffff, 0x909090, 0xb0b0b0, 0xededed, 0xb9b9b9, 0x89899a, 0xc5c5dd}, // gray
          {0xeeeecc, 0x000000, 0xffffff, 0x999966, 0xb0b096, 0xededcb, 0xcccc99, 0xcc6600, 0xffcc66}, // sandstone
          {0xf0f0ff, 0x0000a0, 0xffffff, 0x8080ff, 0xb0b0b0, 0xededed, 0xb0b0ff, 0xff0000, 0xfde0e0}, // sky
          {0x6375d6, 0xffffff, 0x7f8fdd, 0xd6dff5, 0x9caae5, 0x666666, 0x003399, 0xff3333, 0x666666}  // navy
  };

  private int numTerms = 0;
  private static boolean exitOnDestroy = false;
  private Class[] analyzers = null;

  private String baseDir = null;

  private Class[] defaultAnalyzers = { SimpleAnalyzer.class, StandardAnalyzer.class, StopAnalyzer.class,
      WhitespaceAnalyzer.class };

  private static final String MSG_NOINDEX = "FAILED: No index, or index is closed. Reopen it.";
  private static final String MSG_READONLY = "FAILED: Read-Only index.";
  private static final String MSG_CONV_ERROR = "Some values could not be properly represented in this format. " + 
                      "They are marked in grey and presented as a hex dump.";

  /** Default constructor, loads preferences, initializes plugins and GUI. */ 
  public Luke() {
    super();
    Prefs.load();
    try {
      UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
    } catch (Exception e) {}
    setTheme(Prefs.getInteger(Prefs.P_THEME, THEME_DEFAULT));
    String fontName = Prefs.getProperty(Prefs.P_FONT_NAME, "SansSerif");
    String fontSize = Prefs.getProperty(Prefs.P_FONT_SIZE, "12.0");
    float fsize = 12.0f;
    try {
      fsize = Float.parseFloat(fontSize);
    } catch (Exception e) {};
    Font f = new Font(fontName, Font.PLAIN, (int)fsize);
    setFont(f);
    addComponent(this, "/xml/luke.xml", null, null);
    errorDlg = addComponent(null, "/xml/error.xml", null, null);
    infoDlg = addComponent(null, "/xml/info.xml", null, null);
    statmsg = find("statmsg");
    slowstatus = find("slowstat");
    slowmsg = find(slowstatus, "slowmsg");
    // populate analyzers
    try {
      Class[] an = ClassFinder.getInstantiableSubclasses(Analyzer.class);
      if (an == null || an.length == 0) {
        analyzers = defaultAnalyzers;
      } else {
        HashSet<Class> uniq = new HashSet<Class>(Arrays.asList(an));
        analyzers = (Class[])uniq.toArray(new Class[uniq.size()]);
      }
      Object cbType = find("cbType");
      populateAnalyzers(cbType);
    } catch (Exception e) {
      e.printStackTrace();
    }
    loadPlugins();
    
  }

  /**
   * Set color theme for the UI.
   * @param which one of the predefined themes. For custom themes use {@link Thinlet#setColors(int, int, int, int, int, int, int, int, int)}.
   */
  public void setTheme(int which) {
    if (which < 0 || which >= themes.length) which = THEME_DEFAULT;
    int[] t = themes[which];
    setColors(t[0], t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8]);
    Prefs.setProperty(Prefs.P_THEME, which + "");
  }
  
  /**
   * Action handler to select color theme.
   * @param menu
   */
  public void actionTheme(Object menu) {
    String which = (String)getProperty(menu, "t");
    int t = THEME_DEFAULT;
    try {
      t = Integer.parseInt(which);
    } catch (Exception e) {};
    setTheme(t);
  }
  
  /**
   * Populate a combobox with the current list of analyzers.
   * @param combo
   */
  public void populateAnalyzers(Object combo) {
    removeAll(combo);
    String[] aNames = new String[analyzers.length];
    for (int i = 0; i < analyzers.length; i++) {
      aNames[i] = analyzers[i].getName();
    }
    Arrays.sort(aNames);
    for (int i = 0; i < aNames.length; i++) {
      Object choice = create("choice");
      setString(choice, "text", aNames[i]);
      add(combo, choice);
      if (i == 0) {
        setString(combo, "text", aNames[i]);
      }
    }
    int lastAnalyzerIdx = 0;
    String lastAnalyzer = Prefs.getProperty(Prefs.P_ANALYZER);
    if (lastAnalyzer != null) lastAnalyzerIdx = getIndex(combo, lastAnalyzer);
    if (lastAnalyzerIdx < 0) lastAnalyzerIdx = 0;
    setInteger(combo, "selected", lastAnalyzerIdx);
  }

  /**
   * Return an array of available Analyzer implementations.
   * @return
   */
  public Class[] getAnalyzers() {
    return analyzers;
  }

  /**
   * Loads plugins. Plugins are first searched from the CLASSPATH, and then from a
   * plugin list contained in a resource file "/.plugins". The "/.plugins" resource file
   * has a simple format - one fully qualified class name per line. Blank lines and
   * lines starting with '#' are ignored.
   */
  private void loadPlugins() {
    List pluginClasses = new ArrayList();
    // try to find all plugins
    try {
      Class classes[] = ClassFinder.getInstantiableSubclasses(LukePlugin.class);
      if (classes != null && classes.length > 0) {
        pluginClasses.addAll(Arrays.asList(classes));
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
    // load plugins declared in the ".plugins" file
    try {
      InputStream is = getClass().getResourceAsStream("/.plugins");
      if (is != null) {
        BufferedReader br = new BufferedReader(new InputStreamReader(is));
        String line = null;
        while ((line = br.readLine()) != null) {
          if (line.startsWith("#")) continue;
          if (line.trim().equals("")) continue;
          try {
            Class clazz = Class.forName(line.trim());
            if (clazz.getSuperclass().equals(LukePlugin.class) && !pluginClasses.contains(clazz)) {
              pluginClasses.add(clazz);
            }
          } catch (Throwable x) {
            //
          }
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
    try {
      StringBuffer errors = new StringBuffer("Unable to load some plugins:");
      boolean failures = false;
      for (int i = 0; i < pluginClasses.size(); i++) {
        try {
          LukePlugin plugin = (LukePlugin) ((Class) pluginClasses.get(i)).getConstructor(new Class[0]).newInstance(
                  new Object[0]);
          String xul = plugin.getXULName();
          if (xul == null) continue;
          Object ui = parse(xul, plugin);
          plugin.setApplication(this);
          plugin.setMyUi(ui);
          plugins.add(plugin);
        } catch (Exception e) {
          failures = true;
          e.printStackTrace();
          errors.append("\n" + pluginClasses.get(i).toString());
        }
      }
      if (failures) {
        errorMsg(errors.toString());
      }
    } catch (Exception e) {
      e.printStackTrace();
      errorMsg(e.toString());
    }
    if (plugins.size() == 0) return;
    initPlugins();
  }

  /**
   * Create UI for a single plugin.
   * @param tabs parent tabbedpane
   * @param plugin plugin instance
   */
  private void addPluginTab(Object tabs, LukePlugin plugin) {
    Object tab = create("tab");
    setColor(tab, "foreground", new Color(0x006000));
    setString(tab, "text", plugin.getPluginName());
    setFont(tab, getFont().deriveFont(Font.BOLD));
    add(tabs, tab);
    Object panel = create("panel");
    setInteger(panel, "gap", 2);
    setInteger(panel, "weightx", 1);
    setInteger(panel, "weighty", 1);
    setChoice(panel, "halign", "fill");
    setChoice(panel, "valign", "fill");
    setInteger(panel, "columns", 1);
    add(tab, panel);
    Object infobar = create("panel");
    setInteger(infobar, "gap", 8);
    setInteger(infobar, "top", 2);
    setInteger(infobar, "bottom", 2);
    setInteger(infobar, "weightx", 1);
    setChoice(infobar, "halign", "fill");
    setColor(infobar, "background", new Color(0xc0f0c0));
    add(panel, infobar);
    Object label = create("label");
    setString(label, "text", plugin.getPluginInfo());
    add(infobar, label);
    Object link = create("button");
    setChoice(link, "type", "link");
    setString(link, "text", plugin.getPluginHome());
    putProperty(link, "url", plugin.getPluginHome());
    setMethod(link, "action", "goUrl(this)", infobar, this);
    add(infobar, link);
    add(panel, create("separator"));
    add(panel, plugin.getMyUi());
  }

  /**
   * Return the list of active plugin instances.
   * @return
   */
  public List getPlugins() {
    return Collections.unmodifiableList(plugins);
  }
  
  /**
   * Get an already instantiated plugin, or null if such plugin was
   * not loaded on startup.
   * @param className fully qualified plugin classname
   * @return
   */
  public LukePlugin getPlugin(String className) {
    for (int i = 0; i < plugins.size(); i++) {
      Object plugin = plugins.get(i);
      if (plugin.getClass().getName().equals(className))
        return (LukePlugin)plugin;
    }
    return null;
  }
  
  Thread statusThread = null;
  long lastUpdate = 0;
  long statusSleep = 0;
  
  /**
   * Display a message on the status bar for 5 seconds.
   * @param msg message to display. Too long messages will be truncated by the UI.
   */
  public void showStatus(final String msg) {
    if (statusThread != null && statusThread.isAlive()) {
      setString(statmsg, "text", msg);
      statusSleep = 5000;
    } else {
      statusThread = new Thread() {
        public void run() {
          statusSleep = 5000;
          setString(statmsg, "text", msg);
          while (statusSleep > 0) {
            try {
              sleep(500);
            } catch (Exception e) {};
            statusSleep -= 500;
          }
          setString(statmsg, "text", "");
        }
      };
      statusThread.start();
    }
  }
  
  /**
   * As {@link #showStatus(String)} but also sets the "Last search time" label.
   * @param msg
   */
  public void showSearchStatus(String msg) {
    setString(lastST, "text", msg);
    showStatus(msg);
  }
  
  long lastSlowUpdate = 0L;
  long lastSlowCounter = 0L;
  Thread slowThread = null;
  long slowSleep = 0;
  
  public void showSlowStatus(final String msg, final long counter) {
    if (slowThread != null && slowThread.isAlive()) {
      lastSlowCounter += counter;
      setString(slowmsg, "text", msg + " " + lastSlowCounter);
      slowSleep = 5000;
    } else {
      slowThread = new Thread() {
        public void run() {
          slowSleep = 5000;
          lastSlowCounter = counter;
          setBoolean(slowstatus, "visible", true);
          setString(slowmsg, "text", msg + " " + lastSlowCounter);
          while (slowSleep > 0) {
            try {
              sleep(500);
            } catch (Exception e) {};
            slowSleep -= 500;
          }
          setString(slowmsg, "text", "");
          setBoolean(slowstatus, "visible", false);
        }
      };
      slowThread.start();
    }
  }

  /**
   * Add a Thinlet component from XUL file.
   * @param parent add the new component to this parent 
   * @param compView path to the XUL resource
   * @param handlerStr fully qualified classname of the handler to instantiate,
   * or null if the current class will become the handler
   * @param argv if not null, these arguments will be passed to the
   * appropriate constructor.
   * @return
   */
  public Object addComponent(Object parent, String compView, String handlerStr, Object[] argv) {
    Object res = null;
    Object handler = null;
    try {
      if (handlerStr != null) {
        if (argv == null) {
          handler = Class.forName(handlerStr).getConstructor(new Class[] { Thinlet.class }).newInstance(
                  new Object[] { this });
        } else {
          handler = Class.forName(handlerStr).getConstructor(new Class[] { Thinlet.class, Object[].class })
                  .newInstance(new Object[] { this, argv });
        }
      }
      if (handler != null) {
        res = parse(compView, handler);
      } else res = parse(compView);
      if (parent != null) {
        if (parent instanceof Thinlet)
          add(res);
        else add(parent, res);
      }
      return res;
    } catch (Exception exc) {
      exc.printStackTrace();
      errorMsg(exc.getMessage());
      return null;
    }
  }

  /**
   * Show a modal error dialog with OK button.
   * @param msg error message
   */
  public void errorMsg(String msg) {
    Object fMsg = find(errorDlg, "msg");
    setString(fMsg, "text", msg);
    add(errorDlg);
  }

  /**
   * Show a modal info dialog with OK button.
   * @param msg info message
   */
  public void infoMsg(String msg) {
    Object fMsg = find(infoDlg, "msg");
    setString(fMsg, "text", msg);
    add(infoDlg);
  }

  /**
   * Show an "Open Index" dialog.
   *
   */
  public void actionOpen() {
    Object dialog = addComponent(this, "/xml/lukeinit.xml", null, null);
    Object path = find(dialog, "path");
    if (this.baseDir != null)
      setString(path, "text", this.baseDir);
    else setString(path, "text", System.getProperty("user.dir"));
  }

  /**
   * Browse for a directory, and put the selection result in the
   * indicated widget.
   * @param path Thinlet widget to put the result
   */
  public void openBrowse(Object path) {
    JFileChooser fd = new JFileChooser();
    fd.setDialogType(JFileChooser.OPEN_DIALOG);
    fd.setDialogTitle("Select Index directory");
    fd.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES);
    fd.setFileHidingEnabled(false);
    String strPath = getString(path, "text");
    if (strPath != null) strPath.trim();
    if (strPath != null && strPath.length() > 0)
      fd.setCurrentDirectory(new File(strPath));
    else if (this.baseDir != null)
      fd.setCurrentDirectory(new File(this.baseDir));
    else fd.setCurrentDirectory(new File(System.getProperty("user.dir")));
    int res = fd.showOpenDialog(this);
    File iDir = null;
    if (res == JFileChooser.APPROVE_OPTION) iDir = fd.getSelectedFile();
    if (iDir != null && iDir.exists()) {
      if (!iDir.isDirectory()) iDir = iDir.getParentFile();
      setString(path, "text", iDir.toString());
    }
  }

  
  /**
   * Select an output file name, and put the selection result in the
   * indicated widget.
   * @param path Thinlet widget to put the result
   */
  public void saveBrowse(Object path, Object startButton) {
    JFileChooser fd = new JFileChooser();
    fd.setDialogType(JFileChooser.SAVE_DIALOG);
    fd.setDialogTitle("Select Output File");
    fd.setFileSelectionMode(JFileChooser.FILES_ONLY);
    fd.setFileHidingEnabled(false);
    String strPath = getString(path, "text");
    if (strPath != null) strPath.trim();
    if (strPath != null && strPath.length() > 0)
      fd.setCurrentDirectory(new File(strPath));
    else if (this.baseDir != null)
      fd.setCurrentDirectory(new File(this.baseDir));
    else fd.setCurrentDirectory(new File(System.getProperty("user.dir")));
    int res = fd.showSaveDialog(this);
    setBoolean(startButton, "enabled", false);
    File iFile = null;
    if (res == JFileChooser.APPROVE_OPTION) iFile = fd.getSelectedFile();
    if (iFile != null) {
      setString(path, "text", iFile.toString());
      setBoolean(startButton, "enabled", true);
    }
  }

  
  /**
   * Initialize MRU list of indexes in the open index dialog.
   * @param dialog
   */
  public void setupInit(Object dialog) {
    Object path = find(dialog, "path");
    syncMRU(path);
  }

  /**
   * Attempt to load the index with parameters specified in the dialog.
   * <p>NOTE: this method is invoked from the UI. If you need to open an index
   * programmatically, you should use {@link #openIndex(String, boolean, boolean, boolean)} instead.</p>
   * @param dialog UI dialog with parameters
   */
  public void openOk(Object dialog) {
    Object path = find(dialog, "path");
    pName = getString(path, "text").trim();

    boolean force = getBoolean(find(dialog, "force"), "selected");
    boolean noReader = getBoolean(find(dialog, "cbNoReader"), "selected");
    tiiDiv = 1;
    try {
      tiiDiv = Integer.parseInt(getString(find(dialog, "tiiDiv"), "text"));
    } catch (Exception e) {
      e.printStackTrace();
    }
    Object dirImpl = getSelectedItem(find(dialog, "dirImpl"));
    String dirClass = null;
    if (dirImpl == null) {
      dirClass = FSDirectory.class.getName();
    } else {
      String name = getString(dirImpl, "name");
      if (name == null) {
        dirClass = getString(dirImpl, "text");
      } else {
        if (name.equals("fs")) {
          dirClass = FSDirectory.class.getName();
        } else if (name.equals("mmap")) {
          dirClass = MMapDirectory.class.getName();
        } else if (name.equals("niofs")) {
          dirClass = NIOFSDirectory.class.getName();
        }
      }
    }
    if (pName == null || pName.trim().equals("")) {
      errorMsg("Invalid path.");
      return;
    }
    readOnly = getBoolean(find(dialog, "ro"), "selected");
    ram = getBoolean(find(dialog, "ram"), "selected");
    keepCommits = getBoolean(find(dialog, "cbKeepCommits"), "selected");
    slowAccess = getBoolean(find(dialog, "cbSlowIO"), "selected");
    decoders.clear();
    currentCommit = null;
    Prefs.addToMruList(pName);
    syncMRU(path);
    remove(dialog);
    if (noReader) {
      removeAll();
      addComponent(this, "/xml/luke.xml", null, null);
      try {
        Directory d = openDirectory(dirClass, pName, false);
        if (IndexReader.indexExists(d)) {
          throw new Exception("there is no valid Lucene index in this directory.");
        }
        dir = d;
        initOverview();
        infoMsg("There is no IndexReader - most actions are disabled. " +
            "You can open IndexReader from current Directory using 'Re-Open'");
      } catch (Exception e) {
        errorMsg("ERROR: " + e.toString());
      }
    } else {
      openIndex(pName, force, dirClass, readOnly, ram, keepCommits, null, tiiDiv);
    }
  }
  
  public void actionClose() {
    if (ir != null) {
      try {
        if (is != null) is.close();
        ir.close();
        if (dir != null) dir.close();
      } catch (Exception e) {
        e.printStackTrace();
        errorMsg("Close failed: " + e.getMessage());
      }
    }
    ir = null;
    dir = null;
    is = null;
    removeAll();
    addComponent(this, "/xml/luke.xml", null, null);
    initPlugins();
  }
  
  public void actionCommit() {
    if (ir == null) {
      showStatus(MSG_NOINDEX);
      return;
    }
    if (readOnly) {
      showStatus(MSG_READONLY);
      return;
    }
    if (!IndexGate.hasChanges(ir)) {
      showStatus("No changes - commit ignored.");
      return;
    }
    Object dialog = addComponent(this, "/xml/commit.xml", null, null);
    Map userData = ir.getCommitUserData();
    TreeMap ud = new TreeMap(userData);
    putProperty(dialog, "userData", ud);
    _showUserData(dialog);
  }
  
  private void _showUserData(Object dialog) {
    Object table = find(dialog, "data");
    removeAll(table);
    Map<Object,Object> ud = (Map)getProperty(dialog, "userData");
    for (Entry e : ud.entrySet()) {
      Object row = create("row");
      putProperty(row, "key", e.getKey());
      add(table, row);
      Object cell = create("cell");
      setString(cell, "text", e.getKey().toString());
      add(row, cell);
      cell = create("cell");
      setString(cell, "text", e.getValue().toString());
      add(row, cell);
    }
  }
  
  public void putUserData(Object dialog) {
    Object key = find(dialog, "key");
    Object value = find(dialog, "value");
    String k = getString(key, "text");
    String v = getString(value, "text");
    if (k.equals("")) {
      showStatus("Cannot add empty key.");
      return;
    }    
    Map<Object,Object> ud = (Map)getProperty(dialog, "userData");
    ud.put(k, v);
    _showUserData(dialog);
  }
  
  public void deleteUserData(Object dialog) {
    Object table = find(dialog, "data");
    Map ud = (Map)getProperty(dialog, "userData");
    Object[] rows = getSelectedItems(table);
    if (rows == null || rows.length == 0) {
      return;
    }
    for (Object row : rows) {
      Object key = getProperty(row, "key");
      ud.remove(key);
    }
    _showUserData(dialog);
  }
  
  public void commitUserData(Object dialog) {
    Map userData = (Map)getProperty(dialog, "userData");
    remove(dialog);
    try {
      ir.flush(userData);
      initOverview();
      showFiles(dir, Collections.EMPTY_LIST);
    } catch (Exception e) {
      errorMsg("Error: " + e.toString());
    }
  }
  
  public void actionReopen() {
    if (dir == null) {
      return;
    }
    openIndex(pName, false, dir.getClass().getName(), readOnly, ram,
        keepCommits, currentCommit, tiiDiv);
  }
  /**
   * Open indicated index and re-initialize all GUI and plugins.
   * @param pName path to index
   * @param force if true, and the index is locked, unlock it first. If false, and
   * the index is locked, an error will be reported.
   * @param readOnly open in read-only mode, and disallow modifications.
   */
  public void openIndex(String name, boolean force, String dirImpl, boolean ro,
      boolean ramdir, boolean keepCommits, IndexCommit point, int tiiDivisor) {
    pName = name;
    readOnly = ro;
    removeAll();
    File baseFileDir = new File(name);
    this.baseDir = baseFileDir.toString();
    addComponent(this, "/xml/luke.xml", null, null);
    statmsg = find("statmsg");
    if (dir != null) {
      try {
        if (ir != null) ir.close();
      } catch (Exception e) {}
      ;
      try {
        if (dir != null) dir.close();
      } catch (Exception e) {}
      ;
    }
    ArrayList<Directory> dirs = new ArrayList<Directory>();
    try {
      Directory d = openDirectory(dirImpl, pName, false);
      if (IndexWriter.isLocked(d)) {
        if (ro) {
          errorMsg("Index is locked and Read-Only. Open for read-write and 'Force unlock'.");
          d.close();
          d = null;
          return;
        }
        if (force) {
          IndexWriter.unlock(d);
        } else {
          errorMsg("Index is locked. Try 'Force unlock' when opening.");
          d.close();
          d = null;
          return;
        }
      }
      boolean existsSingle = false;
      try {
        existsSingle = IndexReader.indexExists(d);
      } catch (Exception e) {
        //
      }
      
      if (!existsSingle) { // try multi
        File[] files = baseFileDir.listFiles();
        for (File f : files) {
          if (f.isFile()) {
            continue;
          }
          Directory d1 = openDirectory(dirImpl, f.toString(), false);
          if (IndexWriter.isLocked(d1)) {
            if (ro) {
              errorMsg("Index is locked and Read-Only. Open for read-write and 'Force unlock'.");
              d1.close();
              d1 = null;
              return;
            }
            if (force) {
              IndexWriter.unlock(d1);
            } else {
              errorMsg("Index is locked. Try 'Force unlock' when opening.");
              d1.close();
              d1 = null;
              return;
            }
          }
          existsSingle = false;
          try {
            existsSingle = IndexReader.indexExists(d1);
          } catch (Exception e) {};
          if (!existsSingle) {
            d1.close();
            continue;
          }
          dirs.add(d1);
        }
      } else {
        dirs.add(d);
      }
      
      if (dirs.size() == 0) {
        errorMsg("No valid directory at the location, try another location.");
        return;
      }

      if (ramdir) {
        showStatus("Loading index into RAMDirectory ...");
        Directory dir1 = new RAMDirectory();
        IndexWriter iw1 = new IndexWriter(dir1, new SimpleAnalyzer(), MaxFieldLength.UNLIMITED);
        iw1.addIndexesNoOptimize((Directory[])dirs.toArray(new Directory[dirs.size()]));
        iw1.close();
        showStatus("RAMDirectory loading done!");
        dir.close();
        dir = dir1;
      }
      IndexDeletionPolicy policy;
      if (keepCommits) {
        policy = new KeepAllIndexDeletionPolicy();
      } else {
        policy = new KeepLastIndexDeletionPolicy();
      }
      ArrayList<IndexReader> readers = new ArrayList<IndexReader>();
      for (Directory dd : dirs) {
        IndexReader reader;
        if (tiiDivisor > 1) {
          reader = IndexReader.open(dd, policy, ro, tiiDivisor);
        } else {
          reader = IndexReader.open(dd, policy, ro);
        }
        readers.add(reader);
      }
      if (readers.size() == 1) {
        ir = readers.get(0);
        dir = ir.directory();
      } else {
        ir = new MultiReader((IndexReader[])readers.toArray(new IndexReader[readers.size()]));
      }
      is = new IndexSearcher(ir);
      // XXX 
      slowAccess = false;
      initOverview();
      initPlugins();
      showStatus("Index successfully open.");
    } catch (Exception e) {
      e.printStackTrace();
      errorMsg(e.getMessage());
      return;
    }
  }

  /**
   * Open a single directory.
   * @param dirImpl fully-qualified class name of Directory implementation,
   * or "FSDirectory" for {@link FSDirectory}
   * @param file index directory
   * @param create if true, create a new directory
   * @return directory implementation
   */
  Class defaultDirImpl = null;
  
  public Directory openDirectory(String dirImpl, String file, boolean create) throws Exception {
    File f = new File(file);
    if (!f.exists()) {
      throw new Exception("Index directory doesn't exist.");
    }
    Directory res = null;
    if (dirImpl == null || dirImpl.equals(FSDirectory.class.getName())) {
      return FSDirectory.open(f);
    }
    try {
      Class implClass = Class.forName(dirImpl);
      Constructor<Directory> constr = implClass.getConstructor(File.class);
      res = constr.newInstance(f);
    } catch (Throwable e) {
      errorMsg("Invalid directory implementation class: " + dirImpl + " " + e);
      return null;
    }
    if (res != null) return res;
    // fall-back to FSDirectory.
    if (res == null) return FSDirectory.open(f);
    return null;
  }
  
  /**
   * Indicates whether I/O access should be optimized because
   * the index is on a slow medium (e.g. remote).
   * @return true if I/O access is costly and should be minimized
   */
  public boolean isSlowAccess() {
    return slowAccess;
  }
  
  /**
   * Set whether the I/O access to this index is costly and
   * should be minimized.
   */
  public void setSlowAccess(boolean slowAccess) {
    this.slowAccess = slowAccess;
    if (slowAccess) {
      
    }
  }

  /**
   * Initialize plugins. This method should always be called when a new index is open.
   *
   */
  public void initPlugins() {
    Object pluginsTabs = find("pluginsTabs");
    removeAll(pluginsTabs);
    for (int i = 0; i < plugins.size(); i++) {
      LukePlugin plugin = (LukePlugin) plugins.get(i);
      addPluginTab(pluginsTabs, plugin);
      plugin.setDirectory(dir);
      plugin.setIndexReader(ir);
      try {
        plugin.init();
      } catch (Exception e) {
        e.printStackTrace();
        showStatus("PLUGIN ERROR: " + e.getMessage());
      }
    }
  }

  /**
   * Initialize index overview and other GUI elements. This method is called always
   * when a new index is open.
   *
   */
  private void initOverview() {
    try {
      courier = new Font("Courier", getFont().getStyle(), getFont().getSize());
      lastST = find("lastST");
      setBoolean(find("bReload"), "enabled", true);
      setBoolean(find("bClose"), "enabled", true);
      setBoolean(find("bCommit"), "enabled", true);
      Object cbType = find("cbType");
      populateAnalyzers(cbType);
      Object pOver = find("pOver");
      Object iName = find("idx");
      String idxName;
      if (pName.length() > 40) {
        idxName = pName.substring(0, 10) + "..." + pName.substring(pName.length() - 27);
      } else {
        idxName = pName;
      }
      setString(iName, "text", idxName + (readOnly ? " (R)" : ""));
      iName = find(pOver, "iName");
      setString(iName, "text", pName + (readOnly ? " (Read-Only)" : ""));
      Object dirImpl = find("dirImpl");
      String implName = "N/A";
      if (dir == null) {
        if (ir != null) {
          implName = "N/A (reader is " + ir.getClass().getName() + ")";
        }
      } else {
        implName = dir.getClass().getName();
      }
      setString(dirImpl, "text", implName);
      Object fileSize = find("iFileSize");
      long totalFileSize = Util.calcTotalFileSize(pName, dir);
      setString(fileSize, "text", Util.normalizeSize(totalFileSize) + Util.normalizeUnit(totalFileSize));
      if (ir == null) {
        return;
      }      
      // we need IndexReader from now on
      Object iMod = find(pOver, "iMod");
      String modText = "N/A";
      if (dir != null) {
        modText = new Date(IndexReader.lastModified(dir)).toString();
      }
      setString(iMod, "text", modText);
      Object iDocs = find(pOver, "iDocs");
      String numdocs = String.valueOf(ir.numDocs());
      setString(iDocs, "text", numdocs);
      iDocs = find("iDocs1");
      setString(iDocs, "text", String.valueOf(ir.maxDoc() - 1));
      Object iFields = find(pOver, "iFields");
      fn = ir.getFieldNames(IndexReader.FieldOption.ALL);
      if (fn.size() == 0) {
        showStatus("Empty index.");
      }
      showFiles(dir, null);
      showCommits();
      final Object fList = find(pOver, "fList");
      final Object defFld = find("defFld");
      final Object fCombo = find("fCombo");
      TreeSet<String> fields = new TreeSet<String>(fn);
      idxFields = (String[])fields.toArray(new String[fields.size()]);
      setString(iFields, "text", String.valueOf(idxFields.length));
      final Object iTerms = find(pOver, "iTerms");
      if (!slowAccess) {
        Thread t = new Thread() {
          public void run() {
            Object r = create("row");
            Object cell = create("cell");
            add(r, cell);
            add(fList, r);
            setBoolean(cell, "enabled", false);
            setString(cell, "text", "..wait..");
            termCounts.clear();
            FieldTermCount ftc = null;
            try {
              TermEnum te = ir.terms();
              numTerms = 0;
              while (te.next()) {
                Term currTerm = te.term();
                if (ftc == null) {
                  // initialize
                  ftc = new FieldTermCount();
                  ftc.fieldname = currTerm.field();
                  termCounts.put(ftc.fieldname, ftc);
                }
                if (ftc.fieldname == currTerm.field()) {
                  ftc.termCount++;
                } else {
                  ftc = new FieldTermCount();
                  ftc.fieldname = currTerm.field();
                  ftc.termCount++;
                  termCounts.put(ftc.fieldname, ftc);
                }
                numTerms++;
              }
              te.close();
              setString(iTerms, "text", String.valueOf(numTerms));
              initFieldList(fList, fCombo, defFld);
            } catch (Exception e) {
              showStatus("ERROR: can't count terms per field");
            }
          }
        };
        t.start();
      } else {
        setString(iTerms, "text", "N/A");        
        initFieldList(fList, fCombo, defFld);
      }
      Object iDel = find(pOver, "iDelOpt");
      String sDel = ir.hasDeletions() ? "Yes (" + ir.numDeletedDocs() + ")" : "No";
      String sDelOpt = sDel + " / " +
                      (ir.isOptimized() ? "Yes" : "No");
      setString(iDel, "text", sDelOpt);
      Object iVer = find(pOver, "iVer");
      String verText = "N/A";
      if (dir != null) {
        verText = Long.toHexString(IndexReader.getCurrentVersion(dir));
      }
      setString(iVer, "text", verText);
      Object iFormat = find(pOver, "iFormat");
      Object iCaps = find(pOver, "iCaps");
      String formatText = "N/A";
      String formatCaps = "N/A";
      if (dir != null) {
        int format = IndexGate.getIndexFormat(dir);
        IndexGate.FormatDetails formatDetails = IndexGate.getFormatDetails(format);
        formatText = format + " (" + formatDetails.genericName + ")";
        formatCaps = formatDetails.capabilities;
      }
      setString(iFormat, "text", formatText);
      setString(iCaps, "text", formatCaps);
      Object iTiiDiv = find(pOver, "iTiiDiv");
      String divText = "N/A";
      // not available in Lucene 3.0
//      try {
//        divText = String.valueOf(ir.getTermInfosIndexDivisor());
//      } catch (UnsupportedOperationException uoe) {
//      }
      setString(iTiiDiv, "text", divText);
      Object iCommit = find(pOver, "iCommit");
      String commitText = "N/A";
      try {
        IndexCommit commit = ir.getIndexCommit();
        commitText = commit.getSegmentsFileName() + " (" +
          new Date(commit.getTimestamp()).toString() + ")";
      } catch (UnsupportedOperationException uoe) {
      }
      setString(iCommit, "text", commitText);
      Object iUser = find(pOver, "iUser");
      String userData = null;
      try {
        Map userDataMap = ir.getCommitUserData();
        if (userDataMap != null && !userDataMap.isEmpty()) {
          userData = ir.getCommitUserData().toString();
        } else {
          userData = "--";
        }
      } catch (UnsupportedOperationException uoe) {
        userData = "(not supported)";
      }
      setString(iUser, "text", userData);
      final Object nTerms = find("nTerms");
      if (!slowAccess) {
        Thread t = new Thread() {
          public void run() {
            actionTopTerms(nTerms);
          }
        };
        t.start();
      }
    } catch (Exception e) {
      e.printStackTrace();
      errorMsg(e.getMessage());
    }
  }

  private void initFieldList(Object fList, Object fCombo, Object defFld) {
    removeAll(fList);
    removeAll(fCombo);
    removeAll(defFld);
    setString(fCombo, "text", idxFields[0]);
    setString(defFld, "text", idxFields[0]);
    NumberFormat intCountFormat = NumberFormat.getIntegerInstance();
    NumberFormat percentFormat = NumberFormat.getNumberInstance();
    intCountFormat.setGroupingUsed(true);
    percentFormat.setMaximumFractionDigits(2);
    // sort by names now
    for (String s : idxFields) {
      Object row = create("row");
      putProperty(row, "fName", s);
      add(fList, row);
      Object cell = create("cell");
      setString(cell, "text", s);
      add(row, cell);
      cell = create("cell");
      FieldTermCount ftc = termCounts.get(s);
      if (ftc != null) {
        long cnt = ftc.termCount;
        setString(cell, "text", intCountFormat.format(cnt));
        setChoice(cell, "alignment", "right");
        add(row, cell);
        float pcent = (float)(cnt * 100) / (float)numTerms;
        cell = create("cell");
        setString(cell, "text", percentFormat.format(pcent) + " %");
        setChoice(cell, "alignment", "right");
        add(row, cell);
      } else {
        setString(cell, "text", "0");
        setChoice(cell, "alignment", "right");
        add(row, cell);
        cell = create("cell");
        setString(cell, "text", "0.00 %");
        setChoice(cell, "alignment", "right");
        add(row, cell);
      }
      cell = create("cell");
      setChoice(cell, "alignment", "right");
      Decoder dec = decoders.get(s);
      if (dec == null) dec = defDecoder;
      setString(cell, "text", dec.toString());
      add(row, cell);
      // populate combos
      Object choice = create("choice");
      add(fCombo, choice);
      setString(choice, "text", s);
      putProperty(choice, "fName", s);
      choice = create("choice");
      add(defFld, choice);
      setString(choice, "text", s);
      putProperty(choice, "fName", s);
    }
    setString(find("defFld"), "text", idxFields[0]);
    // Remove columns
    Object header = get(find("sTable"), "header");
    removeAll(header);
    Object c = create("column");
    setString(c, "text", "#");
    setInteger(c, "width", 40);
    add(header, c);
    c = create("column");
    setString(c, "text", "Score");
    setInteger(c, "width", 50);
    add(header, c);
    c = create("column");
    setString(c, "text", "Doc. Id");
    setInteger(c, "width", 60);
    add(header, c);
    for (int j = 0; j < idxFields.length; j++) {
      c = create("column");
      setString(c, "text", idxFields[j]);
      add(header, c);
    }
  }
  
  private void showCommits() throws Exception {
    Object commitsTable = find("commitsTable");
    removeAll(commitsTable);
    if (dir == null) {
      Object row = create("row");
      Object cell = create("cell");
      setString(cell, "text", "<not available>");
      setBoolean(cell, "enabled", false);
      add(row, cell);
      add(commitsTable, row);
      return;
    }
    Collection commits = IndexReader.listCommits(dir);
    // commits are ordered from oldest to newest ?
    Iterator it = commits.iterator();
    int rowNum = 0;
    while (it.hasNext()) {
      IndexCommit commit = (IndexCommit)it.next();
      // figure out the name of the segment files
      Collection files = commit.getFileNames();
      Iterator itf = files.iterator();
      Object row = create("row");
      boolean enabled = rowNum < commits.size() - 1;
      Color color = null;
      rowNum++;
      add(commitsTable, row);
      putProperty(row, "commit", commit);
      if (enabled) {
        putProperty(row, "commitDeletable", Boolean.TRUE);
      }
      Object cell = create("cell");
      setString(cell, "text", String.valueOf(commit.getGeneration()));
      add(row, cell);
      cell = create("cell");
      char[] flags = new char[]{'-', '-'};
      if (commit.isDeleted()) flags[0] = 'D';
      if (commit.isOptimized()) flags[1] = 'O';
      setString(cell, "text", new String(flags));
      setFont(cell, "font", courier);
      setChoice(cell, "alignment", "center");
      add(row, cell);
      cell = create("cell");
      setString(cell, "text", Long.toHexString(commit.getVersion()));
      add(row, cell);
      cell = create("cell");
      setString(cell, "text", new Date(commit.getTimestamp()).toString());
      add(row, cell);
      cell = create("cell");
      Map userData = commit.getUserData();
      if (userData != null && !userData.isEmpty()) {
        setString(cell, "text", userData.toString());
      } else {
        setString(cell, "text", "--");
      }
      add(row, cell);
    }
  }
  
  public void showCommitFiles(Object commitTable) throws Exception {
    List commits = new ArrayList();
    Object[] rows = getSelectedItems(commitTable);
    if (rows == null || rows.length == 0) {
      showFiles(dir, commits);
      return;
    }
    for (int i = 0; i < rows.length; i++) {
      IndexCommit commit = (IndexCommit)getProperty(rows[i], "commit");
      if (commit != null) {
        commits.add(commit);
      }
    }
    showFiles(dir, commits);
  }

  private void showFiles(Directory dir, List commits) throws Exception {
    Object filesTable = find("filesTable");
    if (dir == null) {
      removeAll(filesTable);
      Object row = create("row");
      Object cell = create("cell");
      setString(cell, "text", "<not available>");
      setBoolean(cell, "enabled", false);
      add(row, cell);
      add(filesTable, row);
      return;
    }
    String[] physFiles = dir.listAll();
    List<String> files = new ArrayList();
    if (commits != null && commits.size() > 0) {
      for (int i = 0; i < commits.size(); i++) {
        IndexCommit commit = (IndexCommit)commits.get(i);
        files.addAll(commit.getFileNames());
      }
    } else {
      files.addAll(Arrays.asList(physFiles));
    }
    Collections.sort(files);
    List segs = getIndexFileNames(dir);
    List dels = getIndexDeletableNames(dir);
    removeAll(filesTable);
    for (int i = 0; i < files.size(); i++) {
      String fileName = files.get(i);
      String pathName;
      if (pName.endsWith(File.separator)) {
        pathName = pName;
      } else {
        pathName = pName + File.separator;
      }
      File file = new File(pathName + fileName);
      Object row = create("row");
      Object nameCell = create("cell");
      setString(nameCell, "text", fileName);
      add(row, nameCell);
      Object sizeCell = create("cell");
      setString(sizeCell, "text", Util.normalizeSize(file.length()));
      setChoice(sizeCell, "alignment", "right");
      add(row, sizeCell);
      Object unitCell = create("cell");
      setString(unitCell, "text", Util.normalizeUnit(file.length()));
      add(row, unitCell);
      boolean deletable = dels.contains(fileName.intern());
      String inuse = getFileFunction(fileName);
      Object delCell = create("cell");
      setString(delCell, "text", deletable ? "YES" : "-");
      add(row, delCell);
      Object inuseCell = create("cell");
      setString(inuseCell, "text", inuse);
      add(row, inuseCell);
      add(filesTable, row);
    }
  }
  
  private String getFileFunction(String file) {
    String res = IndexGate.getFileFunction(file);
    if (res == null) {
      res = "YES";
    }
    return res;
  }

  private void syncMRU(Object path) {
    removeAll(path);
    for (Iterator iter = Prefs.getMruList().iterator(); iter.hasNext();) {
      String element = (String) iter.next();
      Object choice = create("choice");
      setString(choice, "text", element);
      add(path, choice);
    }
  }

  /**
   * Update the list of top terms.
   * @param nTerms Thinlet widget containing the number of top terms to show
   */
  public void actionTopTerms(Object nTerms) {
    if (ir == null) {
      showStatus(MSG_NOINDEX);
      return;
    }
    String sndoc = getString(nTerms, "text");
    int nd = 50;
    try {
      nd = Integer.parseInt(sndoc);
    } catch (Exception e) {}
    final int ndoc = nd;
    Object[] fields = getSelectedItems(find("fList"));
    String[] flds = null;
    if (fields == null || fields.length == 0) {
      flds = idxFields;
    } else {
      flds = new String[fields.length];
      for (int i = 0; i < fields.length; i++) {
        flds[i] = (String) getProperty(fields[i], "fName");
      }
    }
    final String[] fflds = flds;
    SlowThread st = new SlowThread(this) {
      public void execute() {
        try {
          TermInfo[] tis = HighFreqTerms.getHighFreqTerms(ir, null, ndoc + 1, fflds);
          Object table = find("tTable");
          removeAll(table);
          if (tis == null || tis.length == 0) {
            Object row = create("row");
            Object cell = create("cell");
            add(row, cell);
            cell = create("cell");
            add(row, cell);
            cell = create("cell");
            add(row, cell);
            cell = create("cell");
            setBoolean(cell, "enabled", false);
            setString(cell, "text", "No Results");
            add(row, cell);
            add(table, row);
            return;
          }
          for (int i = 0; i < tis.length; i++) {
            Object row = create("row");
            add(table, row);
            putProperty(row, "term", tis[i].term);
            putProperty(row, "ti", tis[i]);
            Object cell = create("cell");
            setChoice(cell, "alignment", "right");
            setString(cell, "text", String.valueOf(i + 1));
            add(row, cell);
            cell = create("cell");
            setChoice(cell, "alignment", "right");
            setString(cell, "text", String.valueOf(tis[i].docFreq) + "  ");
            add(row, cell);
            cell = create("cell");
            setString(cell, "text", tis[i].term.field());
            add(row, cell);
            cell = create("cell");
            Decoder dec = decoders.get(tis[i].term.field());
            if (dec == null) dec = defDecoder;
            String s;
            try {
              s = dec.decodeTerm(tis[i].term.field(), tis[i].term.text());
            } catch (Throwable e) {
              s = tis[i].term.text();
              setColor(cell, "foreground", Color.RED);
            }
            setString(cell, "text", "  " + s);
            add(row, cell);
          }
        } catch (Exception e) {
          e.printStackTrace();
          errorMsg(e.getMessage());
        }
      }
    };
    if (slowAccess) {
      st.start();
    } else {
      st.execute();
    }
  }
  
  public void clipTopTerms(Object tTable) {
    Object[] rows = getItems(tTable);
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < rows.length; i++) {
      TermInfo ti = (Term…