/src/java/org/infoglue/cms/controllers/kernel/impl/simple/LuceneController.java

Large files files are truncated, but you can click here to view the full file

/* ===============================================================================
 *
 * Part of the InfoGlue Content Management Platform (www.infoglue.org)
 *
 * ===============================================================================
 *
 *  Copyright (C)
 * 
 * This program is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License version 2, as published by the
 * Free Software Foundation. See the file LICENSE.html for more information.
 * 
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY, including the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc. / 59 Temple
 * Place, Suite 330 / Boston, MA 02111-1307 / USA.
 *
 * ===============================================================================
 */

package org.infoglue.cms.controllers.kernel.impl.simple;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.Writer;
import java.nio.channels.OverlappingFileLockException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.Version;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.exolab.castor.jdo.Database;
import org.exolab.castor.jdo.OQLQuery;
import org.exolab.castor.jdo.QueryResults;
import org.infoglue.cms.entities.content.Content;
import org.infoglue.cms.entities.content.ContentCategory;
import org.infoglue.cms.entities.content.ContentVO;
import org.infoglue.cms.entities.content.ContentVersion;
import org.infoglue.cms.entities.content.ContentVersionVO;
import org.infoglue.cms.entities.content.DigitalAsset;
import org.infoglue.cms.entities.content.DigitalAssetVO;
import org.infoglue.cms.entities.content.SmallestContentVersionVO;
import org.infoglue.cms.entities.content.impl.simple.ContentImpl;
import org.infoglue.cms.entities.content.impl.simple.ContentVersionImpl;
import org.infoglue.cms.entities.content.impl.simple.DigitalAssetImpl;
import org.infoglue.cms.entities.content.impl.simple.MediumDigitalAssetImpl;
import org.infoglue.cms.entities.content.impl.simple.SmallestContentVersionImpl;
import org.infoglue.cms.entities.kernel.BaseEntityVO;
import org.infoglue.cms.entities.management.CategoryAttribute;
import org.infoglue.cms.entities.management.ContentTypeDefinitionVO;
import org.infoglue.cms.entities.management.LanguageVO;
import org.infoglue.cms.entities.structure.SiteNode;
import org.infoglue.cms.entities.structure.SiteNodeVO;
import org.infoglue.cms.entities.structure.SiteNodeVersion;
import org.infoglue.cms.entities.structure.SiteNodeVersionVO;
import org.infoglue.cms.entities.structure.impl.simple.SiteNodeImpl;
import org.infoglue.cms.entities.structure.impl.simple.SiteNodeVersionImpl;
import org.infoglue.cms.exception.Bug;
import org.infoglue.cms.exception.SystemException;
import org.infoglue.cms.util.CmsPropertyHandler;
import org.infoglue.cms.util.NotificationListener;
import org.infoglue.cms.util.NotificationMessage;
import org.infoglue.deliver.util.CacheController;
import org.infoglue.deliver.util.RequestAnalyser;
import org.infoglue.deliver.util.Timer;
import org.pdfbox.pdmodel.PDDocument;
import org.pdfbox.util.PDFTextStripper;

public class LuceneController extends BaseController implements NotificationListener
{
	private static Directory directory = null;
	private static IndexWriter writer = null;
	private static IndexReader indexReader = null;
	private static int reopened = 0;
	
    private final static Logger logger = Logger.getLogger(LuceneController.class.getName());
    private static int indexedDocumentsSinceLastOptimize = 0;
    private Integer lastCommitedContentVersionId = -1;
    
    private static Integer numberOfVersionToIndexInBatch = 1000;

    private static AtomicBoolean indexingInitialized = new AtomicBoolean(false);
    private static AtomicBoolean stopIndexing = new AtomicBoolean(false);
    private static AtomicBoolean deleteIndexOnStop = new AtomicBoolean(false);
    
	public static void setNumberOfVersionToIndexInBatch(Integer numberOfVersionToIndexInBatch) 
	{
		numberOfVersionToIndexInBatch = numberOfVersionToIndexInBatch;
	}

    public static void stopIndexing()
    {
    	stopIndexing.set(true);
    }
    
	/**
	 * Default Constructor	
	 */
	
	public static LuceneController getController()
	{
		return new LuceneController();
	}
	
	private static List<NotificationMessage> qeuedMessages = new ArrayList<NotificationMessage>();
	
	private StandardAnalyzer getStandardAnalyzer() throws Exception
	{
		return new StandardAnalyzer(Version.LUCENE_34);
	}
	
	private Directory getDirectory() throws Exception
	{
		if(LuceneController.directory != null)
			return directory;
		
		String index = CmsPropertyHandler.getContextDiskPath() + File.separator + "lucene" + File.separator + "index";

		index = index.replaceAll("//", "/");
		//System.out.println("index:" + index);
    	File INDEX_DIR = new File(index);
		directory = new NIOFSDirectory(INDEX_DIR);
		directory.setLockFactory(new SingleInstanceLockFactory());
		boolean indexExists = IndexReader.indexExists(directory);
		if(!indexExists)
		{
			createIndex(directory);
		}
		
		return directory;
	}

	private void createIndex(Directory directory) throws Exception
	{
		IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_34, getStandardAnalyzer());
		IndexWriter indexWriter = new IndexWriter(directory, config);
    	indexWriter.deleteDocuments(new Term("initializer", "true"));
		indexWriter.close(true);
	}
	
	private IndexWriter getIndexWriter() throws Exception
	{
		//Singleton returns
		if(writer != null)
			return writer;

		Timer t = new Timer();
		Directory directory = getDirectory();
    	StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_34);
		IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_34, analyzer);

		if(getIsIndexedLocked(true))
		{
			logger.warn("Directory is locked - leaving the messages in the qeuedMessages list...");
			throw new Exception("Lock not granted");
		}
		else
		{
			writer = new IndexWriter(directory, config);
	    	return writer;
		}
	}

	private IndexReader getIndexReader() throws Exception
	{
		if(indexReader == null)
		{
			indexReader = IndexReader.open(getDirectory(), true);
		}
		synchronized (indexReader)
		{
			if(!indexReader.isCurrent())
			{
				reopened++;
				indexReader.close();
				indexReader = IndexReader.open(getDirectory(), true);
				//indexReader = IndexReader.openIfChanged(indexReader, true);
				logger.info("reopened:" + reopened);
			}
		}

		return indexReader;
	}

	private IndexSearcher getIndexSearcher() throws Exception
	{
		return new IndexSearcher(getIndexReader());
	}
	
	private Boolean getIsIndexedLocked() throws Exception
	{
		return getIsIndexedLocked(false);
	}

	private Boolean getIsIndexedLocked(boolean returnIfFileLockException) throws Exception
	{
		Directory directory = getDirectory();
		try
		{
			return IndexWriter.isLocked(directory);
		}
		catch (OverlappingFileLockException e) 
		{
			return returnIfFileLockException;
		}
	}

	private void unlockIndex() throws Exception
	{
		Directory directory = getDirectory();
		IndexWriter.unlock(directory);
	}
	
	public Map<String,Object> getIndexInformation() throws Exception
	{
		Map<String,Object> info = new HashMap<String,Object>();
		
		try 
	    {
			Directory directory = getDirectory();
			
		    IndexReader reader = getIndexReader();
		    int maxDoc = reader.maxDoc();
		    int numDoc = reader.numDocs();
		    long lastModified = getIndexReader().lastModified(directory);

		    info.put("maxDoc", new Integer(maxDoc));
		    info.put("numDoc", new Integer(numDoc));
		    info.put("lastModified", new Date(lastModified));
		    info.put("lastCommitedContentVersionId", getLastCommitedContentVersionId());
		    
			List<LanguageVO> languageVOList = LanguageController.getController().getLanguageVOList();
			Iterator<LanguageVO> languageVOListIterator = languageVOList.iterator();
			outer:while(languageVOListIterator.hasNext())
			{
				LanguageVO languageVO = (LanguageVO)languageVOListIterator.next();
			    info.put("indexAllLastCommittedContentVersionId_" + languageVO.getId(), getIndexAllLastCommittedContentVersionId(languageVO.getId()));
			    info.put("indexAllLastCommittedMetaContentVersionId_" + languageVO.getId(), getIndexAllLastCommittedMetaContentVersionId(languageVO.getId()));
			}
			
		    //reader.close();
		    //directory.close();
	    } 
	    catch (Exception e) 
	    {
	    	logger.error("Error creating index:" + e.getMessage(), e);
	    	throw e;
	    }
	    
	    return info;
	}

	public Integer getIndexAllLastCommittedContentVersionId(Integer languageId) throws Exception
	{
		Integer indexAllLastCommittedContentVersionId = null;
		try 
	    {
			Document indexAllDocumentMetaData = getIndexAllStatusDocument();
			if(indexAllDocumentMetaData != null && indexAllDocumentMetaData.get("lastCommitedContentVersionId_" + languageId) != null && !indexAllDocumentMetaData.get("lastCommitedContentVersionId_" + languageId).equals("null"))
				indexAllLastCommittedContentVersionId = new Integer(indexAllDocumentMetaData.get("lastCommitedContentVersionId_" + languageId));
	    } 
	    catch (Exception e) 
	    {
	    	logger.error("Error creating index:" + e.getMessage(), e);
	    	throw e;
	    }
	    
	    return indexAllLastCommittedContentVersionId;
	}

	public Integer getIndexAllLastCommittedMetaContentVersionId(Integer languageId) throws Exception
	{
		Integer indexAllLastCommittedSiteNodeVersionId = null;
		try 
	    {
			Document indexAllDocumentMetaData = getIndexAllStatusDocument();
			if(indexAllDocumentMetaData != null && indexAllDocumentMetaData.get("lastCommitedMetaContentVersionId_" + languageId) != null && !indexAllDocumentMetaData.get("lastCommitedMetaContentVersionId_" + languageId).equals("null"))
 				indexAllLastCommittedSiteNodeVersionId = new Integer(indexAllDocumentMetaData.get("lastCommitedMetaContentVersionId_" + languageId));
	    } 
	    catch (Exception e) 
	    {
	    	logger.error("Error creating index:" + e.getMessage(), e);
	    	throw e;
	    }
	    
	    return indexAllLastCommittedSiteNodeVersionId;
	}

	public Document createStatusDocument(Integer lastCommitedContentVersionId) throws Exception
	{
		Document doc = new Document();

		doc.add(new Field("lastCommitedContentVersionId", "" + lastCommitedContentVersionId, Field.Store.YES, Field.Index.NOT_ANALYZED));
		doc.add(new Field("lastCommitedModifiedDate", "" + new Date().getTime(), Field.Store.YES, Field.Index.NOT_ANALYZED));
		doc.add(new Field("meta", new StringReader("lastCommitedContentVersionId")));

		return doc;
	}

	public Document getStatusDocument() throws Exception
	{
	    List<Document> docs = queryDocuments("meta", "lastCommitedContentVersionId", 5);
		logger.info(docs.size() + " total matching documents for 'lastCommitedContentVersionId'");
		return (docs != null && docs.size() > 0 ? docs.get(0) : null);
	}

	public Document getIndexAllStatusDocument() throws Exception
	{
	    List<Document> docs = queryDocuments(new Term("meta", "indexAllRunning"), 5);
		logger.info(docs.size() + " total matching documents for 'indexAllRunning'");
		return (docs != null && docs.size() > 0 ? docs.get(0) : null);
	}

	public Integer getLastCommitedContentVersionId() throws Exception
	{
		Integer lastCommitedContentVersionId = -1;
		
		Document doc = getStatusDocument();
		logger.info("STATUS doc:" + doc);
		if(doc != null)
		{
			String lastCommitedContentVersionIdString = doc.get("lastCommitedContentVersionId");
			
			logger.info("doc:" + doc);
			logger.info("lastCommitedContentVersionId:" + lastCommitedContentVersionIdString);
			
			lastCommitedContentVersionId = Integer.parseInt(lastCommitedContentVersionIdString);
		}
		
		return lastCommitedContentVersionId;
	}

	private void setLastCommitedContentVersionId(IndexWriter writer, Integer lastCommitedContentVersionId) throws Exception 
	{
		Integer prevLastCommitedContentVersionId = getLastCommitedContentVersionId();
		logger.info("prevLastCommitedContentVersionId:" + prevLastCommitedContentVersionId);
		logger.info("lastCommitedContentVersionId:" + lastCommitedContentVersionId);
		if(lastCommitedContentVersionId == -1 || prevLastCommitedContentVersionId > lastCommitedContentVersionId)
			return;
		
		logger.info("setLastCommitedContentVersionId:" + lastCommitedContentVersionId);
		Query query = new QueryParser(Version.LUCENE_34, "meta", getStandardAnalyzer()).parse("lastCommitedContentVersionId");
		writer.deleteDocuments(query);
		writer.addDocument(createStatusDocument(lastCommitedContentVersionId));
	}
	
	public Date getLastCommitedModifiedDate() throws Exception
	{
		Date lastCommitedModifiedDate = new Date(10000);
		
		Document doc = getStatusDocument();
		if(doc != null)
		{
			String lastCommitedModifiedDateString = doc.get("lastCommitedModifiedDate");
	
			logger.info("doc:" + doc);
			logger.info("lastCommitedModifiedDate:" + lastCommitedModifiedDateString);
			
			Date d = new Date();
			d.setTime(Long.parseLong(lastCommitedModifiedDateString));
			lastCommitedModifiedDate = d;
		}
		
		return lastCommitedModifiedDate;
	}

	private void registerIndexAllProcessOngoing(Integer lastCommitedContentVersionId, Integer lastCommitedSiteNodeVersionId, Integer languageId) throws Exception
	{
		//M�ste skrivas om f�r att uppdatera b�ttre....
		
		//Document doc = new Document();
		IndexWriter writer = getIndexWriter();

		IndexSearcher searcher = getIndexSearcher();
		Term term = new Term("meta", "indexAllRunning");
		TermQuery query = new TermQuery(term);
		
		//Query query = new QueryParser(Version.LUCENE_34, "meta", getStandardAnalyzer()).parse("indexAllRunning");
		TopDocs hits = searcher.search(query, 50);
		//System.out.println("hits:" + hits);
		//System.out.println("hits.scoreDocs.length:" + hits.scoreDocs.length);
		
		if(hits.scoreDocs.length > 1)
			System.out.println("Must be wrong - should only be one of these docs:" + hits.scoreDocs.length);
		
		if(hits.scoreDocs.length > 0)
		{
			for(ScoreDoc scoreDoc : hits.scoreDocs)
			{
				org.apache.lucene.document.Document docExisting = searcher.doc(scoreDoc.doc);
				//System.out.println("Updating doc...:" + docExisting);

				//System.out.println("lastCommitedContentVersionId:" + lastCommitedContentVersionId);
				//System.out.println("lastCommitedSiteNodeVersionId:" + lastCommitedSiteNodeVersionId);
				//System.out.println("languageId:" + languageId);
				if(lastCommitedContentVersionId != null && lastCommitedContentVersionId != -1)
				{
					docExisting.removeFields("lastCommitedContentVersionId_" + languageId);
					docExisting.add(new Field("lastCommitedContentVersionId_" + languageId, "" + lastCommitedContentVersionId, Field.Store.YES, Field.Index.NOT_ANALYZED));
				}
				
				if(lastCommitedSiteNodeVersionId != null && lastCommitedSiteNodeVersionId != -1)
				{
					docExisting.removeFields("lastCommitedMetaContentVersionId_" + languageId);
					docExisting.add(new Field("lastCommitedMetaContentVersionId_" + languageId, "" + lastCommitedSiteNodeVersionId, Field.Store.YES, Field.Index.NOT_ANALYZED));
				}

				docExisting.removeFields("lastCommitedModifiedDate");
				docExisting.add(new Field("lastCommitedModifiedDate", "" + new Date().getTime(), Field.Store.YES, Field.Index.NOT_ANALYZED));
				//docExisting.add(new Field("meta", new StringReader("indexAllRunning")));
				//docExisting.add(new Field("meta", "indexAllRunning", Field.Store.YES, Field.Index.NOT_ANALYZED));
					
				writer.updateDocument(term, docExisting);
				//System.out.println("Updating doc...:" + docExisting);
				//Term t = new Term("meta", "indexAllRunning");
				break;
			}
		}
		else
		{
			Document docExisting = new Document();
			//System.out.println("lastCommitedContentVersionId:" + lastCommitedContentVersionId);
			//System.out.println("lastCommitedSiteNodeVersionId:" + lastCommitedSiteNodeVersionId);
			//System.out.println("languageId:" + languageId);
			
			if(lastCommitedContentVersionId != null)
				docExisting.add(new Field("lastCommitedContentVersionId_" + languageId, "" + lastCommitedContentVersionId, Field.Store.YES, Field.Index.NOT_ANALYZED));
			if(lastCommitedSiteNodeVersionId != null)
				docExisting.add(new Field("lastCommitedMetaContentVersionId_" + languageId, "" + lastCommitedSiteNodeVersionId, Field.Store.YES, Field.Index.NOT_ANALYZED));
			docExisting.add(new Field("lastCommitedModifiedDate", "" + new Date().getTime(), Field.Store.YES, Field.Index.NOT_ANALYZED));
			//docExisting.add(new Field("meta", new StringReader("indexAllRunning")));
			docExisting.add(new Field("meta", "indexAllRunning", Field.Store.YES, Field.Index.NOT_ANALYZED));
			
			writer.addDocument(docExisting);
		}
		searcher.close();
		
		//Query query = new QueryParser(Version.LUCENE_34, "meta", getStandardAnalyzer()).parse("indexAllRunning");
		//writer.deleteDocuments(query);

		
		//writer.updateDocument(term, doc);
		//writer.addDocument(doc);
		//writer.close(true);
		writer.commit();
	}

	private void registerIndexAllProcessDone() throws Exception
	{
		IndexWriter writer = getIndexWriter();

		//Query query = new QueryParser(Version.LUCENE_34, "meta", getStandardAnalyzer()).parse("indexAllRunning");
		Term term = new Term("meta", "indexAllRunning");
		TermQuery query = new TermQuery(term);
		
		writer.deleteDocuments(query);		
		writer.commit();
	}

	public void clearIndex() throws Exception
	{
		if (indexingInitialized.compareAndSet(false, true)) 
		{
			logger.warn("Clearing index..");
			try
			{
				logger.info("NumDocs:" + getIndexReader().numDocs());
				IndexWriter writer = getIndexWriter();
				writer.deleteAll();
				//writer.close(true);
				writer.commit();

				logger.info("NumDocs after delete:" + getIndexReader().numDocs());
			}
			catch (Exception e) 
			{
				stopIndexing.set(true);
				deleteIndexOnStop.set(true);
				logger.error("Error clearing index:" + e.getMessage(), e);
			}
			finally
			{
				logger.info("Releasing indexing flag");
				this.indexingInitialized.set(false);
				stopIndexing.set(false);
			}
		}
		else
		{
			stopIndexing.set(true);
			deleteIndexOnStop.set(true);
			logger.error("Could not delete index while indexing. Queueing it....");
		}
	}

	public TopDocs query(String text, Integer numberOfHits) throws Exception 
	{
		return query("contents", text, numberOfHits);
	}

	public TopDocs query(String field, String text, Integer numberOfHits) throws Exception 
	{
		IndexSearcher searcher = getIndexSearcher();
		Query query = new QueryParser(Version.LUCENE_34, "contents", getStandardAnalyzer()).parse(text);
		TopDocs hits = searcher.search(query, numberOfHits);
		logger.info(hits.totalHits + " total matching documents for '" + text + "'");

		return hits;
	}

	public List<Document> queryDocuments(Term term, Integer numberOfHits) throws Exception 
	{
		IndexSearcher searcher = getIndexSearcher();
		Query query = new TermQuery(term);
		TopDocs hits = searcher.search(query, numberOfHits);
		logger.info(hits.totalHits + " total matching documents for '" + term.field() + ":" + term.text() + "'");
		List<Document> docs = new ArrayList<Document>();
		for(ScoreDoc scoreDoc : hits.scoreDocs)
		{
			org.apache.lucene.document.Document doc = searcher.doc(scoreDoc.doc);
			docs.add(doc);
		}
		searcher.close();
		return docs;
	}

	public List<Document> queryDocuments(String field, String text, Integer numberOfHits) throws Exception 
	{
		IndexSearcher searcher = getIndexSearcher();
		Query query = new QueryParser(Version.LUCENE_34, field, getStandardAnalyzer()).parse(text);
		logger.info("query:" + query);
		TopDocs hits = searcher.search(query, numberOfHits);
		logger.info(hits.totalHits + " total matching documents for '" + field + ":" + text + "'");
		List<Document> docs = new ArrayList<Document>();
		for(ScoreDoc scoreDoc : hits.scoreDocs)
		{
			org.apache.lucene.document.Document doc = searcher.doc(scoreDoc.doc);
			docs.add(doc);
		}
		searcher.close();
		return docs;
	}


	public TopDocs query(String[] fields, BooleanClause.Occur[] flags, String[] queries, Sort sort, Integer numberOfHits) throws Exception 
	{
		IndexSearcher searcher = getIndexSearcher();
		Query query = MultiFieldQueryParser.parse(Version.LUCENE_34, queries, fields, flags, getStandardAnalyzer());
		//Query query = new QueryParser(Version.LUCENE_34, "contents", getStandardAnalyzer()).parse(text);
		TopDocs hits = searcher.search(query, numberOfHits);
		logger.info(hits.totalHits + " total matching documents for '" + queries + "'");

		return hits;
	}

	public List<Document> queryDocuments(String[] fields, BooleanClause.Occur[] flags, String[] queries, Sort sort, Integer numberOfHits, Map searchMetaData) throws Exception 
	{
		IndexSearcher searcher = getIndexSearcher();
		Query query = MultiFieldQueryParser.parse(Version.LUCENE_34, queries, fields, flags, getStandardAnalyzer());
		logger.info("query:" + query);
		
		//Query query = new QueryParser(Version.LUCENE_34, "contents", getStandardAnalyzer()).parse(text);
		TopDocs hits = searcher.search(query, numberOfHits);
		searchMetaData.put("totalHits", hits.totalHits);
		logger.info(hits.totalHits + " total matching documents for '" + query + "'");
		//System.out.println(hits.totalHits + " total matching documents for '" + queries + "'");
		List<Document> docs = new ArrayList<Document>();
		for(ScoreDoc scoreDoc : hits.scoreDocs)
		{
			org.apache.lucene.document.Document doc = searcher.doc(scoreDoc.doc);
			docs.add(doc);
		}
		searcher.close();
		return docs;
	}
	
	private void query(IndexSearcher searcher, Analyzer analyzer, String text) throws Exception 
	{
		Query query = new QueryParser(Version.LUCENE_34, "contents", analyzer).parse(text);
		TopDocs hits = searcher.search(query, 50);
		logger.info(hits.totalHits + " total matching documents for '" + text + "'");
		for(ScoreDoc scoreDoc : hits.scoreDocs)
		{
			org.apache.lucene.document.Document doc = searcher.doc(scoreDoc.doc);
			String cvId = doc.get("contentVersionId");
			logger.info("cvId: " + cvId);
		}
	}

	public boolean indexAll() throws Exception
	{
		if(!CmsPropertyHandler.getInternalSearchEngine().equalsIgnoreCase("lucene"))
			return false;
		
		logger.warn("INDEXING ALL - correct: " + indexingInitialized + "/" + deleteIndexOnStop + "/" + stopIndexing + "?");
		Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
		
		if(deleteIndexOnStop.get())
		{
			clearIndex();
			deleteIndexOnStop.set(false);
			stopIndexing.set(false);
		}
		else
		{
			stopIndexing.set(false);
		}
		
		logger.warn("Resetting stopIndexing to false....");
		logger.warn("------------------------------Got indexAll directive....");
		if (indexingInitialized.compareAndSet(false, true)) 
		{
			//createTestIndex();
			//indexingInitialized.set(false);
			//if(true)
			//	return true;

			try
			{
				Timer t = new Timer();
				Timer t2 = new Timer();
		
				//Indexing all normal contents now
				logger.info("Indexing all normal contents: " + CmsPropertyHandler.getContextDiskPath());
				List<LanguageVO> languageVOList = LanguageController.getController().getLanguageVOList();
				Iterator<LanguageVO> languageVOListIterator = languageVOList.iterator();
				
				outer:while(languageVOListIterator.hasNext())
				{
					LanguageVO languageVO = (LanguageVO)languageVOListIterator.next();
					logger.info("Getting notification messages for " + languageVO.getName());

					Integer previousIndexAllLastContentVersionId = getIndexAllLastCommittedContentVersionId(languageVO.getId());
					int startID = 0;
					if(previousIndexAllLastContentVersionId != null)
						startID = previousIndexAllLastContentVersionId;
					
					logger.info("Starting from " + startID);
					int newLastContentVersionId = getContentNotificationMessages(languageVO, startID);
					logger.info("newLastContentVersionId: " + newLastContentVersionId + " on " + languageVO.getName());
					
					registerIndexAllProcessOngoing(newLastContentVersionId, null, languageVO.getId());
					//previousIndexAllLastContentVersionId = newLastContentVersionId;
					RequestAnalyser.getRequestAnalyser().registerComponentStatistics("getNotificationMessages", t.getElapsedTime());
					logger.info("newLastContentVersionId " + newLastContentVersionId);
					while(newLastContentVersionId != -1)
					{
						logger.info("stopIndexing.get():" + stopIndexing.get());
						if(stopIndexing.get())
							break outer;
						
						Thread.sleep(5000);
						newLastContentVersionId = getContentNotificationMessages(languageVO, newLastContentVersionId);
						logger.info("newLastContentVersionId: " + newLastContentVersionId + " on " + languageVO.getName());
						registerIndexAllProcessOngoing(newLastContentVersionId, null, languageVO.getId());
						//previousIndexAllLastContentVersionId = newLastContentVersionId;
						RequestAnalyser.getRequestAnalyser().registerComponentStatistics("getNotificationMessages 2", t.getElapsedTime());
						logger.info("newLastContentVersionId " + newLastContentVersionId);
					}
				}
				
				languageVOList = LanguageController.getController().getLanguageVOList();
				languageVOListIterator = languageVOList.iterator();
				outer:while(languageVOListIterator.hasNext())
				{
					LanguageVO languageVO = (LanguageVO)languageVOListIterator.next();
					logger.info("languageVO from " + languageVO);
					
					List<NotificationMessage> notificationMessages = new ArrayList<NotificationMessage>();
					
					Integer previousIndexAllLastMetaContentVersionId = getIndexAllLastCommittedMetaContentVersionId(languageVO.getId());
					logger.info("previousIndexAllLastMetaContentVersionId: " + previousIndexAllLastMetaContentVersionId);
					int startID = 0;
					if(previousIndexAllLastMetaContentVersionId != null)
						startID = previousIndexAllLastMetaContentVersionId;

					logger.info("Starting from " + startID);
					int newLastMetaContentVersionId = getPageNotificationMessages(notificationMessages, languageVO, startID);
					logger.info("newLastSiteNodeVersionId " + newLastMetaContentVersionId + " on " + languageVO.getName());
					logger.info("notificationMessages: " + notificationMessages.size());
					registerIndexAllProcessOngoing(null, newLastMetaContentVersionId, languageVO.getId());
					//previousIndexAllLastMetaContentVersionId = newLastMetaContentVersionId;
					RequestAnalyser.getRequestAnalyser().registerComponentStatistics("getNotificationMessagesForStructure", t.getElapsedTime());
					logger.info("newLastMetaContentVersionId " + newLastMetaContentVersionId);
					while(newLastMetaContentVersionId != -1)
					{
						logger.info("stopIndexing.get():" + stopIndexing.get());
						if(stopIndexing.get())
							break outer;

						Thread.sleep(5000);
						newLastMetaContentVersionId = getPageNotificationMessages(notificationMessages, languageVO, newLastMetaContentVersionId);
						logger.info("newLastMetaContentVersionId " + newLastMetaContentVersionId + " on " + languageVO.getName());
						logger.info("notificationMessages: " + notificationMessages.size());
						registerIndexAllProcessOngoing(null, newLastMetaContentVersionId, languageVO.getId());
						//previousIndexAllLastMetaContentVersionId = newLastMetaContentVersionId;
						RequestAnalyser.getRequestAnalyser().registerComponentStatistics("getNotificationMessages 2", t.getElapsedTime());
						logger.info("newLastMetaContentVersionId " + newLastMetaContentVersionId);
					}
				}
				
				registerIndexAllProcessDone();
				
				t2.printElapsedTime("All indexing took");
			}
			catch (Exception e) 
			{
				logger.error("Error indexing notifications:" + e.getMessage(), e);
			}
			finally
			{
				logger.error("Releasing indexing flag");
				this.indexingInitialized.set(false);
			}
		}
		else
		{
			logger.warn("-------------------: Allready running index all...");
			return false;
		}
		
		return true;
	}
	
	
	private void createTestIndex() 
	{
		System.out.println("STARTING TEST");
		try
		{
			clearIndex();

			IndexWriter writer = getIndexWriter();
	
			for(int i=0; i<10000; i++)
			{
				// make a new, empty document
				Document doc = new Document();
		
				doc.add(new NumericField("publishDateTime", Field.Store.YES, true).setLongValue(23423423423L));
				doc.add(new NumericField("modificationDateTime", Field.Store.YES, true).setLongValue(23423423423L));
				doc.add(new Field("modified", DateTools.timeToString(23423423423L, DateTools.Resolution.MINUTE), Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("contentVersionId", "324234234", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("contentId", "324234234", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("contentTypeDefinitionId", "344", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("languageId", "33", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("repositoryId", "22", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("lastModifier", "Mattias Bogeblad", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("stateId", "3", Field.Store.YES, Field.Index.NOT_ANALYZED));
				doc.add(new Field("isAsset", "false", Field.Store.YES, Field.Index.NOT_ANALYZED));
				
				doc.add(new Field("contents", new StringReader(i + " fwe foweif oiwejfoijweoifiweuhfi uehwiufh weiuhfiuwehfiew iufiuwehfi ewiufh iuwehfiuehwiufiweuhfiu ehwifhw eifew efiwehfiuwe" +
						"ff wehfiuehwiufiuwehfiuehw iufhwei uhfiehwiufweiuhf iwefihw eifiuwe ifhwe ifihew iufi weuhfiuwe" +
						"dfbsdjfsjdjfjksdf s f jdsjkfs dkjfh ksdfk sdkfhkds fksd " +
						"fjsd fsdhf uiweo	p fiieowhf iehwiufiewhfiewfhw efn  ewfowe ifioewf owehfowe")));
		
				doc.add(new Field("uid", "" + i, Field.Store.NO, Field.Index.NOT_ANALYZED));
		
		    	writer.addDocument(doc);
		    	
		    	if(i == 1000 || i == 2000 ||i == 3000 ||i == 4000 ||i == 5000 ||i == 6000 ||i == 7000 ||i == 8000 ||i == 9000)
		    	{
		    		//writer.optimize();
		    		//writer.optimize(true);
		    		logger.info("Sleeping...:" + getIndexInformation().get("numDoc"));
		    		Thread.sleep(5000);
		    	}
			}
			
			//writer.close(true);
			writer.commit();

		}
		catch (Exception e) 
		{
			e.printStackTrace();
		}
	}

	/**
	 * This method gets called when a new notification has come. 
	 * It then iterates through the listeners and notifies them.
	 */
	public void addNotificationMessage(NotificationMessage notificationMessage)
	{
		if(notificationMessage.getClassName().equals(ContentImpl.class.getName()) ||
		   notificationMessage.getClassName().equals(ContentVersionImpl.class.getName()) ||
		   notificationMessage.getClassName().equals(SiteNodeImpl.class.getName()) ||
		   notificationMessage.getClassName().equals(SiteNodeVersionImpl.class.getName()) ||
		   notificationMessage.getClassName().equals(DigitalAssetImpl.class.getName()) ||
		   notificationMessage.getClassName().equals(MediumDigitalAssetImpl.class.getName()))
		{
			if(qeuedMessages.size() == 1000)
			{
				logger.warn("qeuedMessages went over 1000 - seems wrong");
				Thread.dumpStack();
			}
				
			synchronized (qeuedMessages)
			{
				qeuedMessages.add(notificationMessage);			
			}
		}
		else
		{
			logger.info("Skipping indexing:" + notificationMessage.getClassName());
		}
	}	

	/**
	 * This method gets called when a new NotificationMessage is available.
	 * The writer just calls the transactionHistoryController which stores it.
	 */

	public void notify(NotificationMessage notificationMessage)
	{
		try
		{
			if(logger.isInfoEnabled())
				logger.info("Indexing:" + notificationMessage.getName() + ":" + notificationMessage.getType() + ":" + notificationMessage.getObjectId() + ":" + notificationMessage.getObjectName());
			addNotificationMessage(notificationMessage);
		}
		catch(Exception e)
		{
			logger.error("Error notifying: " + e.getMessage());
		}
	}

	public void process() throws Exception
	{
		logger.info("Process inside LuceneController");
		notifyListeners(false, true);
	}
	
	public void notifyListeners(boolean forceVersionIndexing, boolean checkForIndexingJobs) throws IOException, Exception
	{
		if(!CmsPropertyHandler.getInternalSearchEngine().equalsIgnoreCase("lucene") || CmsPropertyHandler.getContextDiskPath().contains("@deploy.dir"))
			return;

		boolean initDoneLocally = false;
		boolean finishDoneLocally = false;

		logger.info("------------------------------->notifyListeners before check in " + CmsPropertyHandler.getContextRootPath() + "/" + deleteIndexOnStop.get() + "/" + stopIndexing.get());

		if(deleteIndexOnStop.get())
		{
			clearIndex();
			deleteIndexOnStop.set(false);
			stopIndexing.set(false);
		}
		else
		{
			stopIndexing.set(false);
		}
		
		if (!checkForIndexingJobs || indexingInitialized.compareAndSet(false, true)) 
		{
			if(checkForIndexingJobs)
				initDoneLocally = true;
			
			List<NotificationMessage> internalMessageList = new ArrayList<NotificationMessage>();
			synchronized (qeuedMessages)
			{
				//logger.error("internalMessageList: " + internalMessageList.size() + "/" + qeuedMessages.size());
				internalMessageList.addAll(qeuedMessages);
				//logger.error("internalMessageList: " + internalMessageList.size() + "/" + qeuedMessages.size());
				qeuedMessages.clear();
				//logger.error("internalMessageList: " + internalMessageList.size() + "/" + qeuedMessages.size());
			}

			//Should implement equals on NotificationMessage later
			List<NotificationMessage> baseEntitiesToIndexMessageList = new ArrayList<NotificationMessage>();
			
			List<String> existingSignatures = new ArrayList<String>();
			logger.info("Before AAAAA:" + internalMessageList.size() + ":" + existingSignatures.size());
			Iterator<NotificationMessage> cleanupInternalMessageListIterator = internalMessageList.iterator();
			while(cleanupInternalMessageListIterator.hasNext())
			{
				NotificationMessage notificationMessage = cleanupInternalMessageListIterator.next();
				logger.info("Indexing........:" + notificationMessage.getClassName());

				if(notificationMessage.getClassName().equals(ContentImpl.class.getName()) || notificationMessage.getClassName().equals(Content.class.getName()))
				{
					ContentVO contentVO = ContentController.getContentController().getContentVOWithId((Integer)notificationMessage.getObjectId());

					ContentTypeDefinitionVO ctdVO = null;
					try
					{
						ctdVO = ContentTypeDefinitionController.getController().getContentTypeDefinitionVOWithId(contentVO.getContentTypeDefinitionId());
					}
					catch (SystemException sex)
					{
						logger.warn("Failed to get the content type definition for content with Id: " + contentVO.getContentId() + ". The content will not be indexed. Message: " + sex.getMessage());
						logger.info("Failed to get the content type definition for content with Id: " + contentVO.getContentId(), sex);
					}
					if(ctdVO != null && ctdVO.getName().equals("Meta info"))
					{
						SiteNodeVO siteNodeVO = SiteNodeController.getController().getSiteNodeVOWithMetaInfoContentId(contentVO.getContentId());
						NotificationMessage newNotificationMessage = new NotificationMessage("" + siteNodeVO.getName(), SiteNodeImpl.class.getName(), "SYSTEM", notificationMessage.getType(), siteNodeVO.getId(), "" + siteNodeVO.getName());
						String key = "" + newNotificationMessage.getClassName() + "_" + newNotificationMessage.getObjectId() + "_"  + "_" + newNotificationMessage.getType();
						if(!existingSignatures.contains(key))
						{
							logger.info("++++++++++++++Got an META PAGE notification - just adding it AS A PAGE instead: " + newNotificationMessage.getObjectId());								
							baseEntitiesToIndexMessageList.add(newNotificationMessage);
							existingSignatures.add(key);
						}
						else
						{
							logger.info("++++++++++++++Skipping Content notification - duplicate existed: " + notificationMessage.getObjectId());
						}

					}
					else
					{
						String key = "" + notificationMessage.getClassName() + "_" + notificationMessage.getObjectId() + "_"  + "_" + notificationMessage.getType();
						if(!existingSignatures.contains(key))
						{
							logger.info("++++++++++++++Got an Content notification - just adding it: " + notificationMessage.getObjectId());
							baseEntitiesToIndexMessageList.add(notificationMessage);
							existingSignatures.add(key);
						}
						else
						{
							logger.info("++++++++++++++Skipping Content notification - duplicate existed: " + notificationMessage.getObjectId());
						}
					}
				}
				else if(notificationMessage.getClassName().equals(ContentVersionImpl.class.getName()) || notificationMessage.getClassName().equals(ContentVersion.class.getName()))
				{
					logger.info("++++++++++++++Got an ContentVersion notification - focus on content: " + notificationMessage.getObjectId());
					ContentVersionVO contentVersionVO = ContentVersionController.getContentVersionController().getContentVersionVOWithId((Integer)notificationMessage.getObjectId());
					ContentVO contentVO = ContentController.getContentController().getContentVOWithId(contentVersionVO.getContentId());

					if(contentVO.getContentTypeDefinitionId() != null)
					{
						ContentTypeDefinitionVO ctdVO = null;
						try
						{
							ctdVO = ContentTypeDefinitionController.getController().getContentTypeDefinitionVOWithId(contentVO.getContentTypeDefinitionId());
						}
						catch (SystemException sex)
						{
							logger.warn("Failed to get the content type definition for content with Id: " + contentVO.getContentId() + ". The content version will not be indexed. Message: " + sex.getMessage());
							logger.info("Failed to get the content type definition for content with Id: " + contentVO.getContentId(), sex);
						}
						if(ctdVO != null && ctdVO.getName().equals("Meta info"))
						{
							SiteNodeVO siteNodeVO = SiteNodeController.getController().getSiteNodeVOWithMetaInfoContentId(contentVO.getContentId());

							if (siteNodeVO == null)
							{
								logger.warn("Got meta info notification but could not find a page for the Content-id. Content.id: " + contentVO.getContentId());
							}
							else
							{
								NotificationMessage newNotificationMessage = new NotificationMessage("" + siteNodeVO.getName(), SiteNodeImpl.class.getName(), "SYSTEM", notificationMessage.getType(), siteNodeVO.getId(), "" + siteNodeVO.getName());
								String key = "" + newNotificationMessage.getClassName() + "_" + newNotificationMessage.getObjectId() + "_"  + newNotificationMessage.getType();
								if(!existingSignatures.contains(key))
								{
									logger.info("++++++++++++++Got an META PAGE notification - just adding it AS A PAGE instead: " + newNotificationMessage.getObjectId());
									baseEntitiesToIndexMessageList.add(newNotificationMessage);
									existingSignatures.add(key);
								}
								else
								{
									logger.info("++++++++++++++Skipping Content notification - duplicate existed: " + notificationMessage.getObjectId());
								}
							}
						}
						else
						{
							NotificationMessage newNotificationMessage = new NotificationMessage("" + contentVersionVO.getContentName(), ContentImpl.class.getName(), "SYSTEM", notificationMessage.getType(), contentVersionVO.getContentId(), "" + contentVersionVO.getContentName());
							String key = "" + newNotificationMessage.getClassName() + "_" + newNotificationMessage.getObjectId() + "_" + newNotificationMessage.getType();
							if(!existingSignatures.contains(key))
							{
								logger.info("++++++++++++++Got an Content notification - just adding it: " + newNotificationMessage.getObjectId());
								baseEntitiesToIndexMessageList.add(newNotificationMessage);
								existingSignatures.add(key);
							}
							else
							{
								logger.info("++++++++++++++Skipping Content notification - duplicate existed: " + notificationMessage.getObjectId());
							}
						}
					}
				}
				else if(notificationMessage.getClassName().equals(DigitalAssetImpl.class.getName()) || 
						notificationMessage.getClassName().equals(MediumDigitalAssetImpl.class.getName()) || 
						notificationMessage.getClassName().equals(DigitalAsset.class.getName()) || 
						notificationMessage.getClassName().equals(SiteNodeImpl.class.getName()) || 
						notificationMessage.getClassName().equals(SiteNode.class.getName()) || 
						notificationMessage.getClassName().equals(SiteNodeVersionImpl.class.getName()) || 
						notificationMessage.getClassName().equals(SiteNodeVersion.class.getName()))
				{
					logger.info("notificationMessage.getClassName():" + notificationMessage.getClassName());
					String key = "" + notificationMessage.getClassName() + "_" + notificationMessage.getObjectId() + "_"  + "_" + notificationMessage.getType();

					if(notificationMessage.getClassName().equals(SiteNodeVersionImpl.class.getName()) || notificationMessage.getClassName().equals(SiteNodeVersion.class.getName()))
					{
						logger.info("PPPPPPPPPPPPPPPPPPPPPPPPPP:" + notificationMessage.getObjectId());
						SiteNodeVersionVO siteNodeVersionVO = SiteNodeVersionController.getController().getSiteNodeVersionVOWithId((Integer)notificationMessage.getObjectId());
						SiteNodeVO siteNodeVO = SiteNodeController.getController().getSiteNodeVOWithId(siteNodeVersionVO.getSiteNodeId());
						NotificationMessage newNotificationMessage = new NotificationMessage("" + siteNodeVO.getName(), SiteNodeImpl.class.getName(), "SYSTEM", notificationMessage.getType(), siteNodeVO.getId(), "" + siteNodeVO.getName());
						key = "" + newNotificationMessage.getClassName() + "_" + newNotificationMessage.getObjectId() + "_" + newNotificationMessage.getType();
					
						if(!existingSignatures.contains(key))
						{
							logger.info("++++++++++++++Got an SiteNodeVersionImpl notification - just adding it as SiteNodeImpl: " + newNotificationMessage.getClassName() + ":" + newNotificationMessage.getObjectId());
							baseEntitiesToIndexMessageList.add(newNotificationMessage);
							existingSignatures.add(key);
						}
						else
						{
							logger.info("++++++++++++++Skipping notification - duplicate existed: " + notificationMessage.getClassName() + ":" + notificationMessage.getObjectId());
						}
					}
					else if(notificationMessage.getClassName().equals(SiteNodeImpl.class.getName()) || notificationMessage.getClassName().equals(SiteNode.class.getName()))
					{
						if(!existingSignatures.contains(key))
						{
							logger.info("++++++++++++++Got an Page notification - just adding it: " + notificationMessage.getClassName() + ":" + notificationMessage.getObjectId());
							baseEntitiesToIndexMessageList.add(notificationMessage);
							existingSignatures.add(key);
						}
						else
						{
							logger.info("++++++++++++++Skipping notification - duplicate existed: " + notificationMessage.getClassName() + ":" + notificationMessage.getObjectId());
						}
					}
					else
					{
						NotificationMessage newNotificationMessage = new NotificationMessage("" + notificationMessage.getName(), DigitalAssetImpl.class.getName(), "SYSTEM", notificationMessage.getType(), notificationMessage.getObjectId(), "" + notificationMessage.getName());
						key = "" + newNotificationMessage.getClassName() + "_" + newNotificationMessage.getObjectId() + "_"  + "_" + newNotificationMessage.getType();

						if(!existingSignatures.contains(key))
						{
							logger.info("++++++++++++++Got an Content notification - just adding it: " + notificationMessage.getClassName() + ":" + notificationMessage.getObjectId());
							baseEntitiesToIndexMessageList.add(newNotificationMessage);
							existingSignatures.add(key);
						}
						else
						{
							logger.info("++++++++++++++Skipping notification - duplicate existed: " + notificationMessage.getClassName() + ":" + notificationMessage.getObjectId());
						}
					}
				}
			}
			internalMessageList = baseEntitiesToIndexMessageList;
			logger.info("After in [" + CmsPropertyHandler.getContextRootPath() + "]:" + internalMessageList.size() + ":" + existingSignatures.size());
			
			try
			{
				logger.info("notifyListeners actually running");
				if(getIsIndexedLocked())
				{
					logger.warn("The index should not be locked as no indexing is registered to be carried out. Lets unlock it as it may be the result of a crash.");
					unlockIndex();
				}
				
				//logger.error("Starting indexin of " + qeuedMessages.size());
				Timer t = new Timer();
				
				IndexWriter writer = getIndexWriter();
				//t.printElapsedTime("Creating writer took");
	

				Database db = CastorDatabaseService.getDatabase();
		
				beginTransaction(db);
				
				try
				{			
					int numberOfMessages = internalMessageList.size();
					Iterator internalMessageListIterator = internalMessageList.iterator();
					while(internalMessageListIterator.hasNext())
					{
						NotificationMessage notificationMessage = (NotificationMessage)internalMessageListIterator.next();
						try 
						{
							if(logger.isInfoEnabled())
								logger.info("Starting indexin of " + notificationMessage);
							indexInformation(notificationMessage, writer, internalMessageList, forceVersionIndexing, db);
							internalMessageListIterator.remove();
						} 
						catch (Exception e) 
						{
							e.printStackTrace();
						}
					}	
					//t.printElapsedTime("Indexing " + numberOfMessages + " documents took");
					//Map<String,String> commitUserData = new HashMap<String,String>();
					//internalMessageList.clear();
					//writer.commit(commitUserData);
					logger.info("##############lastCommitedContentVersionId before close:" + lastCommitedContentVersionId);
					if(lastCommitedContentVersionId > -1)
					{
						Integer previousLastCommittedContentVersionId = getLastCommitedContentVersionId();
						logger.info("##############previousLastCommittedContentVersionId before close:" + previousLastCommittedContentVersionId);
						if(previousLastCommittedContentVersionId < lastCommitedContentVersionId)
						{
							try 
							{
								logger.info("*************ADDING status doc " + lastCommitedContentVersionId + "**************");
								setLastCommitedContentVersionId(writer, lastCommitedContentVersionId);
							} 
							catch (Exception e) 
							{
								logger.error("*************ERROR: ADDING status doc**************", e);
							}			
						}
						else
						{
							logger.warn("The content version was not a higher number than what was allready indexed - lets not add status....");
						}
					}
					
					commitTransaction(db);
				}
				catch(Exception e)
				{
					logger.error("An error occurred so we should not complete the transaction:" + e.getMessage(), e);
					rollbackTransaction(db);
				}
				finally
				{
					writer.commit();
					//writer.close(true);
				}
				
				logger.info("OOOOOOOOOOOOOO:" + getLastCommitedContentVersionId());
			}
			catch (Exception e) 
			{
				logger.error("Error indexing notifications:" + e.getMessage());
				logger.warn("Error indexing notifications:" + e.getMessage(), e);
			}
			finally
			{
				logger.info("Releasing indexing flag");
				try
				{
					if(internalMessageList.size() > 0)
					{
						synchronized (qeuedMessages)
						{
							logger.info("Returning internalMessageList:" + internalMessageList.size() + " to qeuedMessages as some failed.");
							qeuedMessages.addAll(internalMessageList);
							internalMessageList.clear();
						}
					}
				}
				catch (Exception e) 
				{
					e.printStackTrace();
				}
				
				if(checkForIndexingJobs)
				{
					this.indexingInitialized.set(false);
					finishDoneLocally = true;
				}
			}
			
			if(initDoneLocally && !finishDoneLocally)
				logger.error("EEEEEEEEEEEEEEERRRRRRRRRRRRRRROOOOOOOOOOOORRRRRRRR aaaaaaa");

			logger.info("internalMessageList 1:" + internalMessageList.size() + " / " + qeuedMessages.size());
		}
		else
		{
			logger.info("------------------------------->Indexing job allready running... skipping in " + CmsPropertyHandler.getContextRootPath());
		}
		logger.info("queued messages 1:" + qeuedMessages.size());
	}
	

	public void index() throws Exception
	{
		if(!CmsPropertyHandler.getInternalSearchEngine().equalsIgnoreCase("lucene"))
			return;
		
		logger.info("Start index: " + CmsPropertyHandle…