/Mosaic-Catalogue-Connectors/src/main/java/com/augiq/external/source/nosql/MongoDBScanner.java

package com.augiq.external.source.nosql;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.bson.BsonArray;
import org.bson.BsonDocument;
import org.bson.BsonType;
import org.bson.BsonValue;
import org.bson.Document;
import org.bson.codecs.configuration.CodecRegistry;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;

import com.augiq.external.source.scanning.Scanner;
import com.augmentiq.constant.maxiq.LoggerConstants;
import com.augmentiq.maxiq.base.dao.sql.operation.configuration.DateUtils;
import com.augmentiq.maxiq.base.dao.sql.operation.configuration.ParamUtils;
import com.augmentiq.maxiq.constant.configuration.enums.ConnectorType;
import com.augmentiq.maxiq.constant.configuration.enums.DataSourceType;
import com.augmentiq.maxiq.constant.configuration.enums.FieldDataTypes;
import com.augmentiq.maxiq.constant.configuration.generic.GenericConstants;
import com.augmentiq.maxiq.constant.external.source.task.status.enums.TaskStatusEnums;
import com.augmentiq.maxiq.entity.model.configuration.bean.DataSource;
import com.augmentiq.maxiq.entity.model.configuration.bean.FieldMapping;
import com.augmentiq.maxiq.external.source.connection.message.constants.TestConnectionEnums;
import com.augmentiq.maxiq.model.connector.datanode.DataNode;
import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionConfig;
import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionSources;
import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionSubSources;
import com.augmentiq.maxiq.repository.connector.external.connectiondao.ConnectorListingDAO;
import com.augmentiq.maxiq.util.component.configuration.util.ConfigurationCreateUtil;
import com.augmentiq.maxiq.util.external.source.rdbms.connection.lookup.directory.ConnectionLookUpDirectory;
import com.augumentiq.maxiq.configuration.dao.DataRepositoryDao;
import com.augumentiq.maxiq.configuration.dao.DataSourceInstanceDao;
import com.google.gson.Gson;
import com.mongodb.BasicDBObject;
import com.mongodb.CommandResult;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoTimeoutException;
import com.mongodb.client.FindIterable;
import com.mongodb.client.ListIndexesIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.MongoIterable;
import com.mongodb.gridfs.GridFS;
import com.mongodb.gridfs.GridFSDBFile;
import com.sforce.ws.wsdl.Port;

import ch.epfl.lamp.fjbg.JConstantPool.FieldOrMethodRefEntry;

public class MongoDBScanner extends NoSQLScanner {

	@Autowired
	private ConnectorListingDAO noSqlConnectorListingDAO;

	@Autowired
	private ConfigurationCreateUtil createUtil;

	private ConnectionConfig noSqlConnectionConfig;

	private String subSourceType;

	private static final Logger logger = LoggerFactory.getLogger(MongoDBScanner.class);

	public String testConnection(Map<String, Object> args) {
		Connection connection = null;
		ConnectionConfig connectionConfig = (ConnectionConfig) args.get("connectionConfig");
		this.subSourceType = args.get("subSourceType").toString();
		String message = TestConnectionEnums.SUCCESS.name();
		message = establishedConnection(connectionConfig);
		return message;
	}

	public DataNode scan(Map<String, Object> args) throws Exception {
		ConnectionSources connectionSources = (ConnectionSources) args.get("ConnectionSources");
		ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
		noSqlConnectionConfig = getConnectionConfig(connectionSources);
		DataNode node = getMongoDBCollections();
		Gson gson = new Gson();
		System.out.println(gson.toJson(node));
		return node;
	}

	public Map<String, List<String>> publishDataSources(Map<String, Object> args) throws Exception {
		Map<String, List<String>> existingDSNotifier = new HashMap<String, List<String>>();
		existingDSNotifier.put(TaskStatusEnums.SUCCESS.name(), new ArrayList<String>());
		args.put("existingDataSourceNotifier", existingDSNotifier);
		DataNode dataNode = new DataNode();
		ConnectionSources connectionSources = objectParserHandler((Object) args.get("connectionSources"),
				ConnectionSources.class);
		ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
		ConnectionConfig connectionConfig = connectionSubSources.getConnectionConfig().get(0);
		DataNode dataNodefromUi = objectParserHandler((Object) args.get("dataNode"), DataNode.class);
		DataNode filteredNodes = dataNode.getSelectedNodes(dataNodefromUi);
		Collection<DataNode> nodesToPublish = filterNodesToPublish(filteredNodes.getChildren());
		Collection<Map<String, List<String>>> dbWithCollectionAsChild = getDbWithCollectionList(filteredNodes);
		Iterator<Map<String, List<String>>> dbWithCollectionItr = dbWithCollectionAsChild.iterator();
		Boolean doSampling = (Boolean) args.get("doSampling");
		Integer sampleRowCount = (Integer) args.get("sampleRowCount");
		while (dbWithCollectionItr.hasNext()) {
			List<FieldMapping> fieldMappings = null;
			String databaseName = null;
			List<String> currentCollectionAndViewsList = null;
			Map<String, List<String>> currentDbWithCollectiondetails = dbWithCollectionItr.next();
			Set<Entry<String, List<String>>> set = currentDbWithCollectiondetails.entrySet();
			Iterator<Map.Entry<String, List<String>>> entryItr = set.iterator();
			while (entryItr.hasNext()) {
				Map.Entry<String, List<String>> entry = entryItr.next();
				databaseName = entry.getKey();
				currentCollectionAndViewsList = entry.getValue();
				Iterator<String> iterator = currentCollectionAndViewsList.iterator();
				while (iterator.hasNext()) {
					Connection connection = null;
					this.subSourceType = StringUtils.upperCase(connectionSubSources.getSubConnectionType());
					MongoClient client = getMongoClient();
					String tableName = iterator.next();
					args.put(GenericConstants.DATA_SOURCE.DATASOURCENAME, tableName);
					Long dsId = createUtil.createDataSources(args);
					if (dsId == null) {
						continue;
					}
					fieldMappings = getFieldMapping(databaseName, tableName, client);
					if (doSampling != null && sampleRowCount != null && doSampling == true) {
						 fieldMappings = addSampleData(fieldMappings, client, databaseName,
						 tableName, sampleRowCount);
					}
					DataSource dataSource = DataRepositoryDao.getDataSource(dsId);
					dataSource.setAdvancedValidations(null);
					dataSource.setAdvancedValidationsStore(null);
					dataSource.setTransformations(null);
					dataSource.setDataCleansers(null);
					dataSource.setAppId(null);
					dataSource.setNodeId(null);
					if (null != dataSource && null != fieldMappings) {
						dataSource.setConfig(connectionConfig);

						String prestoTableAlias = connectionSubSources.getSubConnectionType().toLowerCase() + "."
								+ databaseName + "." + tableName;
						dataSource.setPrestoTableAlias(prestoTableAlias);
						dataSource.setDataSourceType(DataSourceType.MONGODB);
						dataSource.setFieldMappings(fieldMappings);
						dataSource.setConnectionSources(connectionSources);
						DataRepositoryDao.insertDataSource(dataSource);
						Map<String, String> updatedValue = new LinkedHashMap<String, String>();
						updatedValue.put("dataSourceType", (dataSource.getDataSourceType()).toString());
						DataSourceInstanceDao.updateDataRepoIdIntoDataSourceInstance(dataSource.getId(), updatedValue);
					}

				}

			}

		}

		return existingDSNotifier;
	}

	public String establishedConnection(ConnectionConfig connectionConfig) {
		String message = TestConnectionEnums.SUCCESS.name();
		String userName = connectionConfig.getDbUserName();
		String password = connectionConfig.getDbPassword();
		String host = connectionConfig.getIpAddress();
		int port = connectionConfig.getPort();
		String uri = "mongodb://" + userName + ":" + password + "@" + host + ":" + port;
		MongoClient mongoClient = null;
		DB db = null;
		List<String> dblist = null;
		try {
			mongoClient = new MongoClient(new MongoClientURI(uri));
			mongoClient.listDatabaseNames();
		} catch (MongoTimeoutException ex) {
			message = TestConnectionEnums.FAIL.name();
		} finally {
			mongoClient.close();
		}
		return message;
	}

	@Transactional
	private ConnectionConfig getConnectionConfig(ConnectionSources connectionSources) {
		ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
		Long connectionId = connectionSubSources.getConnectionConfig().get(0).getConnectionId();
		return noSqlConnectorListingDAO.getConnectionConfig(connectionId);
	}

	public DataNode getMongoDBCollections() {
		MongoIterable<String> dbIterableList = null;
		MongoClient client = getMongoClient();
		DataNode schemaNode = new DataNode();
		schemaNode.setLabel("Schemas");
		try {
			MongoIterable<String> mongoIterable = client.listDatabaseNames();
			MongoCursor<String> mongoCursor = mongoIterable.iterator();

			while (mongoCursor.hasNext()) {
				String dbName = mongoCursor.next();
				if ("admin".equals(dbName) || "local".equals(dbName) || "config".equals(dbName)) {
					continue;
				}
				DataNode dataBaseNode = new DataNode(dbName, false);
				DataNode collectionNode = new DataNode();
				collectionNode.setLabel("Collections");
				MongoDatabase mongoDataBase = client.getDatabase(dbName);
				MongoIterable<String> mongoCollectionIterable = mongoDataBase.listCollectionNames();
				MongoCursor<String> mongoCollectionCursor = mongoCollectionIterable.iterator();
				while (mongoCollectionCursor.hasNext()) {
					DataNode collectionNameNode = new DataNode(mongoCollectionCursor.next(), true);
					collectionNode.getChildren().add(collectionNameNode);
				}
				dataBaseNode.getChildren().add(collectionNode);
				schemaNode.getChildren().add(dataBaseNode);
			}
		} catch (MongoTimeoutException e) {
			e.printStackTrace();
		}
		return schemaNode;
	}

	public MongoClient getMongoClient() {
		MongoClient mongoClient = null;
		String userName = noSqlConnectionConfig.getDbUserName();
		String password = noSqlConnectionConfig.getDbPassword();
		String host = noSqlConnectionConfig.getIpAddress();
		int port = noSqlConnectionConfig.getPort();
		String uri = "mongodb://" + userName + ":" + password + "@" + host + ":" + port;
		DB db = null;
		List<String> dblist = null;
		try {
			mongoClient = new MongoClient(new MongoClientURI(uri));
		} catch (Exception ex) {
			ex.printStackTrace();
		}

		return mongoClient;
	}

	public Collection<DataNode> filterNodesToPublish(Collection<DataNode> dataNodes) {

		Collection<DataNode> validNodes = new ArrayList<DataNode>();

		for (DataNode itrDataNode : dataNodes) {
			if (!itrDataNode.getChildren().isEmpty()) {
				Collection<DataNode> subList = filterNodesToPublish(itrDataNode.getChildren());
				validNodes.addAll(subList);
			} else {
				if (itrDataNode.getIsDsNode() && itrDataNode.getSelected())
					validNodes.add(itrDataNode);
			}
		}

		return validNodes;
	}

	public Collection<Map<String, List<String>>> getDbWithCollectionList(DataNode dataNode) {
		Collection<DataNode> databaseList = dataNode.getChildren();// All
		// databases
		// node list
		// will be
		// stored
		Collection<Map<String, List<String>>> dbWithTablesAsChild = new ArrayList<Map<String, List<String>>>();
		if (databaseList != null && !databaseList.isEmpty()) {
			for (DataNode currentDatabase : databaseList) {
				Map<String, List<String>> currentDbWithtbldetails = new HashMap<>();
				String DatabaseName = currentDatabase.getLabel();// current db
				Collection<DataNode> TablesAndViewsList = currentDatabase.getChildren();// list
				// of
				// tables
				// and
				// vies
				// in
				// that
				// db
				Iterator<DataNode> tblAndviewlistItr = TablesAndViewsList.iterator();
				List<String> tableNames = new ArrayList<>();
				while (tblAndviewlistItr.hasNext()) {
					Collection<DataNode> tableOrViewList = tblAndviewlistItr.next().getChildren();// list
					// of
					// table
					// or
					// view
					Iterator<DataNode> tbl0rviewItr = tableOrViewList.iterator();
					while (tbl0rviewItr.hasNext()) {
						DataNode currentTable = tbl0rviewItr.next();
						tableNames.add(currentTable.getLabel());
					}

				}
				currentDbWithtbldetails.put(DatabaseName, tableNames);
				dbWithTablesAsChild.add(currentDbWithtbldetails);
			}

		}
		return dbWithTablesAsChild;
	}

	public List<FieldMapping> getFieldMapping(String databaseName, String tableName, MongoClient client) {
		List<FieldMapping> fieldMappings = new ArrayList<>();
		if (client != null) {
			try {
				MongoDatabase db = client.getDatabase(databaseName);
				MongoCollection<Document> mongoCollection = db.getCollection(tableName);
				Document document = mongoCollection.find().first();
				CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
				
				int i = 1;
					Set<Entry<String, Object>> setField = document.entrySet();
					for (Entry<String, Object> entry : setField) {
						System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
						String propertyName = entry.getKey();
						long fieldId = i;
						FieldMapping fieldMapping = new FieldMapping();
						List<FieldMapping> listFieldMapping = new ArrayList<FieldMapping>();
						fieldMapping.setListFieldMapping(listFieldMapping);
						fieldMapping.setId(fieldId);
						fieldMapping.setFieldName(propertyName);
						if ("_id".equals(propertyName)) {
							fieldMapping.setIsPrimaryKey(true);
						}
						BsonDocument bdoc = document.toBsonDocument(Document.class, codecRegistry);
						System.out.println(bdoc.isDocument(propertyName));
						if (bdoc.isDocument(propertyName)) {
							getNestedDocumentAndArraysProperty(document,bdoc, fieldMapping, propertyName);
						} else if (bdoc.isArray(entry.getKey())) {
							getNestedDocumentAndArraysProperty(document,bdoc, fieldMapping, propertyName);
						} 
						
						FieldDataTypes propertyDataType = getCollectionPropertyDataType(document, propertyName);
						Integer j = i - 1;
						Long pos = j.longValue();
						fieldMapping.setPosition(pos);
						fieldMapping.setFieldDataType(propertyDataType);
						i++;
						fieldMappings.add(fieldMapping);
					}
			} catch (MongoTimeoutException e) {

				e.printStackTrace();
			}
		}
		return fieldMappings;

	}

	public List<FieldMapping> addSampleData(List<FieldMapping> fieldMappings, MongoClient client, String databaseName,
			String tableName, Integer sampleRows) throws SQLException {
		logger.debug(LoggerConstants.LOG_MAXIQWEB + " >> addSampleData()" + ParamUtils.getString(fieldMappings));
		
		try {
			MongoDatabase db = client.getDatabase(databaseName);
			MongoCollection<Document> mongoCollection = db.getCollection(tableName);
			FindIterable<Document> findCollectionIterable = mongoCollection.find().limit(sampleRows);
			MongoCursor<Document> cursorFields = findCollectionIterable.iterator();
			CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
			int count = 0;
			while (cursorFields.hasNext()) {
				int i = 0;
				Document doc = cursorFields.next();
				BsonDocument bdoc = doc.toBsonDocument(Document.class, codecRegistry);
				Set<Entry<String, Object>> entryFields = doc.entrySet();
				for (Entry<String, Object> entry : entryFields) {
					FieldMapping fieldMapping = fieldMappings.get(i);
					if (null != fieldMapping.getListFieldMapping() && fieldMapping.getListFieldMapping().size() > 0) {
						String propertyName = entry.getKey();
						fieldMapping.getSampleRecords().add(count, propertyName);
						getNestedDocumentAndArrayValues(doc, bdoc, fieldMapping, propertyName, count,i);
					} else {
						fieldMapping.getSampleRecords().add(count, entry.getValue().toString());
					}
					
					i++;
				}
				count++;
			}
			logger.debug(LoggerConstants.LOG_MAXIQWEB + " << addSampleData()" + ParamUtils.getString(fieldMappings));

		} catch (MongoTimeoutException e) {
			e.printStackTrace();
		}

		logger.debug(LoggerConstants.LOG_MAXIQWEB + " << addSampleData()" + ParamUtils.getString(fieldMappings));

		addDateFormat(fieldMappings);
		return fieldMappings;
	}

	public FieldDataTypes getCollectionPropertyDataType(Document doc, String propertyName) {
		FieldDataTypes type = null;
		CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
		BsonDocument bsonDoc = doc.toBsonDocument(Document.class, codecRegistry);
		if (bsonDoc.isArray(propertyName)) {
			type = FieldDataTypes.ARRAYLIST;
		} else if (bsonDoc.isBinary(propertyName)) {
			type = FieldDataTypes.BINARY;
		} else if (bsonDoc.isBoolean(propertyName)) {
			type = FieldDataTypes.BOOLEAN;
		} else if (bsonDoc.isDateTime(propertyName)) {
			type = FieldDataTypes.DATE_TIME;
		} else if (bsonDoc.isDecimal128(propertyName)) {
			type = FieldDataTypes.DECIMAL128;
		} else if (bsonDoc.isDocument(propertyName)) {
			type = FieldDataTypes.DOCUMENT;
		} else if (bsonDoc.isDouble(propertyName)) {
			type = FieldDataTypes.DOUBLE;
		} else if (bsonDoc.isInt32(propertyName)) {
			type = FieldDataTypes.INT32;
		} else if (bsonDoc.isInt64(propertyName)) {
			type = FieldDataTypes.INT64;
		} else if (bsonDoc.isNull(propertyName)) {
			type = FieldDataTypes.NULL;
		} else if (bsonDoc.isNumber(propertyName)) {
			type = FieldDataTypes.NUMBER;
		} else if (bsonDoc.isString(propertyName)) {
			type = FieldDataTypes.STRING;
		} else if (bsonDoc.isObjectId(propertyName)) {
			type = FieldDataTypes.OBJECTID;
		}

		return type;
	}
	
	public FieldDataTypes getCollectionPropertyBsonDataType(BsonDocument bsonDoc, String propertyName ) {
		FieldDataTypes type = null;
		
		if (bsonDoc.isArray(propertyName)) {
			type = FieldDataTypes.ARRAYLIST;
		}else if (bsonDoc.isBinary(propertyName)) {
			type = FieldDataTypes.BINARY;
		} else if (bsonDoc.isBoolean(propertyName)) {
			type = FieldDataTypes.BOOLEAN;
		} else if (bsonDoc.isDateTime(propertyName)) {
			type = FieldDataTypes.DATE_TIME;
		} else if (bsonDoc.isDecimal128(propertyName)) {
			type = FieldDataTypes.DECIMAL128;
		} else if (bsonDoc.isDocument(propertyName)) {
			type = FieldDataTypes.DOCUMENT;
		} else if (bsonDoc.isDouble(propertyName)) {
			type = FieldDataTypes.DOUBLE;
		} else if (bsonDoc.isInt32(propertyName)) {
			type = FieldDataTypes.INT32;
		} else if (bsonDoc.isInt64(propertyName)) {
			type = FieldDataTypes.INT64;
		} else if (bsonDoc.isNull(propertyName)) {
			type = FieldDataTypes.NULL;
		} else if (bsonDoc.isNumber(propertyName)) {
			type = FieldDataTypes.NUMBER;
		} else if (bsonDoc.isString(propertyName)) {
			type = FieldDataTypes.STRING;
		} else if (bsonDoc.isObjectId(propertyName)) {
			type = FieldDataTypes.OBJECTID;
		}
		
		return type;
	}

	private void addDateFormat(List<FieldMapping> fieldMappings) {

		if (fieldMappings != null && fieldMappings.size() > 0) {
			for (FieldMapping field : fieldMappings) {
				if (field.getFieldDataType() == FieldDataTypes.DATE) {
					// SampleRecordExtractor.checkDate(field);
					checkDate(field);
				} else if (field.getFieldDataType() == FieldDataTypes.TIMESTAMP) {
					// SampleRecordExtractor.checkTimeStamp(field);
					checkTimeStamp(field);
				}
			}
		}
	}

	public static Boolean checkDate(FieldMapping field) {

		Map<String, Integer> formatCounter = getSuitableFromats(field, FieldDataTypes.DATE);

		if (formatCounter.size() > 0) {
			String maxFormat = "";
			Integer max = 0;

			for (Entry<String, Integer> e : formatCounter.entrySet()) {
				String format = e.getKey();
				Integer counter = e.getValue();

				if (counter > max) {
					max = counter;
					maxFormat = format;
				}
			}

			if (StringUtils.isNotBlank(maxFormat)) {
				field.setFormat(maxFormat);
				field.setFieldDataType(FieldDataTypes.DATE);
				return true;
			}
		}

		if (field.getFieldDataType() == FieldDataTypes.DATE) {
			return true; // If header template says it is a date field, then it
			// is, even if there is no valid dates found
		}

		return false;
	}

	public static Boolean checkTimeStamp(FieldMapping field) {

		Map<String, Integer> formatCounter = getSuitableFromats(field, FieldDataTypes.TIMESTAMP);

		if (formatCounter.size() > 0) {
			String maxFormat = "";
			Integer max = 0;

			for (Entry<String, Integer> e : formatCounter.entrySet()) {
				String format = e.getKey();
				Integer counter = e.getValue();

				if (counter > max) {
					max = counter;
					maxFormat = format;
				}
			}

			if (StringUtils.isNotBlank(maxFormat)) {
				field.setFormat(maxFormat);
				field.setFieldDataType(FieldDataTypes.TIMESTAMP);
				return true;
			}
		}

		if (field.getFieldDataType() == FieldDataTypes.TIMESTAMP) {
			return true; // If header template says it is a TimeStamp field,
			// then it
			// is, even if there is no valid dates found
		}

		return false;
	}

	public static Map<String, Integer> getSuitableFromats(FieldMapping field, FieldDataTypes type) {

		List<String> allPossibleFormats = new ArrayList<String>();
		List<String> sampleRecordList = field.getSampleRecords();
		Iterator<String> iterator = sampleRecordList.iterator();
		while (iterator.hasNext()) {
			allPossibleFormats.addAll(DateUtils.findMyDateAndTSFormat(iterator.next(), type));
		}
		Map<String, Integer> formatCounter = new HashMap<String, Integer>();
		if (allPossibleFormats.size() > 0) {
			for (String format : allPossibleFormats) {
				if (!formatCounter.containsKey(format)) {
					formatCounter.put(format, 0);
				}
				formatCounter.put(format, formatCounter.get(format) + 1);
			}
		}

		return formatCounter;
	}
	
	public void getNestedDocumentAndArraysProperty(Document document, BsonDocument bdoc, FieldMapping fieldMapping , String propertyName) {
		System.out.println("recursive block start--");
		if (bdoc.isDocument(propertyName)) {
			System.out.println("document starts -----");
			BsonDocument bsonDoc = bdoc.getDocument(propertyName);
			int i =1;
			Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
			for (Entry<String, BsonValue> entry : entrySet) {
				FieldMapping nestedFieldMapping = new FieldMapping();
				String nestedPropertyName = entry.getKey();
				System.out.println("Key -: " + nestedPropertyName);
				long fieldId = i;
				nestedFieldMapping.setId(fieldId);
				nestedFieldMapping.setFieldName(nestedPropertyName);
				FieldDataTypes propertyDataType = getCollectionPropertyBsonDataType(bsonDoc, nestedPropertyName);
				System.out.println(propertyDataType.toString());
				Integer j = i - 1;
				Long pos = j.longValue();
				nestedFieldMapping.setPosition(pos);
				nestedFieldMapping.setFieldDataType(propertyDataType);
				
				if (bdoc.isDocument(nestedPropertyName)) {
					getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
				} else if (bdoc.isArray(entry.getKey())) {
					getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
				}
				fieldMapping.getListFieldMapping().add(nestedFieldMapping);
				i++;
			}
			
			System.out.println("document ends -----");
			
		} else if (bdoc.isArray(propertyName)) {
			System.out.println("Array starts----");
			BsonArray bsonArray = bdoc.getArray(propertyName);
			BsonValue bsonValue = bdoc.getArray(propertyName).get(0);
				System.out.println(bsonValue);
				
				if (bsonValue.isDocument()) {
					BsonDocument bsonDoc = bsonValue.asDocument();
					Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
					int i = 1;
					for (Entry<String, BsonValue> entry : bsonSet) {
						String nestedPropertyName = entry.getKey();
						FieldMapping nestedFieldMapping = new FieldMapping();
						System.out.println("Key -: " + nestedPropertyName);
						long fieldId = i;
						nestedFieldMapping.setId(fieldId);
						nestedFieldMapping.setFieldName(nestedPropertyName);
						FieldDataTypes propertyDataType = getCollectionPropertyDataType(document, nestedPropertyName);
						Integer j = i - 1;
						Long pos = j.longValue();
						nestedFieldMapping.setPosition(pos);
						nestedFieldMapping.setFieldDataType(propertyDataType);
						if (bsonDoc.isDocument(nestedPropertyName)) {
							getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
						} else if (bsonDoc.isArray(nestedPropertyName)) {
							getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
						}
						fieldMapping.getListFieldMapping().add(nestedFieldMapping);
						i++;
					}
				} else if (bsonValue.isArray()) {
					getNestedDocumentAndArraysProperty(document, bdoc, fieldMapping, bsonValue.toString());
				}
			System.out.println("Array ends -----");
		}
		
		System.out.println("recursive block end--");
	}
	
	public void getNestedDocumentAndArrayValues(Document document, BsonDocument bdoc, FieldMapping fieldMapping , String propertyName, int count, int fileldMappingIndex) {
		System.out.println("sample recursive block start--");
		if (bdoc.isDocument(propertyName)) {
			System.out.println("sample document starts -----");
			BsonDocument bsonDoc = bdoc.getDocument(propertyName);
			int i =0;
			Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
			
			List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
			
			for (FieldMapping fieldMap : listFieldMapping) {
				for (Entry<String, BsonValue> entry : entrySet) {
					String nestedPropertyName = entry.getKey();
					if (nestedPropertyName.equals(fieldMap.getFieldName())) {
						if (bdoc.isDocument(nestedPropertyName)) {
							System.out.println(nestedPropertyName);
							fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
							getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
						} else if (bdoc.isArray(entry.getKey())) {
							System.out.println(nestedPropertyName);
							fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
							getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
						}else {
							System.out.println(entry.getValue().toString());
							fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, getBsonPropertyValue(entry.getValue()));
						}
					}
				}
				i++;
			}
			
			
			System.out.println("sample document ends -----");
			
		}else if (bdoc.isArray(propertyName)) {
			System.out.println("Array starts----");
			BsonArray bsonArray = bdoc.getArray(propertyName);
//			List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
			for (BsonValue bsonValue : bsonArray) {
				System.out.println(bsonValue);
				if (bsonValue.isDocument()) {
					BsonDocument bsonDoc = bsonValue.asDocument();
					Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
					List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
					
					int i = 0;
					
					for (FieldMapping fieldMap : listFieldMapping) {
						
						for (Entry<String, BsonValue> entry : bsonSet) {
							String nestedPropertyName = entry.getKey();
							if (nestedPropertyName.equals(fieldMap.getFieldName())) {
								if (bsonDoc.isDocument(nestedPropertyName)) {
									System.out.println(nestedPropertyName);
									fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(nestedPropertyName);
									getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
								} else if (bsonDoc.isArray(nestedPropertyName)) {
									System.out.println(nestedPropertyName);
									fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
									getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
								}else {
									System.out.println(entry.getValue().toString());
									fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(getBsonPropertyValue(entry.getValue()));
//									fieldMapping.getListFieldMapping().get(i).getArraySampleRecords().add(fieldMapping.getListFieldMapping().get(i).getSampleRecords());
									
								}
							}
							
						}
						
						i++;
						
					}
					
					
				} else if (bsonValue.isArray()) {
					getNestedDocumentAndArraysProperty(document, bdoc, fieldMapping, bsonValue.toString());
				}
			}
			
			for (int i = 0; i < fieldMapping.getListFieldMapping().size(); i++) {
				fieldMapping.getListFieldMapping().get(i).getArraySampleRecords().add(fieldMapping.getListFieldMapping().get(i).getSampleRecords());
				fieldMapping.getListFieldMapping().get(i).setSampleRecords(new ArrayList<String>());
			}
			
			System.out.println("Array ends -----");
		}
	}
	
	public String getBsonPropertyValue( BsonValue bsonValue) {
		String value = null;
		if (bsonValue.isBinary()) {
			value = bsonValue.asBinary().getData().toString();
		}else if (bsonValue.isString()) {
			value = bsonValue.asString().getValue();
		}else if (bsonValue.isDocument()) {
			value = bsonValue.asDocument().toString();
		}else if (bsonValue.isArray()) {
			StringBuilder sb = new StringBuilder();
			for (BsonValue bValue : bsonValue.asArray().getValues()) {
				sb.append(",");
				sb.append(getBsonPropertyValue(bValue));
			}
			value = sb.toString().replaceFirst(",", "");
			
		}else if (bsonValue.isDouble()) {
			value = Double.toString(bsonValue.asDouble().getValue());
		}else if (bsonValue.isDateTime()) {
			value = Long.toString(bsonValue.asDateTime().getValue());
		}else if (bsonValue.isInt32()) {
			value = Integer.toString(bsonValue.asInt32().getValue());
		}
		return value;
	}

	public static void main(String[] args) {
		// MongoClient mongo = new MongoClient("localhost", 27017);
		List<String> dbs = null;
		DB dbss = null;
		// try {
		MongoClient mongo = new MongoClient(new MongoClientURI("mongodb://admin:mongodb@localhost:27017"));
		// MongoClient mongo = new MongoClient();
		dbss = mongo.getDB("hotel");
		CommandResult result = dbss.getStats();
		MongoDatabase db = mongo.getDatabase("hotel");
		CodecRegistry codecRegistry = mongo.getDefaultCodecRegistry();
//		System.out.println("loop start -j-");
		// dbs = mongo.getDatabaseNames();
		MongoIterable<String> dbIterable = mongo.listDatabaseNames();
		MongoCursor<String> mongoCursor = dbIterable.iterator();
		
//		while (mongoCursor.hasNext()) {
//			System.out.println(mongoCursor.next());
//		}

//		System.out.println("loop end --");

		
		/*MongoIterable<String> mongoCollectionIterable = db.listCollectionNames();
		MongoCursor<String> mongoCollectionCursor = mongoCollectionIterable.iterator();*/

	
//		while (mongoCollectionCursor.hasNext()) {
//			String tableName = mongoCollectionCursor.next();
			MongoCollection<Document> mongoCollection = db.getCollection("restaurants");
//			List<Document> documents = mongoCollection.find().limit(1).into(new ArrayList<Document>());
//			System.out.println("document start -----");
//			for (Document document : documents) {
//				System.out.println(document);

//			}
//			System.out.println("document end -----");
			/*ListIndexesIterable<Document> indexIterable = db.getCollection(tableName).listIndexes();
			MongoCursor<Document> indexMongoCursor = indexIterable.iterator();
			while (indexMongoCursor.hasNext()) {
				System.out.println(indexMongoCursor.next());

			}*/

			FindIterable<Document> findCollectionIterable = mongoCollection.find().limit(1);
			MongoCursor<Document> cursorFileds = findCollectionIterable.iterator();
			while (cursorFileds.hasNext()) {
				// System.out.println(cursorFileds.next());
				Document dom = cursorFileds.next();
				Set<Entry<String, Object>> setField = dom.entrySet();

				for (Entry<String, Object> entry : setField) {
//					System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
//					System.out.println(dom.get(entry.getKey()));
					BsonDocument bdoc = dom.toBsonDocument(Document.class, codecRegistry);
//					System.out.println(bdoc.isDocument(entry.getKey()));
					if (bdoc.isDocument(entry.getKey())) {
//						recursiveDocument(dom,bdoc, entry.getKey());
					} else if (bdoc.isArray(entry.getKey())) {
//						recursiveDocument(dom,bdoc, entry.getKey());
					} 
					

				}
			}
//		}
			
			MongoDBScanner scan = new MongoDBScanner();
			List<FieldMapping> listSample = null;
			List<FieldMapping> fieldMappings = scan.getFieldMapping("hotel", "restaurants", mongo);
			try {
				listSample = scan.addSampleData(fieldMappings, mongo, "hotel", "restaurants", 2);
			} catch (Exception e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
			for (FieldMapping fieldMapping : listSample) {
				System.out.println(fieldMapping.getFieldName());
			}

	}
	
	static public void recursiveDocument(Document dom,BsonDocument bdoc, String key) {
		System.out.println("recursive block start--");
		if (bdoc.isDocument(key)) {
			System.out.println("document starts -----");
			BsonDocument bsonDoc = bdoc.getDocument(key);
			Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
			for (Entry<String, BsonValue> entry : entrySet) {
				System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
				if (bdoc.isDocument(entry.getKey())) {
					recursiveDocument(dom, bsonDoc, entry.getKey());
				} else if (bdoc.isArray(entry.getKey())) {
					recursiveDocument(dom, bsonDoc, entry.getKey());
				}
			}
			System.out.println("document ends -----");
			
		} else if (bdoc.isArray(key)) {
			System.out.println("Array starts----");
			BsonArray bsonArray = bdoc.getArray(key);
			for (BsonValue bsonValue : bsonArray) {
				System.out.println(bsonValue);
				
				if (bsonValue.isDocument()) {
					BsonDocument bsonDoc = bsonValue.asDocument();
					Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
					for (Entry<String, BsonValue> entry : bsonSet) {
						System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
						if (bsonDoc.isDocument(entry.getKey())) {
							recursiveDocument(dom, bsonDoc, entry.getKey());
						} else if (bsonDoc.isArray(entry.getKey())) {
							recursiveDocument(dom, bsonDoc, entry.getKey());
						}
					}
					recursiveDocument(dom, bsonDoc, bsonValue.toString());
				} else if (bsonValue.isArray()) {
					recursiveDocument(dom, bdoc, bsonValue.toString());
				}
			}
			System.out.println("Array ends -----");
		}
		
		System.out.println("recursive block end--");
	}
	
	static String getBsonValue(BsonDocument bsonDoc, String key) {
		String value = null;
		
		return value;
	}
}