PageRenderTime 59ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 1ms

/Mosaic-Catalogue-Connectors/src/main/java/com/augiq/external/source/nosql/MongoDBScanner.java

https://bitbucket.org/gireeshp/data-catalog-team
Java | 899 lines | 730 code | 114 blank | 55 comment | 180 complexity | 79f8747181a937424ae61fbcf972ad4f MD5 | raw file
  1. package com.augiq.external.source.nosql;
  2. import java.sql.Connection;
  3. import java.sql.PreparedStatement;
  4. import java.sql.ResultSet;
  5. import java.sql.SQLException;
  6. import java.sql.Types;
  7. import java.util.ArrayList;
  8. import java.util.Collection;
  9. import java.util.HashMap;
  10. import java.util.Iterator;
  11. import java.util.LinkedHashMap;
  12. import java.util.List;
  13. import java.util.Map;
  14. import java.util.Map.Entry;
  15. import java.util.Set;
  16. import org.apache.commons.lang.StringUtils;
  17. import org.bson.BsonArray;
  18. import org.bson.BsonDocument;
  19. import org.bson.BsonType;
  20. import org.bson.BsonValue;
  21. import org.bson.Document;
  22. import org.bson.codecs.configuration.CodecRegistry;
  23. import org.bson.conversions.Bson;
  24. import org.bson.types.ObjectId;
  25. import org.slf4j.Logger;
  26. import org.slf4j.LoggerFactory;
  27. import org.springframework.beans.factory.annotation.Autowired;
  28. import org.springframework.transaction.annotation.Transactional;
  29. import com.augiq.external.source.scanning.Scanner;
  30. import com.augmentiq.constant.maxiq.LoggerConstants;
  31. import com.augmentiq.maxiq.base.dao.sql.operation.configuration.DateUtils;
  32. import com.augmentiq.maxiq.base.dao.sql.operation.configuration.ParamUtils;
  33. import com.augmentiq.maxiq.constant.configuration.enums.ConnectorType;
  34. import com.augmentiq.maxiq.constant.configuration.enums.DataSourceType;
  35. import com.augmentiq.maxiq.constant.configuration.enums.FieldDataTypes;
  36. import com.augmentiq.maxiq.constant.configuration.generic.GenericConstants;
  37. import com.augmentiq.maxiq.constant.external.source.task.status.enums.TaskStatusEnums;
  38. import com.augmentiq.maxiq.entity.model.configuration.bean.DataSource;
  39. import com.augmentiq.maxiq.entity.model.configuration.bean.FieldMapping;
  40. import com.augmentiq.maxiq.external.source.connection.message.constants.TestConnectionEnums;
  41. import com.augmentiq.maxiq.model.connector.datanode.DataNode;
  42. import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionConfig;
  43. import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionSources;
  44. import com.augmentiq.maxiq.orm.entity.model.connector.ConnectionSubSources;
  45. import com.augmentiq.maxiq.repository.connector.external.connectiondao.ConnectorListingDAO;
  46. import com.augmentiq.maxiq.util.component.configuration.util.ConfigurationCreateUtil;
  47. import com.augmentiq.maxiq.util.external.source.rdbms.connection.lookup.directory.ConnectionLookUpDirectory;
  48. import com.augumentiq.maxiq.configuration.dao.DataRepositoryDao;
  49. import com.augumentiq.maxiq.configuration.dao.DataSourceInstanceDao;
  50. import com.google.gson.Gson;
  51. import com.mongodb.BasicDBObject;
  52. import com.mongodb.CommandResult;
  53. import com.mongodb.DB;
  54. import com.mongodb.DBCollection;
  55. import com.mongodb.DBCursor;
  56. import com.mongodb.MongoClient;
  57. import com.mongodb.MongoClientURI;
  58. import com.mongodb.MongoTimeoutException;
  59. import com.mongodb.client.FindIterable;
  60. import com.mongodb.client.ListIndexesIterable;
  61. import com.mongodb.client.MongoCollection;
  62. import com.mongodb.client.MongoCursor;
  63. import com.mongodb.client.MongoDatabase;
  64. import com.mongodb.client.MongoIterable;
  65. import com.mongodb.gridfs.GridFS;
  66. import com.mongodb.gridfs.GridFSDBFile;
  67. import com.sforce.ws.wsdl.Port;
  68. import ch.epfl.lamp.fjbg.JConstantPool.FieldOrMethodRefEntry;
  69. public class MongoDBScanner extends NoSQLScanner {
  70. @Autowired
  71. private ConnectorListingDAO noSqlConnectorListingDAO;
  72. @Autowired
  73. private ConfigurationCreateUtil createUtil;
  74. private ConnectionConfig noSqlConnectionConfig;
  75. private String subSourceType;
  76. private static final Logger logger = LoggerFactory.getLogger(MongoDBScanner.class);
  77. public String testConnection(Map<String, Object> args) {
  78. Connection connection = null;
  79. ConnectionConfig connectionConfig = (ConnectionConfig) args.get("connectionConfig");
  80. this.subSourceType = args.get("subSourceType").toString();
  81. String message = TestConnectionEnums.SUCCESS.name();
  82. message = establishedConnection(connectionConfig);
  83. return message;
  84. }
  85. public DataNode scan(Map<String, Object> args) throws Exception {
  86. ConnectionSources connectionSources = (ConnectionSources) args.get("ConnectionSources");
  87. ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
  88. noSqlConnectionConfig = getConnectionConfig(connectionSources);
  89. DataNode node = getMongoDBCollections();
  90. Gson gson = new Gson();
  91. System.out.println(gson.toJson(node));
  92. return node;
  93. }
  94. public Map<String, List<String>> publishDataSources(Map<String, Object> args) throws Exception {
  95. Map<String, List<String>> existingDSNotifier = new HashMap<String, List<String>>();
  96. existingDSNotifier.put(TaskStatusEnums.SUCCESS.name(), new ArrayList<String>());
  97. args.put("existingDataSourceNotifier", existingDSNotifier);
  98. DataNode dataNode = new DataNode();
  99. ConnectionSources connectionSources = objectParserHandler((Object) args.get("connectionSources"),
  100. ConnectionSources.class);
  101. ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
  102. ConnectionConfig connectionConfig = connectionSubSources.getConnectionConfig().get(0);
  103. DataNode dataNodefromUi = objectParserHandler((Object) args.get("dataNode"), DataNode.class);
  104. DataNode filteredNodes = dataNode.getSelectedNodes(dataNodefromUi);
  105. Collection<DataNode> nodesToPublish = filterNodesToPublish(filteredNodes.getChildren());
  106. Collection<Map<String, List<String>>> dbWithCollectionAsChild = getDbWithCollectionList(filteredNodes);
  107. Iterator<Map<String, List<String>>> dbWithCollectionItr = dbWithCollectionAsChild.iterator();
  108. Boolean doSampling = (Boolean) args.get("doSampling");
  109. Integer sampleRowCount = (Integer) args.get("sampleRowCount");
  110. while (dbWithCollectionItr.hasNext()) {
  111. List<FieldMapping> fieldMappings = null;
  112. String databaseName = null;
  113. List<String> currentCollectionAndViewsList = null;
  114. Map<String, List<String>> currentDbWithCollectiondetails = dbWithCollectionItr.next();
  115. Set<Entry<String, List<String>>> set = currentDbWithCollectiondetails.entrySet();
  116. Iterator<Map.Entry<String, List<String>>> entryItr = set.iterator();
  117. while (entryItr.hasNext()) {
  118. Map.Entry<String, List<String>> entry = entryItr.next();
  119. databaseName = entry.getKey();
  120. currentCollectionAndViewsList = entry.getValue();
  121. Iterator<String> iterator = currentCollectionAndViewsList.iterator();
  122. while (iterator.hasNext()) {
  123. Connection connection = null;
  124. this.subSourceType = StringUtils.upperCase(connectionSubSources.getSubConnectionType());
  125. MongoClient client = getMongoClient();
  126. String tableName = iterator.next();
  127. args.put(GenericConstants.DATA_SOURCE.DATASOURCENAME, tableName);
  128. Long dsId = createUtil.createDataSources(args);
  129. if (dsId == null) {
  130. continue;
  131. }
  132. fieldMappings = getFieldMapping(databaseName, tableName, client);
  133. if (doSampling != null && sampleRowCount != null && doSampling == true) {
  134. fieldMappings = addSampleData(fieldMappings, client, databaseName,
  135. tableName, sampleRowCount);
  136. }
  137. DataSource dataSource = DataRepositoryDao.getDataSource(dsId);
  138. dataSource.setAdvancedValidations(null);
  139. dataSource.setAdvancedValidationsStore(null);
  140. dataSource.setTransformations(null);
  141. dataSource.setDataCleansers(null);
  142. dataSource.setAppId(null);
  143. dataSource.setNodeId(null);
  144. if (null != dataSource && null != fieldMappings) {
  145. dataSource.setConfig(connectionConfig);
  146. String prestoTableAlias = connectionSubSources.getSubConnectionType().toLowerCase() + "."
  147. + databaseName + "." + tableName;
  148. dataSource.setPrestoTableAlias(prestoTableAlias);
  149. dataSource.setDataSourceType(DataSourceType.MONGODB);
  150. dataSource.setFieldMappings(fieldMappings);
  151. dataSource.setConnectionSources(connectionSources);
  152. DataRepositoryDao.insertDataSource(dataSource);
  153. Map<String, String> updatedValue = new LinkedHashMap<String, String>();
  154. updatedValue.put("dataSourceType", (dataSource.getDataSourceType()).toString());
  155. DataSourceInstanceDao.updateDataRepoIdIntoDataSourceInstance(dataSource.getId(), updatedValue);
  156. }
  157. }
  158. }
  159. }
  160. return existingDSNotifier;
  161. }
  162. public String establishedConnection(ConnectionConfig connectionConfig) {
  163. String message = TestConnectionEnums.SUCCESS.name();
  164. String userName = connectionConfig.getDbUserName();
  165. String password = connectionConfig.getDbPassword();
  166. String host = connectionConfig.getIpAddress();
  167. int port = connectionConfig.getPort();
  168. String uri = "mongodb://" + userName + ":" + password + "@" + host + ":" + port;
  169. MongoClient mongoClient = null;
  170. DB db = null;
  171. List<String> dblist = null;
  172. try {
  173. mongoClient = new MongoClient(new MongoClientURI(uri));
  174. mongoClient.listDatabaseNames();
  175. } catch (MongoTimeoutException ex) {
  176. message = TestConnectionEnums.FAIL.name();
  177. } finally {
  178. mongoClient.close();
  179. }
  180. return message;
  181. }
  182. @Transactional
  183. private ConnectionConfig getConnectionConfig(ConnectionSources connectionSources) {
  184. ConnectionSubSources connectionSubSources = connectionSources.getConnectionSubSources().get(0);
  185. Long connectionId = connectionSubSources.getConnectionConfig().get(0).getConnectionId();
  186. return noSqlConnectorListingDAO.getConnectionConfig(connectionId);
  187. }
  188. public DataNode getMongoDBCollections() {
  189. MongoIterable<String> dbIterableList = null;
  190. MongoClient client = getMongoClient();
  191. DataNode schemaNode = new DataNode();
  192. schemaNode.setLabel("Schemas");
  193. try {
  194. MongoIterable<String> mongoIterable = client.listDatabaseNames();
  195. MongoCursor<String> mongoCursor = mongoIterable.iterator();
  196. while (mongoCursor.hasNext()) {
  197. String dbName = mongoCursor.next();
  198. if ("admin".equals(dbName) || "local".equals(dbName) || "config".equals(dbName)) {
  199. continue;
  200. }
  201. DataNode dataBaseNode = new DataNode(dbName, false);
  202. DataNode collectionNode = new DataNode();
  203. collectionNode.setLabel("Collections");
  204. MongoDatabase mongoDataBase = client.getDatabase(dbName);
  205. MongoIterable<String> mongoCollectionIterable = mongoDataBase.listCollectionNames();
  206. MongoCursor<String> mongoCollectionCursor = mongoCollectionIterable.iterator();
  207. while (mongoCollectionCursor.hasNext()) {
  208. DataNode collectionNameNode = new DataNode(mongoCollectionCursor.next(), true);
  209. collectionNode.getChildren().add(collectionNameNode);
  210. }
  211. dataBaseNode.getChildren().add(collectionNode);
  212. schemaNode.getChildren().add(dataBaseNode);
  213. }
  214. } catch (MongoTimeoutException e) {
  215. e.printStackTrace();
  216. }
  217. return schemaNode;
  218. }
  219. public MongoClient getMongoClient() {
  220. MongoClient mongoClient = null;
  221. String userName = noSqlConnectionConfig.getDbUserName();
  222. String password = noSqlConnectionConfig.getDbPassword();
  223. String host = noSqlConnectionConfig.getIpAddress();
  224. int port = noSqlConnectionConfig.getPort();
  225. String uri = "mongodb://" + userName + ":" + password + "@" + host + ":" + port;
  226. DB db = null;
  227. List<String> dblist = null;
  228. try {
  229. mongoClient = new MongoClient(new MongoClientURI(uri));
  230. } catch (Exception ex) {
  231. ex.printStackTrace();
  232. }
  233. return mongoClient;
  234. }
  235. public Collection<DataNode> filterNodesToPublish(Collection<DataNode> dataNodes) {
  236. Collection<DataNode> validNodes = new ArrayList<DataNode>();
  237. for (DataNode itrDataNode : dataNodes) {
  238. if (!itrDataNode.getChildren().isEmpty()) {
  239. Collection<DataNode> subList = filterNodesToPublish(itrDataNode.getChildren());
  240. validNodes.addAll(subList);
  241. } else {
  242. if (itrDataNode.getIsDsNode() && itrDataNode.getSelected())
  243. validNodes.add(itrDataNode);
  244. }
  245. }
  246. return validNodes;
  247. }
  248. public Collection<Map<String, List<String>>> getDbWithCollectionList(DataNode dataNode) {
  249. Collection<DataNode> databaseList = dataNode.getChildren();// All
  250. // databases
  251. // node list
  252. // will be
  253. // stored
  254. Collection<Map<String, List<String>>> dbWithTablesAsChild = new ArrayList<Map<String, List<String>>>();
  255. if (databaseList != null && !databaseList.isEmpty()) {
  256. for (DataNode currentDatabase : databaseList) {
  257. Map<String, List<String>> currentDbWithtbldetails = new HashMap<>();
  258. String DatabaseName = currentDatabase.getLabel();// current db
  259. Collection<DataNode> TablesAndViewsList = currentDatabase.getChildren();// list
  260. // of
  261. // tables
  262. // and
  263. // vies
  264. // in
  265. // that
  266. // db
  267. Iterator<DataNode> tblAndviewlistItr = TablesAndViewsList.iterator();
  268. List<String> tableNames = new ArrayList<>();
  269. while (tblAndviewlistItr.hasNext()) {
  270. Collection<DataNode> tableOrViewList = tblAndviewlistItr.next().getChildren();// list
  271. // of
  272. // table
  273. // or
  274. // view
  275. Iterator<DataNode> tbl0rviewItr = tableOrViewList.iterator();
  276. while (tbl0rviewItr.hasNext()) {
  277. DataNode currentTable = tbl0rviewItr.next();
  278. tableNames.add(currentTable.getLabel());
  279. }
  280. }
  281. currentDbWithtbldetails.put(DatabaseName, tableNames);
  282. dbWithTablesAsChild.add(currentDbWithtbldetails);
  283. }
  284. }
  285. return dbWithTablesAsChild;
  286. }
  287. public List<FieldMapping> getFieldMapping(String databaseName, String tableName, MongoClient client) {
  288. List<FieldMapping> fieldMappings = new ArrayList<>();
  289. if (client != null) {
  290. try {
  291. MongoDatabase db = client.getDatabase(databaseName);
  292. MongoCollection<Document> mongoCollection = db.getCollection(tableName);
  293. Document document = mongoCollection.find().first();
  294. CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
  295. int i = 1;
  296. Set<Entry<String, Object>> setField = document.entrySet();
  297. for (Entry<String, Object> entry : setField) {
  298. System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
  299. String propertyName = entry.getKey();
  300. long fieldId = i;
  301. FieldMapping fieldMapping = new FieldMapping();
  302. List<FieldMapping> listFieldMapping = new ArrayList<FieldMapping>();
  303. fieldMapping.setListFieldMapping(listFieldMapping);
  304. fieldMapping.setId(fieldId);
  305. fieldMapping.setFieldName(propertyName);
  306. if ("_id".equals(propertyName)) {
  307. fieldMapping.setIsPrimaryKey(true);
  308. }
  309. BsonDocument bdoc = document.toBsonDocument(Document.class, codecRegistry);
  310. System.out.println(bdoc.isDocument(propertyName));
  311. if (bdoc.isDocument(propertyName)) {
  312. getNestedDocumentAndArraysProperty(document,bdoc, fieldMapping, propertyName);
  313. } else if (bdoc.isArray(entry.getKey())) {
  314. getNestedDocumentAndArraysProperty(document,bdoc, fieldMapping, propertyName);
  315. }
  316. FieldDataTypes propertyDataType = getCollectionPropertyDataType(document, propertyName);
  317. Integer j = i - 1;
  318. Long pos = j.longValue();
  319. fieldMapping.setPosition(pos);
  320. fieldMapping.setFieldDataType(propertyDataType);
  321. i++;
  322. fieldMappings.add(fieldMapping);
  323. }
  324. } catch (MongoTimeoutException e) {
  325. e.printStackTrace();
  326. }
  327. }
  328. return fieldMappings;
  329. }
  330. public List<FieldMapping> addSampleData(List<FieldMapping> fieldMappings, MongoClient client, String databaseName,
  331. String tableName, Integer sampleRows) throws SQLException {
  332. logger.debug(LoggerConstants.LOG_MAXIQWEB + " >> addSampleData()" + ParamUtils.getString(fieldMappings));
  333. try {
  334. MongoDatabase db = client.getDatabase(databaseName);
  335. MongoCollection<Document> mongoCollection = db.getCollection(tableName);
  336. FindIterable<Document> findCollectionIterable = mongoCollection.find().limit(sampleRows);
  337. MongoCursor<Document> cursorFields = findCollectionIterable.iterator();
  338. CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
  339. int count = 0;
  340. while (cursorFields.hasNext()) {
  341. int i = 0;
  342. Document doc = cursorFields.next();
  343. BsonDocument bdoc = doc.toBsonDocument(Document.class, codecRegistry);
  344. Set<Entry<String, Object>> entryFields = doc.entrySet();
  345. for (Entry<String, Object> entry : entryFields) {
  346. FieldMapping fieldMapping = fieldMappings.get(i);
  347. if (null != fieldMapping.getListFieldMapping() && fieldMapping.getListFieldMapping().size() > 0) {
  348. String propertyName = entry.getKey();
  349. fieldMapping.getSampleRecords().add(count, propertyName);
  350. getNestedDocumentAndArrayValues(doc, bdoc, fieldMapping, propertyName, count,i);
  351. } else {
  352. fieldMapping.getSampleRecords().add(count, entry.getValue().toString());
  353. }
  354. i++;
  355. }
  356. count++;
  357. }
  358. logger.debug(LoggerConstants.LOG_MAXIQWEB + " << addSampleData()" + ParamUtils.getString(fieldMappings));
  359. } catch (MongoTimeoutException e) {
  360. e.printStackTrace();
  361. }
  362. logger.debug(LoggerConstants.LOG_MAXIQWEB + " << addSampleData()" + ParamUtils.getString(fieldMappings));
  363. addDateFormat(fieldMappings);
  364. return fieldMappings;
  365. }
  366. public FieldDataTypes getCollectionPropertyDataType(Document doc, String propertyName) {
  367. FieldDataTypes type = null;
  368. CodecRegistry codecRegistry = MongoClient.getDefaultCodecRegistry();
  369. BsonDocument bsonDoc = doc.toBsonDocument(Document.class, codecRegistry);
  370. if (bsonDoc.isArray(propertyName)) {
  371. type = FieldDataTypes.ARRAYLIST;
  372. } else if (bsonDoc.isBinary(propertyName)) {
  373. type = FieldDataTypes.BINARY;
  374. } else if (bsonDoc.isBoolean(propertyName)) {
  375. type = FieldDataTypes.BOOLEAN;
  376. } else if (bsonDoc.isDateTime(propertyName)) {
  377. type = FieldDataTypes.DATE_TIME;
  378. } else if (bsonDoc.isDecimal128(propertyName)) {
  379. type = FieldDataTypes.DECIMAL128;
  380. } else if (bsonDoc.isDocument(propertyName)) {
  381. type = FieldDataTypes.DOCUMENT;
  382. } else if (bsonDoc.isDouble(propertyName)) {
  383. type = FieldDataTypes.DOUBLE;
  384. } else if (bsonDoc.isInt32(propertyName)) {
  385. type = FieldDataTypes.INT32;
  386. } else if (bsonDoc.isInt64(propertyName)) {
  387. type = FieldDataTypes.INT64;
  388. } else if (bsonDoc.isNull(propertyName)) {
  389. type = FieldDataTypes.NULL;
  390. } else if (bsonDoc.isNumber(propertyName)) {
  391. type = FieldDataTypes.NUMBER;
  392. } else if (bsonDoc.isString(propertyName)) {
  393. type = FieldDataTypes.STRING;
  394. } else if (bsonDoc.isObjectId(propertyName)) {
  395. type = FieldDataTypes.OBJECTID;
  396. }
  397. return type;
  398. }
  399. public FieldDataTypes getCollectionPropertyBsonDataType(BsonDocument bsonDoc, String propertyName ) {
  400. FieldDataTypes type = null;
  401. if (bsonDoc.isArray(propertyName)) {
  402. type = FieldDataTypes.ARRAYLIST;
  403. }else if (bsonDoc.isBinary(propertyName)) {
  404. type = FieldDataTypes.BINARY;
  405. } else if (bsonDoc.isBoolean(propertyName)) {
  406. type = FieldDataTypes.BOOLEAN;
  407. } else if (bsonDoc.isDateTime(propertyName)) {
  408. type = FieldDataTypes.DATE_TIME;
  409. } else if (bsonDoc.isDecimal128(propertyName)) {
  410. type = FieldDataTypes.DECIMAL128;
  411. } else if (bsonDoc.isDocument(propertyName)) {
  412. type = FieldDataTypes.DOCUMENT;
  413. } else if (bsonDoc.isDouble(propertyName)) {
  414. type = FieldDataTypes.DOUBLE;
  415. } else if (bsonDoc.isInt32(propertyName)) {
  416. type = FieldDataTypes.INT32;
  417. } else if (bsonDoc.isInt64(propertyName)) {
  418. type = FieldDataTypes.INT64;
  419. } else if (bsonDoc.isNull(propertyName)) {
  420. type = FieldDataTypes.NULL;
  421. } else if (bsonDoc.isNumber(propertyName)) {
  422. type = FieldDataTypes.NUMBER;
  423. } else if (bsonDoc.isString(propertyName)) {
  424. type = FieldDataTypes.STRING;
  425. } else if (bsonDoc.isObjectId(propertyName)) {
  426. type = FieldDataTypes.OBJECTID;
  427. }
  428. return type;
  429. }
  430. private void addDateFormat(List<FieldMapping> fieldMappings) {
  431. if (fieldMappings != null && fieldMappings.size() > 0) {
  432. for (FieldMapping field : fieldMappings) {
  433. if (field.getFieldDataType() == FieldDataTypes.DATE) {
  434. // SampleRecordExtractor.checkDate(field);
  435. checkDate(field);
  436. } else if (field.getFieldDataType() == FieldDataTypes.TIMESTAMP) {
  437. // SampleRecordExtractor.checkTimeStamp(field);
  438. checkTimeStamp(field);
  439. }
  440. }
  441. }
  442. }
  443. public static Boolean checkDate(FieldMapping field) {
  444. Map<String, Integer> formatCounter = getSuitableFromats(field, FieldDataTypes.DATE);
  445. if (formatCounter.size() > 0) {
  446. String maxFormat = "";
  447. Integer max = 0;
  448. for (Entry<String, Integer> e : formatCounter.entrySet()) {
  449. String format = e.getKey();
  450. Integer counter = e.getValue();
  451. if (counter > max) {
  452. max = counter;
  453. maxFormat = format;
  454. }
  455. }
  456. if (StringUtils.isNotBlank(maxFormat)) {
  457. field.setFormat(maxFormat);
  458. field.setFieldDataType(FieldDataTypes.DATE);
  459. return true;
  460. }
  461. }
  462. if (field.getFieldDataType() == FieldDataTypes.DATE) {
  463. return true; // If header template says it is a date field, then it
  464. // is, even if there is no valid dates found
  465. }
  466. return false;
  467. }
  468. public static Boolean checkTimeStamp(FieldMapping field) {
  469. Map<String, Integer> formatCounter = getSuitableFromats(field, FieldDataTypes.TIMESTAMP);
  470. if (formatCounter.size() > 0) {
  471. String maxFormat = "";
  472. Integer max = 0;
  473. for (Entry<String, Integer> e : formatCounter.entrySet()) {
  474. String format = e.getKey();
  475. Integer counter = e.getValue();
  476. if (counter > max) {
  477. max = counter;
  478. maxFormat = format;
  479. }
  480. }
  481. if (StringUtils.isNotBlank(maxFormat)) {
  482. field.setFormat(maxFormat);
  483. field.setFieldDataType(FieldDataTypes.TIMESTAMP);
  484. return true;
  485. }
  486. }
  487. if (field.getFieldDataType() == FieldDataTypes.TIMESTAMP) {
  488. return true; // If header template says it is a TimeStamp field,
  489. // then it
  490. // is, even if there is no valid dates found
  491. }
  492. return false;
  493. }
  494. public static Map<String, Integer> getSuitableFromats(FieldMapping field, FieldDataTypes type) {
  495. List<String> allPossibleFormats = new ArrayList<String>();
  496. List<String> sampleRecordList = field.getSampleRecords();
  497. Iterator<String> iterator = sampleRecordList.iterator();
  498. while (iterator.hasNext()) {
  499. allPossibleFormats.addAll(DateUtils.findMyDateAndTSFormat(iterator.next(), type));
  500. }
  501. Map<String, Integer> formatCounter = new HashMap<String, Integer>();
  502. if (allPossibleFormats.size() > 0) {
  503. for (String format : allPossibleFormats) {
  504. if (!formatCounter.containsKey(format)) {
  505. formatCounter.put(format, 0);
  506. }
  507. formatCounter.put(format, formatCounter.get(format) + 1);
  508. }
  509. }
  510. return formatCounter;
  511. }
  512. public void getNestedDocumentAndArraysProperty(Document document, BsonDocument bdoc, FieldMapping fieldMapping , String propertyName) {
  513. System.out.println("recursive block start--");
  514. if (bdoc.isDocument(propertyName)) {
  515. System.out.println("document starts -----");
  516. BsonDocument bsonDoc = bdoc.getDocument(propertyName);
  517. int i =1;
  518. Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
  519. for (Entry<String, BsonValue> entry : entrySet) {
  520. FieldMapping nestedFieldMapping = new FieldMapping();
  521. String nestedPropertyName = entry.getKey();
  522. System.out.println("Key -: " + nestedPropertyName);
  523. long fieldId = i;
  524. nestedFieldMapping.setId(fieldId);
  525. nestedFieldMapping.setFieldName(nestedPropertyName);
  526. FieldDataTypes propertyDataType = getCollectionPropertyBsonDataType(bsonDoc, nestedPropertyName);
  527. System.out.println(propertyDataType.toString());
  528. Integer j = i - 1;
  529. Long pos = j.longValue();
  530. nestedFieldMapping.setPosition(pos);
  531. nestedFieldMapping.setFieldDataType(propertyDataType);
  532. if (bdoc.isDocument(nestedPropertyName)) {
  533. getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
  534. } else if (bdoc.isArray(entry.getKey())) {
  535. getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
  536. }
  537. fieldMapping.getListFieldMapping().add(nestedFieldMapping);
  538. i++;
  539. }
  540. System.out.println("document ends -----");
  541. } else if (bdoc.isArray(propertyName)) {
  542. System.out.println("Array starts----");
  543. BsonArray bsonArray = bdoc.getArray(propertyName);
  544. BsonValue bsonValue = bdoc.getArray(propertyName).get(0);
  545. System.out.println(bsonValue);
  546. if (bsonValue.isDocument()) {
  547. BsonDocument bsonDoc = bsonValue.asDocument();
  548. Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
  549. int i = 1;
  550. for (Entry<String, BsonValue> entry : bsonSet) {
  551. String nestedPropertyName = entry.getKey();
  552. FieldMapping nestedFieldMapping = new FieldMapping();
  553. System.out.println("Key -: " + nestedPropertyName);
  554. long fieldId = i;
  555. nestedFieldMapping.setId(fieldId);
  556. nestedFieldMapping.setFieldName(nestedPropertyName);
  557. FieldDataTypes propertyDataType = getCollectionPropertyDataType(document, nestedPropertyName);
  558. Integer j = i - 1;
  559. Long pos = j.longValue();
  560. nestedFieldMapping.setPosition(pos);
  561. nestedFieldMapping.setFieldDataType(propertyDataType);
  562. if (bsonDoc.isDocument(nestedPropertyName)) {
  563. getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
  564. } else if (bsonDoc.isArray(nestedPropertyName)) {
  565. getNestedDocumentAndArraysProperty(document, bsonDoc, nestedFieldMapping, nestedPropertyName);
  566. }
  567. fieldMapping.getListFieldMapping().add(nestedFieldMapping);
  568. i++;
  569. }
  570. } else if (bsonValue.isArray()) {
  571. getNestedDocumentAndArraysProperty(document, bdoc, fieldMapping, bsonValue.toString());
  572. }
  573. System.out.println("Array ends -----");
  574. }
  575. System.out.println("recursive block end--");
  576. }
  577. public void getNestedDocumentAndArrayValues(Document document, BsonDocument bdoc, FieldMapping fieldMapping , String propertyName, int count, int fileldMappingIndex) {
  578. System.out.println("sample recursive block start--");
  579. if (bdoc.isDocument(propertyName)) {
  580. System.out.println("sample document starts -----");
  581. BsonDocument bsonDoc = bdoc.getDocument(propertyName);
  582. int i =0;
  583. Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
  584. List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
  585. for (FieldMapping fieldMap : listFieldMapping) {
  586. for (Entry<String, BsonValue> entry : entrySet) {
  587. String nestedPropertyName = entry.getKey();
  588. if (nestedPropertyName.equals(fieldMap.getFieldName())) {
  589. if (bdoc.isDocument(nestedPropertyName)) {
  590. System.out.println(nestedPropertyName);
  591. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
  592. getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
  593. } else if (bdoc.isArray(entry.getKey())) {
  594. System.out.println(nestedPropertyName);
  595. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
  596. getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
  597. }else {
  598. System.out.println(entry.getValue().toString());
  599. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, getBsonPropertyValue(entry.getValue()));
  600. }
  601. }
  602. }
  603. i++;
  604. }
  605. System.out.println("sample document ends -----");
  606. }else if (bdoc.isArray(propertyName)) {
  607. System.out.println("Array starts----");
  608. BsonArray bsonArray = bdoc.getArray(propertyName);
  609. // List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
  610. for (BsonValue bsonValue : bsonArray) {
  611. System.out.println(bsonValue);
  612. if (bsonValue.isDocument()) {
  613. BsonDocument bsonDoc = bsonValue.asDocument();
  614. Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
  615. List<FieldMapping> listFieldMapping = fieldMapping.getListFieldMapping();
  616. int i = 0;
  617. for (FieldMapping fieldMap : listFieldMapping) {
  618. for (Entry<String, BsonValue> entry : bsonSet) {
  619. String nestedPropertyName = entry.getKey();
  620. if (nestedPropertyName.equals(fieldMap.getFieldName())) {
  621. if (bsonDoc.isDocument(nestedPropertyName)) {
  622. System.out.println(nestedPropertyName);
  623. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(nestedPropertyName);
  624. getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
  625. } else if (bsonDoc.isArray(nestedPropertyName)) {
  626. System.out.println(nestedPropertyName);
  627. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(count, nestedPropertyName);
  628. getNestedDocumentAndArraysProperty(document, bsonDoc, fieldMapping, nestedPropertyName);
  629. }else {
  630. System.out.println(entry.getValue().toString());
  631. fieldMapping.getListFieldMapping().get(i).getSampleRecords().add(getBsonPropertyValue(entry.getValue()));
  632. // fieldMapping.getListFieldMapping().get(i).getArraySampleRecords().add(fieldMapping.getListFieldMapping().get(i).getSampleRecords());
  633. }
  634. }
  635. }
  636. i++;
  637. }
  638. } else if (bsonValue.isArray()) {
  639. getNestedDocumentAndArraysProperty(document, bdoc, fieldMapping, bsonValue.toString());
  640. }
  641. }
  642. for (int i = 0; i < fieldMapping.getListFieldMapping().size(); i++) {
  643. fieldMapping.getListFieldMapping().get(i).getArraySampleRecords().add(fieldMapping.getListFieldMapping().get(i).getSampleRecords());
  644. fieldMapping.getListFieldMapping().get(i).setSampleRecords(new ArrayList<String>());
  645. }
  646. System.out.println("Array ends -----");
  647. }
  648. }
  649. public String getBsonPropertyValue( BsonValue bsonValue) {
  650. String value = null;
  651. if (bsonValue.isBinary()) {
  652. value = bsonValue.asBinary().getData().toString();
  653. }else if (bsonValue.isString()) {
  654. value = bsonValue.asString().getValue();
  655. }else if (bsonValue.isDocument()) {
  656. value = bsonValue.asDocument().toString();
  657. }else if (bsonValue.isArray()) {
  658. StringBuilder sb = new StringBuilder();
  659. for (BsonValue bValue : bsonValue.asArray().getValues()) {
  660. sb.append(",");
  661. sb.append(getBsonPropertyValue(bValue));
  662. }
  663. value = sb.toString().replaceFirst(",", "");
  664. }else if (bsonValue.isDouble()) {
  665. value = Double.toString(bsonValue.asDouble().getValue());
  666. }else if (bsonValue.isDateTime()) {
  667. value = Long.toString(bsonValue.asDateTime().getValue());
  668. }else if (bsonValue.isInt32()) {
  669. value = Integer.toString(bsonValue.asInt32().getValue());
  670. }
  671. return value;
  672. }
  673. public static void main(String[] args) {
  674. // MongoClient mongo = new MongoClient("localhost", 27017);
  675. List<String> dbs = null;
  676. DB dbss = null;
  677. // try {
  678. MongoClient mongo = new MongoClient(new MongoClientURI("mongodb://admin:mongodb@localhost:27017"));
  679. // MongoClient mongo = new MongoClient();
  680. dbss = mongo.getDB("hotel");
  681. CommandResult result = dbss.getStats();
  682. MongoDatabase db = mongo.getDatabase("hotel");
  683. CodecRegistry codecRegistry = mongo.getDefaultCodecRegistry();
  684. // System.out.println("loop start -j-");
  685. // dbs = mongo.getDatabaseNames();
  686. MongoIterable<String> dbIterable = mongo.listDatabaseNames();
  687. MongoCursor<String> mongoCursor = dbIterable.iterator();
  688. // while (mongoCursor.hasNext()) {
  689. // System.out.println(mongoCursor.next());
  690. // }
  691. // System.out.println("loop end --");
  692. /*MongoIterable<String> mongoCollectionIterable = db.listCollectionNames();
  693. MongoCursor<String> mongoCollectionCursor = mongoCollectionIterable.iterator();*/
  694. // while (mongoCollectionCursor.hasNext()) {
  695. // String tableName = mongoCollectionCursor.next();
  696. MongoCollection<Document> mongoCollection = db.getCollection("restaurants");
  697. // List<Document> documents = mongoCollection.find().limit(1).into(new ArrayList<Document>());
  698. // System.out.println("document start -----");
  699. // for (Document document : documents) {
  700. // System.out.println(document);
  701. // }
  702. // System.out.println("document end -----");
  703. /*ListIndexesIterable<Document> indexIterable = db.getCollection(tableName).listIndexes();
  704. MongoCursor<Document> indexMongoCursor = indexIterable.iterator();
  705. while (indexMongoCursor.hasNext()) {
  706. System.out.println(indexMongoCursor.next());
  707. }*/
  708. FindIterable<Document> findCollectionIterable = mongoCollection.find().limit(1);
  709. MongoCursor<Document> cursorFileds = findCollectionIterable.iterator();
  710. while (cursorFileds.hasNext()) {
  711. // System.out.println(cursorFileds.next());
  712. Document dom = cursorFileds.next();
  713. Set<Entry<String, Object>> setField = dom.entrySet();
  714. for (Entry<String, Object> entry : setField) {
  715. // System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
  716. // System.out.println(dom.get(entry.getKey()));
  717. BsonDocument bdoc = dom.toBsonDocument(Document.class, codecRegistry);
  718. // System.out.println(bdoc.isDocument(entry.getKey()));
  719. if (bdoc.isDocument(entry.getKey())) {
  720. // recursiveDocument(dom,bdoc, entry.getKey());
  721. } else if (bdoc.isArray(entry.getKey())) {
  722. // recursiveDocument(dom,bdoc, entry.getKey());
  723. }
  724. }
  725. }
  726. // }
  727. MongoDBScanner scan = new MongoDBScanner();
  728. List<FieldMapping> listSample = null;
  729. List<FieldMapping> fieldMappings = scan.getFieldMapping("hotel", "restaurants", mongo);
  730. try {
  731. listSample = scan.addSampleData(fieldMappings, mongo, "hotel", "restaurants", 2);
  732. } catch (Exception e) {
  733. // TODO Auto-generated catch block
  734. e.printStackTrace();
  735. }
  736. for (FieldMapping fieldMapping : listSample) {
  737. System.out.println(fieldMapping.getFieldName());
  738. }
  739. }
  740. static public void recursiveDocument(Document dom,BsonDocument bdoc, String key) {
  741. System.out.println("recursive block start--");
  742. if (bdoc.isDocument(key)) {
  743. System.out.println("document starts -----");
  744. BsonDocument bsonDoc = bdoc.getDocument(key);
  745. Set<Entry<String, BsonValue>>entrySet = bsonDoc.entrySet();
  746. for (Entry<String, BsonValue> entry : entrySet) {
  747. System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
  748. if (bdoc.isDocument(entry.getKey())) {
  749. recursiveDocument(dom, bsonDoc, entry.getKey());
  750. } else if (bdoc.isArray(entry.getKey())) {
  751. recursiveDocument(dom, bsonDoc, entry.getKey());
  752. }
  753. }
  754. System.out.println("document ends -----");
  755. } else if (bdoc.isArray(key)) {
  756. System.out.println("Array starts----");
  757. BsonArray bsonArray = bdoc.getArray(key);
  758. for (BsonValue bsonValue : bsonArray) {
  759. System.out.println(bsonValue);
  760. if (bsonValue.isDocument()) {
  761. BsonDocument bsonDoc = bsonValue.asDocument();
  762. Set<Entry<String, BsonValue>> bsonSet = bsonDoc.entrySet();
  763. for (Entry<String, BsonValue> entry : bsonSet) {
  764. System.out.println("Key -: " + entry.getKey() + " value -: " + entry.getValue());
  765. if (bsonDoc.isDocument(entry.getKey())) {
  766. recursiveDocument(dom, bsonDoc, entry.getKey());
  767. } else if (bsonDoc.isArray(entry.getKey())) {
  768. recursiveDocument(dom, bsonDoc, entry.getKey());
  769. }
  770. }
  771. recursiveDocument(dom, bsonDoc, bsonValue.toString());
  772. } else if (bsonValue.isArray()) {
  773. recursiveDocument(dom, bdoc, bsonValue.toString());
  774. }
  775. }
  776. System.out.println("Array ends -----");
  777. }
  778. System.out.println("recursive block end--");
  779. }
  780. static String getBsonValue(BsonDocument bsonDoc, String key) {
  781. String value = null;
  782. return value;
  783. }
  784. }