/MercerPoC/src/in/lnt/validations/FormValidator.java

package in.lnt.validations;



import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.io.UnsupportedEncodingException;

import java.sql.SQLException;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Date;

import java.util.HashMap;

import java.util.HashSet;

import java.util.Iterator;

import java.util.LinkedHashMap;

import java.util.List;

import java.util.Map;

import java.util.Map.Entry;

import java.util.Set;

import java.util.UUID;

import java.util.regex.Matcher;

import java.util.regex.Pattern;



import org.apache.commons.lang3.EnumUtils;

import org.apache.commons.lang3.StringUtils;

import org.json.JSONArray;

import org.json.JSONException;

import org.json.JSONObject;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



import com.fasterxml.jackson.core.JsonParseException;

import com.fasterxml.jackson.core.JsonProcessingException;

import com.fasterxml.jackson.databind.JsonMappingException;

import com.fasterxml.jackson.databind.JsonNode;

import com.fasterxml.jackson.databind.ObjectMapper;

import com.fasterxml.jackson.databind.node.ArrayNode;

import com.fasterxml.jackson.databind.node.JsonNodeFactory;

import com.fasterxml.jackson.databind.node.JsonNodeType;

import com.fasterxml.jackson.databind.node.NullNode;

import com.fasterxml.jackson.databind.node.ObjectNode;

import com.lti.mosaic.function.def.InversionAndRangeCheckFunctions;

import com.lti.mosaic.parser.driver.ExpressionEvalutionDriver;

import com.lti.mosaic.parser.utils.ExpressionBuilderConstants;



import in.lnt.constants.Constants;

import in.lnt.enums.ValidationTypes;

import in.lnt.exceptions.CustomStatusException;

import in.lnt.ml.MachineLearningRestClient;

import in.lnt.parser.CSVParser;

import in.lnt.service.db.DBUtils;

import in.lnt.utility.constants.CacheConstats;

import in.lnt.utility.constants.ErrorCacheConstant;

import in.lnt.utility.general.Cache;

import in.lnt.utility.general.DataUtility;

import in.lnt.utility.general.JsonUtils;

import in.lnt.validations.evaluator.APIExpressionEvaluator;

import in.lti.mosaic.api.mongo.MongoObject;

import in.lti.mosaic.api.mongo.MongoThread;



public class FormValidator {

	private static final Logger logger = LoggerFactory.getLogger(FormValidator.class);

	static ObjectMapper mapper = new ObjectMapper();

	ArrayList<String> paramValueList=null;

	static Pattern VALID_EMAIL_ADDRESS_REGEX = 

		    Pattern.compile("^(.+)@(.+)$", Pattern.CASE_INSENSITIVE);

	static Pattern VALID_PHONE_NUMBER_REGEX = 

		    Pattern.compile("^(\\(?\\+?[0-9]*\\)?)?[0-9_\\- \\(\\)]*$", Pattern.CASE_INSENSITIVE);

	// ^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$

	

	private static String MONGODB_ARCHIVE_FLAG = Cache

			.getMLProperty(in.lnt.utility.constants.CacheConstats.MONGODUMP_FLAG);

	

	

	public JsonNode parseAndValidate(String entitiesJson, String csvJsonString, boolean isAggregateRequest,

			Map<String, String> xslJsonMap, String firstSheetName, boolean isMultiPartRequest) throws Exception  {

		JsonNode resultNode = null;

		

	    //Generating UUID for a Request and Response

	    final String uuid = UUID.randomUUID().toString().replace("-", "");

		

		JsonNode entitiesNodeResponse = preparingL1Input(entitiesJson, csvJsonString, isAggregateRequest, xslJsonMap,

				firstSheetName, isMultiPartRequest, null);

		

		//Adding Request into MongoDB

		dumpToMongo(uuid, mapper.writeValueAsString(entitiesNodeResponse), Constants.REQUEST);



		if (isAggregateRequest) {

			resultNode = performValidateAggregateRequest(entitiesNodeResponse, isAggregateRequest, isMultiPartRequest);

		} else

			resultNode = perform_L1_L2Validations(entitiesNodeResponse, isAggregateRequest, isMultiPartRequest);

		

		//Adding Response into MongoDB

		dumpToMongo(uuid, mapper.writeValueAsString(resultNode), Constants.RESPONSE);



		return resultNode;

	}



	/**

	 * @param uuid

	 * @param data

	 * @throws JsonProcessingException

	 * @author Vivek Kasadwar

	 */

	public static void dumpToMongo(final String uuid, String data,String type){

		if (StringUtils.equals(MONGODB_ARCHIVE_FLAG,Constants.YES)) {

			MongoObject mongoObject = new MongoObject();

			mongoObject.setRequestId(uuid);

			mongoObject.setObjectType(type);

			mongoObject.setObjectJson(data);

			mongoObject.setTimeStamp(Calendar.getInstance().getTime());

			MongoThread.addToQueue(mongoObject);

		}

	}

	

	public HashMap<String, JsonNode> parseJsonObjectMap(String key, JsonNode value, HashMap<String, JsonNode> map,

			String datakey, JsonNode columnListJson,List<String> otherAndContextKeys) throws Exception {		// PE-7056

		Iterator<Map.Entry<String, JsonNode>> it = value.fields();

		Map.Entry<String, JsonNode> entry = null;

		try {

			while (it.hasNext()) {

				entry = it.next();

				if (entry.getValue().getNodeType() != JsonNodeType.OBJECT) {

					if (datakey != null) {

						map.put((datakey + "." + key + "." + entry.getKey()).trim(), entry.getValue());

						if(otherAndContextKeys!=null)

						otherAndContextKeys.add((datakey + "." + key + "." + entry.getKey()).trim());

					} else {

						map.put((key + "." + entry.getKey()).trim(), entry.getValue());

						if(otherAndContextKeys!=null)

						otherAndContextKeys.add((key + "." + entry.getKey()).trim());

					}	

				} else {

					parseJsonObjectMap(key + "." + entry.getKey(), entry.getValue(), map, datakey, columnListJson,otherAndContextKeys);

				}

			}

		} finally {

			it=null;

			if(null!=entry)	entry=null;

		}

		return map;

	}



	public JsonNode validate(HashMap<String, JsonNode> dataMap, ArrayList<JsonNode> columnNodeList, JsonNode dataNode,

			 APIExpressionEvaluator apiExpressionEvaluator, boolean isAggregateRequest,

			HashMap<String, String> columnMappingMap,HashMap<String, JsonNode> dimMap, List<String> metaDataColumnsOnly,

			boolean mlAutoCoorect) throws Exception {

		

		logger.debug("validate >> ExpressionEvaluatorDriver Cache : {}",

				apiExpressionEvaluator.getExpressionEvaluatorDriver().getAggregateFunctionCache());



		JsonNode validationNode = null;

		ObjectNode obj = null;

		ArrayList<ObjectNode> predefinedObjList = null;

		List<ObjectNode> rangeOrSeqObjList = null;

		String code = null;

		String mappedColumnName = null;

		ArrayNode arr = null;

		JsonNode nodeDTObject = null;

		JsonNode dnCodeObject = null;

		Boolean flag = null;

		String result = null;

		String skip = null;

		JsonNode validationObjNode = null;

		ObjectNode jobInversionObjNode = null;

		ArrayNode jobInversionArr = null;

		// PE-5643 by default true because for true type do not fire validations.

		boolean isOptionType = true;

		String countryCode = null;

		String __COMPENSATION = null;

		String jobInvParameter = null;



		// PE-5691

		boolean isEmtpyRecord = false;



		// PE - 7257

		ArrayNode datePatternArray = null;

		ArrayNode numberPatternArray = null;

		String columnName = null;

		JsonNode tempOriginalValue = null;

		// 8421 code changes.

		Map<String, String> incorrectDataType = new HashMap<>();

		JsonNode questionType = null; // Added for 8916 Remove %

		String dataType = null;

			if (columnNodeList != null && !columnNodeList.isEmpty()) {

				arr = mapper.createArrayNode();

				if (!JsonUtils.isNullOrBlankOrNullNode(dataMap.get(Constants.CONTEXTDATAKEY+Constants.CONTEXTDATA_DATEFORMAT))){

					datePatternArray = (ArrayNode)dataMap.get(Constants.CONTEXTDATAKEY+Constants.CONTEXTDATA_DATEFORMAT);

				}

				if (!JsonUtils.isNullOrBlankOrNullNode(dataMap.get(Constants.CONTEXTDATAKEY+Constants.CONTEXTDATA_NUMBERFORMAT))){

					numberPatternArray = (ArrayNode)dataMap.get(Constants.CONTEXTDATAKEY+Constants.CONTEXTDATA_NUMBERFORMAT);

				}

				

				// If request is AggregateRequest

				// L3A - Inversion/Aggregate Request Starts Here

				if (isAggregateRequest) {

					// PE - 7257

					for (JsonNode node : columnNodeList) {

						if (node.get(Constants.VALIDATIONS) != null && node.get(Constants.VALIDATIONS).size() > 0) {



							for (int i = 0; i < node.get(Constants.VALIDATIONS).size(); i++) {



								validationNode = node.get(Constants.VALIDATIONS).get(i);



								// For Inversion

								obj = apiExpressionEvaluator.expressionEvaluator(dataMap, validationNode, node,

										dataNode,isAggregateRequest, columnMappingMap, columnNodeList, metaDataColumnsOnly);

								if (obj != null) {

									arr.add(obj);

									obj = null;

								}

							}

						}

					}



					// PE-4843 Append JobInversion Object

					

					

					

					InversionAndRangeCheckFunctions invAndRangeChkFuncObj 

							= new InversionAndRangeCheckFunctions(CSVParser.dataForAggregation, apiExpressionEvaluator.getExpressionEvaluatorDriver().getOtherData());  

					Map<String, Object> requestMap =  invAndRangeChkFuncObj.jobInversion();  // Changed by Nikhil

					

					if (requestMap.containsKey(ExpressionBuilderConstants.CONST_INVERSION_JSON_ARRAY)

							&& null != requestMap.get(ExpressionBuilderConstants.CONST_INVERSION_JSON_ARRAY)) {

						jobInversionArr = (ArrayNode) mapper.readTree(

										requestMap.get(ExpressionBuilderConstants.CONST_INVERSION_JSON_ARRAY).toString());

					}



					if(requestMap.containsKey(ExpressionBuilderConstants.CONST_COUNTRYCODE) 

							&& null!=requestMap.get(ExpressionBuilderConstants.CONST_COUNTRYCODE)) {

						countryCode = requestMap.get(ExpressionBuilderConstants.CONST_COUNTRYCODE).toString();

					}

					

					logger.info("Creating JobInversion object...");

					jobInversionObjNode = mapper.createObjectNode();

					

					if(requestMap.containsKey(ExpressionBuilderConstants.CONST_COMPENSATION) 

							&& null!=requestMap.get(ExpressionBuilderConstants.CONST_COMPENSATION)) {

						__COMPENSATION = requestMap.get(ExpressionBuilderConstants.CONST_COMPENSATION).toString();

						

						if(__COMPENSATION.equals(ExpressionBuilderConstants.SALARY_TYPE_TOTAL_GUARANTEED_CASH)) {

							__COMPENSATION = ExpressionBuilderConstants.COL_TOTAL_GUARANTEED_CASH;

							jobInvParameter = ExpressionBuilderConstants.SALARY_TYPE_MSG_TOTAL_GUARANTEED_CASH;



							//PE-8454

							jobInversionObjNode.put(Constants.MESSAGE_KEY, Constants.JOB_INVERSION_ERROR_TOTAL_CASH);

							



						} else {

							__COMPENSATION = ExpressionBuilderConstants.COL_ANNUAL_BASE;

							jobInvParameter = ExpressionBuilderConstants.SALARY_TYPE_MSG_REGULAR;

							

							//PE-8454

							jobInversionObjNode.put(Constants.MESSAGE_KEY, Constants.JOB_INVERSION_ERROR_BASE_SALARY);

							



						}

					}

				

					jobInversionObjNode.put(Constants.ERROR_GROUP, Constants.PERSONAL);

					jobInversionObjNode.put(Constants.ERROR_TYPE, Constants.ERROR);

					jobInversionObjNode.put(Constants.VALIDATION_TYPE, Constants.EXPRESSION);

					jobInversionObjNode.put(Constants.FIELD, __COMPENSATION);

					jobInversionObjNode.put(Constants.CATEGORY, Constants.JOB_INVERSION);

					jobInversionObjNode.put(Constants.PARAMETER, jobInvParameter);

					

					jobInversionObjNode.putArray(Constants.DATA).addAll(jobInversionArr);

					

					arr.add(jobInversionObjNode);

					

					return mapper.convertValue(arr, JsonNode.class);

				}

				

				// L3A - Inversion/Aggregate Request Ends Here



				for (JsonNode node : columnNodeList) {

					JsonNode currentDataMapCol = null;

					isEmtpyRecord=false;

					if (node.get(Constants.CODE) != null)

						code = node.get(Constants.CODE).asText();

					if (node.get(Constants.SKIP) != null)

						skip = node.get(Constants.SKIP).asText();

					if (node.get(Constants.MAPPED_COLUMN_NAME) != null)

						mappedColumnName = node.get(Constants.MAPPED_COLUMN_NAME).asText();

					

					if ( dataNode != null || (dataNode!=null && dataNode.get(mappedColumnName) != null)) {

						

						//Following code written for Mercer JIRA Story PE-5691



						if(skip==null ) {

							if(code !=null && mappedColumnName!=null && JsonUtils.isNullOrBlankOrNullNode(dataNode.get(code))

									&&  JsonUtils.isNullOrBlankOrNullNode(dataNode.get(mappedColumnName))) 

							{

								

								isEmtpyRecord = true;

							}

							if(code !=null && mappedColumnName ==null && JsonUtils.isNullOrBlankOrNullNode(dataNode.get(code)) )

							{

								isEmtpyRecord = true;

							}

							if(mappedColumnName !=null && code==null &&JsonUtils.isNullOrBlankOrNullNode(dataNode.get(mappedColumnName)))

							{

								isEmtpyRecord = true;

							}

							if(isEmtpyRecord)

							{

								((ObjectNode)dataNode).set(code, JsonNodeFactory.instance.nullNode()) ;  

							}

							}

						//PE-5691 End.	

						flag = false;

						if ((mappedColumnName != null && dataNode.get(mappedColumnName) != null)

								&& (null != code && mappedColumnName != null && !code.equals(mappedColumnName))) {

							// Changing logic

 							((ObjectNode) (dataNode)).set(code, dataMap.get(mappedColumnName));

							((ObjectNode) (dataNode)).remove(mappedColumnName);

						}

						if (skip != null && (skip.equals("true") || Boolean.parseBoolean(skip) == true)) {

							if (code != null && !code.equals(""))

								((ObjectNode) (dataNode)).remove(code);

							else

								((ObjectNode) (dataNode)).remove(mappedColumnName);

							skip = null;

							continue;

						}

						// Validate the data type object block starts here.

						if(null != node.get(Constants.DATA_TYPE)){

							dataType = node.get(Constants.DATA_TYPE).asText();

						}

						if (dataType != null) {

							logger.info("Check if  dataType is not empty");

							nodeDTObject = node.get(Constants.DATA_TYPE);

							if (dataNode.get(code) != null)

								dnCodeObject = dataNode.get(code);

							else if (dataNode.get(mappedColumnName) != null)

								dnCodeObject = dataNode.get(mappedColumnName);

							

							validationObjNode = node.get(Constants.VALIDATIONS);

							

							if ( StringUtils.isEmpty(mappedColumnName)) {

								columnName = code;

							}else {

								columnName = mappedColumnName;

							}

							

							if(null != dataMap && !JsonUtils.isNullOrBlankOrNullNode(dataMap.get(columnName))){

								currentDataMapCol = dataMap.get(columnName);

							}

							//Added for 8916 Remove %

							 questionType = node.get(Constants.QUESTION_TYPE);

						     if(null != currentDataMapCol  && dataNode.get(columnName) != null

						    	&&	 !JsonUtils.isNullOrBlankOrNullNode(questionType) && questionType.asText().equals(Constants.PERCENTAGE) ){

						    	 String percentageValue = currentDataMapCol.asText().trim();

						    	 if(!percentageValue.isEmpty() && percentageValue.endsWith("%")){

						    		 percentageValue = percentageValue.substring(0, percentageValue.length()-1);

						    		 dataMap.put(columnName, 

						    				 mapper.readValue(mapper.writeValueAsString(percentageValue),JsonNode.class));

						    		 ((ObjectNode) dataNode).put(code, percentageValue);

						    		 dnCodeObject = dataNode.get(columnName);

						    	 }

						    	 }

						     

				          // If data node is not empty then do data type validation

							if (!isAggregateRequest && !JsonUtils.isNullOrBlankOrNullNode(dnCodeObject)  

									&& !isEmtpyRecord && !StringUtils.isEmpty(dnCodeObject.asText().trim())) { //PE-9268

								//8421 code changes.

								incorrectDataType = dataTypeCheck(dataMap, dataNode, columnName, nodeDTObject, dnCodeObject, flag,

										datePatternArray, numberPatternArray);

								flag = Boolean.parseBoolean(incorrectDataType.get("dataTypeFlag"));

							} else {

								flag = true;

							}

						} else { // if data node is empty then skip data type validations.

							flag = true;

						}

						

						//PE -5216 and PE-5643 Check for drop down options exist in metadata or in database.

						// PE-5643 : Required this functionality at /uploadMulttipleFile also. (Mentioned by Alex)

						if (questionType != null

								&& ( questionType.asText().equals(Constants.DROPDOWN) || 

										questionType.asText().equals(Constants.RADIO_BUTTONS) )) {

							

							

							// PE-8070 Clear & Prepare outputJson for dimensions

							if( !JsonUtils.isNullOrBlankOrNullNode(dataNode.get(code)) 

									&& dataNode.get(code).getNodeType()==JsonNodeType.OBJECT) {

								Iterator<String> iter = dataNode.get(code).fieldNames();

								String dimensionKey = null;

								while(null!=iter && iter.hasNext() ) {

									dimensionKey = iter.next();

	   								isOptionType =  FormValidator_CSC.checkAnswerExists(node, dataNode.get(code).get(dimensionKey)); 

	   									   								

	   								if ( ! isOptionType ) {

		   								

//	   									PE-8397

//	   									// Implementation start PE : 5643	

//	   									// Clear the value anyways if checkAnswerExists==false

//	   										

//		   								// Implementation end PE : 5643

	   									

	   									// Generate error for dimension in format : Question1.Dimension1

	   									obj = APIExpressionEvaluator.prepareOutputJson(null, null, null, ValidationTypes.DROPDOWN,

	   										code+"."+dimensionKey, null, null,node.has(Constants.DISPLAYLABEL) ? node.get(Constants.DISPLAYLABEL).asText() : null);

	   								}

	   								if (obj != null) {

	   									arr.add(obj);

	   									obj = null;

	   								}

								}



							// Clear & Prepare outputJson for non-dimensional questions (i.e. simple values)

							} else {

								// PE-6524 ML Integration works start

								if(mlAutoCoorect )

								{

									isOptionType =  FormValidator_CSC.checkAnswerExists(node, dataMap.get(code));

								}

								else

								{

									isOptionType =  FormValidator_CSC.checkAnswerExists(node, dataNode.get(code));

								}

								

								// PE-6524 ML Integration works end

								if ( ! isOptionType ) {

									

									obj = APIExpressionEvaluator.prepareOutputJson(null, null, null, ValidationTypes.DROPDOWN,

										code, null, null,JsonUtils.isNullOrBlankOrNullNode(node.get(Constants.DISPLAYLABEL))? "":node.get(Constants.DISPLAYLABEL).asText() );

								}

	

								if (obj != null) {

									arr.add(obj);

									obj = null;

								}

								

								// PE-6254 ML Integration works end

							}

						}

						

						//PE-9490 Round off upto 2 precison places for money question type

						if(!JsonUtils.isNullOrBlankOrNullNode(questionType) && questionType.asText().equals(Constants.MONEY) && flag

								&& null != dataMap && !JsonUtils.isNullOrBlankOrNullNode(currentDataMapCol) && dataNode.get(columnName) != null) {

							String colStringVal = currentDataMapCol.asText().trim();

							double colDoubleVal = currentDataMapCol.asDouble();

							if(colStringVal.contains(".")) {

								//RoundOff the value and return error object if it has more than 2 precison places

								obj = DataUtility.roundOffMoneyQuestionType(dataMap, dataNode, obj, code, dataType, colStringVal, colDoubleVal);

							}

						}

						

						tempOriginalValue = dataNode.get(code);

						if(mlAutoCoorect && !isAggregateRequest)

						{	

							if(!JsonUtils.isNullOrBlankOrNullNode(dataNode.get(code+Constants.UNDERSCORE_ACFLAG)) && dataNode.get(code+Constants.UNDERSCORE_ACFLAG).asText().equalsIgnoreCase("R"))

							{

								arr.add(prepareMLAutocorrectObject(code,"validationType",tempOriginalValue));

								if( !JsonUtils.isNullOrBlankOrNullNode(dataMap.get(code)) ) {	// Handled NullPointerExc

									((ObjectNode) (dataNode)).put(code,dataMap.get(code).asText());

								} else {

									((ObjectNode) (dataNode)).putNull(code);								

								}

							}

						} 



 						if (node.get(Constants.VALIDATIONS) != null && node.get(Constants.VALIDATIONS).size() > 0

								&& flag) { // Aggregate

							for (int i = 0; i < node.get(Constants.VALIDATIONS).size(); i++) {



								validationNode = node.get(Constants.VALIDATIONS).get(i);

								if (validationNode.get(Constants.VALIDATION_TYPE) != null

										&& EnumUtils.isValidEnum(ValidationTypes.class,

												validationNode.get(Constants.VALIDATION_TYPE).asText().toUpperCase() )) {



									switch (ValidationTypes.valueOf(validationNode.get(Constants.VALIDATION_TYPE).asText().toUpperCase())) {

									case REQUIRED:

										// PE - 7241

										predefinedObjList = validatePredefinedValidation(dataNode,node,validationNode,dataMap,mappedColumnName,Constants.REQUIRED);

										break;

									case ONEOF:

										obj = apiExpressionEvaluator.checkValidationTypeOneOf(dataMap, code, node,

												validationNode, mappedColumnName);

										break;

									case RANGE:

										rangeOrSeqObjList = apiExpressionEvaluator.prepareRangeObject(dataMap, validationNode, node,

												dataNode);

										break;

									case RANGEVALIDATIONREFTABLE:

										rangeOrSeqObjList = apiExpressionEvaluator.prepareRefernceRangeObject(dataMap, validationNode,

												node, dataNode);

										break;

									case ELIGIBILITY:

										// Last Parameter is weather request of  MDA

										arr.addAll(apiExpressionEvaluator.checkValidity(dataMap, validationNode, node,

												dataNode,false,metaDataColumnsOnly));

										break;

									case PHONE:

										// PE - 7241

										predefinedObjList = validatePredefinedValidation(dataNode,node,validationNode,null,mappedColumnName,Constants.PHONE);

										break;

									case EMAIL:

										// PE - 7241

										predefinedObjList = validatePredefinedValidation(dataNode,node,validationNode,null,mappedColumnName,Constants.EMAIL);

										break;

									case EXPRESSION:

										obj = apiExpressionEvaluator.expressionEvaluator(dataMap, validationNode, node,	dataNode,

												isAggregateRequest, columnMappingMap, columnNodeList, metaDataColumnsOnly);

										

										break;

										

									/**

									 * @author Akhileshwar

									 * PE-7045 

									 **/

									case SEQUENTIALCHECK://PE-7989 metaDataColumnsOnly added in method signature

										rangeOrSeqObjList = apiExpressionEvaluator.performSequentialCheck(dataMap, validationNode, node,

												dataNode, dimMap,apiExpressionEvaluator.getColumnDataTypeMapping(), true, metaDataColumnsOnly);

										break;

									default:

									}

								}else {

									result = Constants.MANDATORY_VALIDATIONTYPE;

									obj = APIExpressionEvaluator.prepareOutputJson(null, null, result, null, null, null,

											null, null);

								}

								if (obj != null) { 

									arr.add(obj);

									obj = null;

								}

								if(rangeOrSeqObjList!=null)

								{

									for(int j=0;j<rangeOrSeqObjList.size();j++)

									{

										arr.add(rangeOrSeqObjList.get(j));

									}

									rangeOrSeqObjList=null;

								}

								if((null != predefinedObjList) && (predefinedObjList.size() > 0)){

									arr.addAll(predefinedObjList);

									predefinedObjList = null;

								}

							}

						} else if (!isAggregateRequest && flag == false) {

							logger.info("{}","Check if flag is false");

							//8421 code changes.

							if(null != dataType && !incorrectDataType.containsKey(dataType)){

								if(Constants.INT.equals(dataType)){

									result = Cache.getPropertyFromError(ErrorCacheConstant.ERR_061);

								}else if(Constants.DOUBLE.equals(dataType)){

									result = Cache.getPropertyFromError(ErrorCacheConstant.ERR_062);

								}else if(Constants.DATE.equals(dataType)){

									result = Cache.getPropertyFromError(ErrorCacheConstant.ERR_063);

								}else{

									

									result = Cache.getPropertyFromError(ErrorCacheConstant.ERR_043);

								}

							}

							

							if (null != nodeDTObject && null != dnCodeObject) {

								obj = APIExpressionEvaluator.prepareOutputJson(validationObjNode, node, result,

										ValidationTypes.DATATYPEERROR, dnCodeObject.asText(), nodeDTObject.asText(),

										nodeDTObject.asText(), null);

								

							}



							if (obj != null) {

								arr.add(obj);

								obj = null;

							}



						}

 						

 						// PE-6874 : Fixed by shifting this code down here

						//Following code written for Mercer JIRA Story PE-5691

						// If data node code value is not present then set to Blank and Data map also.

						// PE-7071 : Remove CODE if there is no validation OR validation_errorType != "AUTOCORRECT"

						if(isEmtpyRecord && 

								(validationNode==null || 	

								(null!=validationNode && !JsonUtils.isNullOrBlankOrNullNode(validationNode.get(Constants.ERROR_TYPE)) 

									&& !(validationNode.get(Constants.ERROR_TYPE).asText()).equalsIgnoreCase("AUTOCORRECT")))) {

							((ObjectNode)dataNode).remove(code); 

						}

						//PE-5691 End.	

 							

						// PE : 2449 to create auto-correct object for missing EEID

 						

						if ((code!=null &&code.equalsIgnoreCase(Constants.EMPLOYEE_EEID)) || 

								(mappedColumnName !=null && mappedColumnName.equalsIgnoreCase(Constants.EMPLOYEE_EEID)))

							obj = prepareEeidAutoCorrectObject(code, dataMap, dataNode,columnMappingMap);



						if (obj != null) {

							arr.add(obj);

							obj = null;

						}



						if (node.get("questionType") != null

								&& node.get("questionType").asText().equals("checkboxes")) {

							buildCheckboxArray(dataNode,code, mappedColumnName);

						}

						

					}

					

					code = null;

					validationNode = null;

				}

				// PE-6254 ML Integration work

				if(mlAutoCoorect && !isAggregateRequest)

				{	

					removeDataNodeACFields(dataNode);

				}

				if ((arr != null && arr.size() > 0)) {

					if (!isAggregateRequest) {

						/*

						 * if(dataNode.get(k).get(code)!=null) ((ObjectNode)

						 * (dataNode.get(k))).set(code, dataNode.get(k).get(code));

						 */

							

						((ObjectNode) (dataNode)).set(Constants.VALIDATION_ERRORS, arr);

						arr = null;

					} else { // Aggregate

						JsonNode objectNode = mapper.convertValue(arr, JsonNode.class);

						return objectNode;

					}

				}



			}

		

		return dataNode;

	}



	/**

	 * @param dataNode

	 * @param mappedColumnName

	 * @param nodeDTObject

	 * @param dnCodeObject

	 * @param flag

	 * @param datePatternArray

	 * @param numberPatternArray

	 * @return

	 */

	protected Map<String,String> dataTypeCheck(HashMap<String, JsonNode> dataMap, JsonNode dataNode, String mappedColumnName, JsonNode nodeDTObject,

			JsonNode dnCodeObject, Boolean flag, ArrayNode datePatternArray, ArrayNode numberPatternArray) {

		

		String dnObjetText;

		Map<String,String> incorrectDataTypeMap = new HashMap<>();

		boolean isNullNode = NullNode.class.equals(dnCodeObject.getClass());

		if (!isNullNode) {

			dnObjetText = dnCodeObject.asText();

		}else {

			dnObjetText = null;

		}

		boolean dataTypeFlag = flag;

		String incorrectintegerDataType = "";

		

		boolean isDateArray = false;

		boolean isNumberArray = false;

		

		// 7257  For Multi-part request check following arrays

		if ( null != datePatternArray &&  datePatternArray.size() > 0 ) {

			isDateArray  = true ;

		}

		

		if ( null != numberPatternArray && numberPatternArray.size() > 0 ) {

			isNumberArray = true;

		}

		// Check for integer,boolean,double ,date and string data type object.

		if ("int".equalsIgnoreCase(nodeDTObject.asText()) && (StringUtils.isEmpty(dnObjetText) || isNullNode

				|| (isNumberArray ? DataUtility.isNumeric(dnObjetText, numberPatternArray)

						: (DataUtility.isNumeric(dnObjetText)||DataUtility.isDecimal(dnObjetText)) ))) {

			//8421 code changes.

			dataTypeFlag = true;

			incorrectintegerDataType = "integer";

			

			// PE-9689 : If the value is string integer("45"), then convert it to integer(45).

			// PE-9708 : If the value is string integer("45.0" or "45.7"), then convert it to integer(45).

			if(dnCodeObject.asText().matches(Constants.INT_PATTERN) || dnCodeObject.asText().matches(Constants.DATATYPE_DOUBLE_PATTERN)) {

				((ObjectNode) dataNode).set(mappedColumnName, JsonNodeFactory.instance.numberNode(dnCodeObject.asInt())); 				

				dataMap.put(mappedColumnName,dataNode.get(mappedColumnName));



			// Else if Dataformat is defined, convert it to appropriate string. 

			} else if (isNumberArray && !dnCodeObject.isNumber() && !StringUtils.isEmpty(dnObjetText)) {

			   ((ObjectNode) dataNode).put(mappedColumnName, DataUtility.toNumber(dnObjetText, numberPatternArray)); 

			   //PE-8731

			   dataMap.put(mappedColumnName,dataNode.get(mappedColumnName));

		   }

		} else if ("boolean".equalsIgnoreCase(nodeDTObject.asText()) && ("false".equalsIgnoreCase(dnObjetText)

				|| isNullNode || StringUtils.isEmpty(dnObjetText) || "true".equalsIgnoreCase(dnObjetText))) {

			//8421 code changes.

			dataTypeFlag = true;

			

			incorrectintegerDataType = "boolean";

			((ObjectNode) dataNode).put(mappedColumnName, DataUtility.toNumber(dnObjetText, numberPatternArray));

		} else if ("date".equalsIgnoreCase(nodeDTObject.asText()) && (StringUtils.isEmpty(dnObjetText) || isNullNode

				|| (isDateArray ? DataUtility.isValidDate(dnObjetText, datePatternArray)

						: DataUtility.isValidDate(dnObjetText)))) {

			//8421 code changes.

			dataTypeFlag = true;

			incorrectintegerDataType = "date";



			// PE-9689 : If the value is string long("45454545"), then convert it to integer(45454545).

			if(dnCodeObject.asText().matches(Constants.LONG_PATTERN)) {

				((ObjectNode) dataNode).set(mappedColumnName, JsonNodeFactory.instance.numberNode(Long.parseLong(dnCodeObject.asText())));

				dataMap.put(mappedColumnName,dataNode.get(mappedColumnName));

			

			// Else if Dateformat is defined, convert it to appropriate string. 

			// Convert to ISO-8601 Format

			} else if (isDateArray && !StringUtils.isEmpty(dnObjetText)) {

				((ObjectNode) dataNode).put(mappedColumnName, DataUtility.customDateConvertor(dnObjetText,datePatternArray));

			}

		} else if ("double".equalsIgnoreCase(nodeDTObject.asText()) && (StringUtils.isEmpty(dnObjetText) || isNullNode

				|| (isNumberArray ? DataUtility.isDecimal(dnObjetText, numberPatternArray)

						: DataUtility.isDecimal(dnObjetText)))) {

			dataTypeFlag = true;

			incorrectintegerDataType = "double";

			

			// PE-9689 Note : If user passes integer(454545) while the dataype="double", it will get converted to double(454545.0)

			// If the value is string long("4545.45"), then convert it to integer(4545.45).

			if(dnCodeObject.asText().matches(Constants.DATATYPE_DOUBLE_PATTERN)) {

				((ObjectNode) dataNode).set(mappedColumnName, JsonNodeFactory.instance.numberNode(Double.parseDouble(dnCodeObject.asText())));

				dataMap.put(mappedColumnName, dataNode.get(mappedColumnName));

				

			// Else if Dataformat is defined, convert it to appropriate string. 

			//Convert to  java decimal

			} else if (isNumberArray && !StringUtils.isEmpty(dnObjetText)) {

				((ObjectNode) dataNode).put(mappedColumnName, DataUtility.toDecimal(dnObjetText, numberPatternArray));

				//PE-8731

				dataMap.put(mappedColumnName,dataNode.get(mappedColumnName));

			}

		} else if (StringUtils.isEmpty(dnObjetText) || isNullNode || "string".equalsIgnoreCase(nodeDTObject.asText())) {

			dataTypeFlag = true;

		

		}

		//8421 code changes.

		incorrectDataTypeMap.put("dataTypeFlag", Boolean.toString(dataTypeFlag));

		incorrectDataTypeMap.put("datatype", incorrectintegerDataType);

		return incorrectDataTypeMap; 

	}



	public ArrayNode findAndMerge(JsonNode dataNode, ArrayNode csvNode, String uniqueIdColumnCode) {



		Iterator<String> it = null;

		String key = "";

		boolean isMerged = false;

		if (dataNode != null && dataNode.size() > 0) {

			for (int i = 0; i < dataNode.size(); i++) {

				isMerged = false;

				if (dataNode.get(i).get(uniqueIdColumnCode) != null)

				{

					for (int j = 0; j < csvNode.size(); j++) {

						if (csvNode.get(j).get(uniqueIdColumnCode) != null && csvNode.get(j).get(uniqueIdColumnCode)

								.equals(dataNode.get(i).get(uniqueIdColumnCode))) {

							it = dataNode.get(i).fieldNames();

							while (it != null && it.hasNext()) {

								key = it.next();

								if (JsonUtils.isNullOrBlankOrNullNode(csvNode.get(j).get(key)) || csvNode.get(j).get(key).asText().equals("")) {

									((ObjectNode) csvNode.get(j)).set(key, dataNode.get(i).get(key));

								}

							}

							isMerged = true;

						}

					}

					if (!isMerged) {

						csvNode.add(dataNode.get(i));

					}

				}

			}

		}

		return csvNode;

	}



	public void buildCheckboxArray(JsonNode dataNode,String code, String mappedColumnName) {

		if (code == null || code.equals("")) {

			code = mappedColumnName;

		}

		if (dataNode.get(code) != null)



		{

			if (dataNode.get(code).getNodeType() != JsonNodeType.OBJECT

					&& dataNode.get(code).getNodeType() != JsonNodeType.ARRAY) {

				ArrayNode arrNode = mapper.createArrayNode();

				// PE-7795 Array was giving arrayName["null"] wrong array.

				// Below changes will give arraName[null] requires correct array.

				if( !JsonUtils.isNullNode(dataNode.get(code))) {

					String arr1[] =   dataNode.get(code).asText().split(",");

					if (arr1 != null && arr1.length > 0) {

						for (int i = 0; i < arr1.length; i++) {

							arrNode.add(arr1[i]);

						}

					}

				}

				((ObjectNode) (dataNode)).set(code, arrNode);

				

			} else {

				Iterator<Map.Entry<String, JsonNode>> it = dataNode.get(code).fields();

				Map.Entry<String, JsonNode> entry = null;

				while (it != null && it.hasNext()) {

					entry = it.next();

					ArrayNode arrNode = mapper.createArrayNode();

					switch (entry.getValue().getNodeType()) {

					case ARRAY:

						for (final JsonNode objNode : entry.getValue()) {

							arrNode.add(objNode);

						}

						break;

					case STRING:

						String[] arr1 = entry.getValue().asText().split(",");

						if (arr1 != null && arr1.length > 0) {

							for (int i = 0; i < arr1.length; i++) {

								arrNode.add(arr1[i]);

							}

						}

						break;

					default:

						break;

					}

					((ObjectNode) (dataNode.get(code))).set(entry.getKey(), arrNode);

				}

			}

		}

	}



	/**

	 * 

	 * @param header

	 * @param entitiesJson

	 * @return

	 * @throws JsonParseException

	 * @throws JsonMappingException

	 * @throws IOException

	 */

	@SuppressWarnings("unchecked")

	public HashMap<String, Object> compareFields(List<HashMap<String, String>> records,List<String> header, String entitiesJson,

			HashMap<String, Object> xlsxMap,Object eeidColFlag) throws Exception {

		ArrayNode entitiesNode = mapper.readValue(entitiesJson, ArrayNode.class);

		JsonNode columnNode = null;

		ArrayNode arr = null;

		int columnNodeSize = 0;

		boolean found = false;

		boolean multiSheetExcel=false;

		ObjectNode resultNode = null;

		HashMap<String, Object> resultMap = new HashMap<String, Object>();

		String entityCode = null;

		JsonNode skip = null;

		String mappedColName = null;

		boolean sheetMatchingFlag = true;

		String sheetName = "";

		JsonNode dataNode = null;

		String currentHeaderCol = null;//PE-8247 duplicate cols validation

		Set<String> set = null;//PE-8247

		HashMap<String, List<String>> headerMap=null;

		if(xlsxMap!=null)

		{

			headerMap = (HashMap<String, List<String>>)xlsxMap.get(Constants.XLSXHEADERMAP);

		}

		if (entitiesNode != null && xlsxMap != null && headerMap != null) {

			if(headerMap.size() > 1 )

				multiSheetExcel=true;

			if (entitiesNode.size() < headerMap.size()) {

				sheetMatchingFlag = false;

			} else {

				for (int i = 0; i < entitiesNode.size(); i++) {

					sheetName = entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.COMPANYNAME).asText() + "_"

							+ entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.CTRYCDOE).asText();

					if (headerMap.get(sheetName) == null) {

						sheetMatchingFlag = false;

					}

					break;

				}



			}

			

		}

		

		

		if(xlsxMap!=null && (records !=null || !multiSheetExcel) && entitiesNode.size()> 1)

		{

			HashMap<String, Object> map = new HashMap<>();

			if(records !=null)

			{

				map.put("data", records);

				map.put(Constants.COLUMNS,header );

			}

			else

			{

				map.put("data", ((HashMap<String, List<HashMap<String, String>>>)xlsxMap.get(Constants.XLSXDATAMAP)).get(xlsxMap.get(Constants.FIRSTSHEETNAME)));

				map.put(Constants.COLUMNS,headerMap.get(xlsxMap.get(Constants.FIRSTSHEETNAME)) );

			}

			dataNode = mapper.readValue(mapper.writeValueAsString(map), JsonNode.class);

			resultMap.put(Constants.RESULTNODE, dataNode);

			return resultMap;

		}



		for (int i = 0; i < entitiesNode.size(); i++) {

            		set = new HashSet<>();

			dataNode = mapper.createObjectNode();

			columnNode = entitiesNode.get(i).get(Constants.SECTION_STRUCTURE).get(Constants.COLUMNS);

			if (entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.COMPANYNAME) != null

					&& entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.CTRYCDOE) != null) {

				sheetName = entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.COMPANYNAME).asText() + "_"

						+ entitiesNode.get(i).get(Constants.CONTEXTDATAKEY).get(Constants.CTRYCDOE).asText();

			}

			columnNodeSize = (columnNode).size();

			if (xlsxMap != null && xlsxMap.get("xlsxHeaderMap") != null) {

				if (!sheetMatchingFlag) {

					header = headerMap.get(xlsxMap.get(Constants.FIRSTSHEETNAME));

				} else {

					header = headerMap.get(sheetName);

				}

			}

			resultNode = mapper.createObjectNode();

			if (header != null) {

				for (int j = 0; j < header.size(); j++) {

					//PE-8247 Duplicate Cols Validation

					if(!set.add(header.get(j).trim())){

						throw new CustomStatusException(Constants.HTTPSTATUS_400,

								in.lnt.utility.general.Cache.getPropertyFromError(ErrorCacheConstant.ERR_065));

					}

				}

				for (int j = 0; j < header.size(); j++) {

					currentHeaderCol = header.get(j);  

					found = false;

					for (int k = 0; k < columnNodeSize; k++) {

						skip = columnNode.get(k).get(Constants.SKIP);

						if (null != columnNode.get(k).get("code")) {

							entityCode = columnNode.get(k).get("code").asText();

						}

						if (null != columnNode.get(k).get(Constants.MAPPED_COLUMN_NAME)) {

							mappedColName = columnNode.get(k).get(Constants.MAPPED_COLUMN_NAME).asText();

						}

						//Code modified to skip mapping in case of skip=true in request PE : 6858

						if(skip!=null && skip.asText().equals("true"))

						{

							if(currentHeaderCol.equals(entityCode) || currentHeaderCol.equals(mappedColName))

							{	

							found = true;

							break;

							}

							else

							{

							continue;

							}

							

						}

						if ((!StringUtils.isEmpty(entityCode)|| !StringUtils.isEmpty(mappedColName)) && skip==null && (currentHeaderCol.equals(entityCode) || currentHeaderCol.equals(mappedColName))) {

							

								found = true;

								break;

							

						}

					}

					if (!found) {

						break;

					}

				}

			}

			if (!found) {

				break;

			}



		}

		if (!found || (xlsxMap!=null &&((Integer)xlsxMap.get(Constants.EEIDINHEADER)!= (headerMap.size()))) ) {

			if(!multiSheetExcel)

			{

			HashMap<String, Object> map = new HashMap<String, Object>();

			if(xlsxMap!=null)

			{

			if(!sheetMatchingFlag)

			{

				map.put("data", ((HashMap<String, List<HashMap<String, String>>>)xlsxMap.get(Constants.XLSXDATAMAP)).get(xlsxMap.get(Constants.FIRSTSHEETNAME)));

			}

			else{

				map.put("data", ((HashMap<String, List<HashMap<String, String>>>)xlsxMap.get(Constants.XLSXDATAMAP)).get(sheetName));

			}

			

			}

			else if(records!=null)

			{

				map.put("data", records);

			}

			if((boolean)eeidColFlag)

			{

			map.put(Constants.COLUMNS,header );

			}

			else

			{

				header.remove(header.size() - 1);

				map.put(Constants.COLUMNS,header);

			}

			dataNode = mapper.readValue(mapper.writeValueAsString(map), JsonNode.class);

			resultMap.put(Constants.RESULTNODE, dataNode);

			}

			else

			{

				resultMap.put(Constants.RESULTNODE, ErrorCacheConstant.ERR_055);

			}

		} else {

			resultMap.put(Constants.RESULTNODE, null);

			if (sheetMatchingFlag == false) {

				resultMap.put("sheetMatchingFlag",

						null != xlsxMap.get(Constants.FIRSTSHEETNAME) ? xlsxMap.get(Constants.FIRSTSHEETNAME) : "");

			}

			return resultMap;

		}

		if (sheetMatchingFlag == false) {

			resultMap.put("sheetMatchingFlag",

					null != xlsxMap.get(Constants.FIRSTSHEETNAME) ? xlsxMap.get(Constants.FIRSTSHEETNAME) : "");

		}

		return resultMap;

	}



	@SuppressWarnings("unchecked")

	public HashMap<String, Object> convertXlsxToMap(Map<String, Object> workBookMap,String empIdColumn) throws Exception {

		

		HashMap<String, Object> map = null;

		List<String> headerList = null;

		List<ArrayList<String>> dataList = null;

		HashMap<String, List<HashMap<String, String>>> xlsxDataMap = null;

		HashMap<String, List<String>> xlsxHeaderMap = null;

		HashMap<String, Object> xlsxMap = null;

		xlsxDataMap = new HashMap<>();

		xlsxHeaderMap = new HashMap<>();

		Set<Object> eeidSet = null;

		List<HashMap<String, String>> allRecords = null;

		HashMap<String, String> record = null;

		boolean eeidFlag = true;

		//PE-8933

		boolean eeidMissingFlag = false;

				

		

		String tempDataListRecord ;

		String tempHeaderListRecord;

		int eeIdInHeader=0;

		if (workBookMap != null && workBookMap.size() > 0) {

			xlsxMap = new HashMap<>();

			//Iterate through each xlsx tab.

			for (Entry<String, Object> entry : workBookMap.entrySet()) {

				eeidSet = new HashSet<>();

				if (entry.getKey().equals(Constants.FIRSTSHEETNAME)) {

					xlsxMap.put(Constants.FIRSTSHEETNAME, entry.getValue());

					continue;

				}

				allRecords = new ArrayList<>();

				map = (HashMap<String, Object>) entry.getValue();

				headerList = (List<String>) map.get("header");

				dataList = (List<ArrayList<String>>) map.get("data");

				

				//Iterate through data records from each xlsx tab.

				for (int i = 0; i < dataList.size(); i++) {

					 record = new HashMap<>();

					 

					//Iterate through each header from each tab.

					for (int j = 0; j < headerList.size(); j++) {

						

						tempDataListRecord = dataList.get(i).get(j);

						tempHeaderListRecord = headerList.get(j);

						if (tempHeaderListRecord.equalsIgnoreCase(Constants.EMPLOYEE_EEID) || (empIdColumn!=null 

								&& !empIdColumn.equals("") && tempHeaderListRecord.equals(empIdColumn))) {

							try {

								if(i==0)

								{

								eeIdInHeader++;

								}

								if (tempDataListRecord != null && !tempDataListRecord.equals(""))

									eeidFlag = eeidSet.add(tempDataListRecord);

								else {

									// PE-8933

									eeidMissingFlag = true;

								}

							} catch (IndexOutOfBoundsException e) {

								eeidFlag = true;

							}

						}

						if (eeidFlag) {

								try {

									boolean isNotBlankEEID = !StringUtils.isEmpty(tempDataListRecord);

									record.put(tempHeaderListRecord,

											(isNotBlankEEID ? tempDataListRecord : Constants.BLANK));

								} catch (IndexOutOfBoundsException e) {

								record.put(tempHeaderListRecord, Constants.BLANK);

							}

						} else {

							xlsxMap.put("isEeidDuplicate", "true");

							eeidSet=null;

							//PE-8933

							if(eeidMissingFlag)

								xlsxMap.put(Constants.MISSING_EEID, true);

							else

								xlsxMap.put(Constants.MISSING_EEID, false);

							

							return xlsxMap;

						}

					}

					allRecords.add(record);

					

					//Clean up block

					record = null;

				}

				xlsxDataMap.put(entry.getKey(), allRecords);

				xlsxHeaderMap.put(entry.getKey(), headerList);

				xlsxMap.put(Constants.XLSXHEADERMAP, xlsxHeaderMap);

				xlsxMap.put(Constants.XLSXDATAMAP, xlsxDataMap);



				//PE-8933

				if(eeidMissingFlag)

					xlsxMap.put(Constants.MISSING_EEID, true);

				else

					xlsxMap.put(Constants.MISSING_EEID, false);

				

				//Clean up records

				allRecords = null;

			}

			xlsxMap.put(Constants.EEIDINHEADER, eeIdInHeader);

		  }

		eeidSet = null;

		//logger.info(String.format("%s %s", Constants.EXIT ,Constants.LOG_CONVERXLSXTOMAP));

		return xlsxMap;

	}



	@SuppressWarnings("unchecked")

	public LinkedHashMap<String, String> prepareJsonString(HashMap<String, Object> xslsMap) throws Exception{



		HashMap<String, Object> map = new HashMap<String, Object>();

		HashMap<String, List<HashMap<String, String>>> xlsxDataMap = null;

		String csvJsonString = "";

		LinkedHashMap<String, String> xslJsonMap = new LinkedHashMap<String, String>();

		HashMap<String, List<HashMap<String, String>>> xlsxDataMap_Temp = null;

		if (xslsMap != null && xslsMap.size() > 0) {

			xlsxDataMap_Temp = (HashMap<String, List<HashMap<String, String>>>) xslsMap.get(Constants.XLSXDATAMAP); //PE-9452 Empty cols removal

			xlsxDataMap = RemoveColsWithNoValue(xlsxDataMap_Temp);                                                  //PE-9452 Empty cols removal

			for (Entry<String, List<HashMap<String, String>>> entry : xlsxDataMap.entrySet()) {

				map.put("data", entry.getValue());

				ObjectMapper objmapper = new ObjectMapper();

					csvJsonString = objmapper.writeValueAsString(map);

					xslJsonMap.put(entry.getKey(), csvJsonString);



			}

		}

		

		return xslJsonMap;

	}

	

	//Method added to remove cols having no values PE-9452 

	public HashMap<String, List<HashMap<String, String>>> RemoveColsWithNoValue(HashMap<String, List<HashMap<String, String>>> xlsxDataMap) throws Exception{

		List<HashMap<String, String>> xslJsonMapEntry = null;;

		HashMap<String, List<HashMap<String, String>>> xlsxDataMapWithoutEmptyCols = new HashMap<String, List<HashMap<String, String>>>();

		HashMap<String, String>  hm = null;

		for (Entry<String, List<HashMap<String, String>>> entry : xlsxDataMap.entrySet()) {

			xslJsonMapEntry = new ArrayList<HashMap<String, String>>();

			for(HashMap<String, String>  hm1 : entry.getValue()){

				hm = new HashMap<String, String>();

				for(Entry<String,String> entry1 : hm1.entrySet()){

					if(entry1.getValue().length() == 0){

						//hm.remove(entry.getKey());

					}else{

						hm.put(entry1.getKey(), entry1.getValue());

					}

				}

				xslJsonMapEntry.add(hm);

			}

			xlsxDataMapWithoutEmptyCols.put(entry.getKey(),xslJsonMapEntry);



		}

		

		return xlsxDataMapWithoutEmptyCols;

	}





	// PE : 2449 to check duplication of EEID start in case of entity data in

	// request body



	public Map<String,String> checkUniqueNessForEEID(String enetityJson,String dataAsJson) throws Exception{

		//logger.info(String.format("%s %s", Constants.INTIATE ,Constants.LOG_CHECKUNIQUENESSFOREEID));

		@SuppressWarnings("unchecked")

		Map<String,String> empIdMap= new HashMap<>();

		empIdMap.put(Constants.ISUNIQUE, "true");

		JsonNode entityNode = null;

		boolean flag = false;

		String code = "";

		JsonNode sectionStructureNode = null;

		String empIdColumn="";

		ArrayList<JsonNode> columnNodeList = null;

			columnNodeList = new ArrayList<JsonNode>();

			JsonNode entittiesNode = mapper.readValue(enetityJson, JsonNode.class);

			Set<String> eeidSet = null;

			if (entittiesNode != null) {

				for (int i = 0; i < entittiesNode.size(); i++) {

					eeidSet = new HashSet<>();

					entityNode = entittiesNode.get(i);

					sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

					

					for (int j = 0; j < sectionStructureNode.get(Constants.COLUMNS).size(); j++) {

						columnNodeList.add(sectionStructureNode.get(Constants.COLUMNS).get(j));



					}

					if (columnNodeList != null && columnNodeList.size() > 0) {



						for (JsonNode node : columnNodeList) {

							if (node.get(Constants.CODE) != null)

								code = node.get(Constants.CODE).asText();

							

							if (code.equals(Constants.EMPLOYEE_EEID))

							{

								flag = true;

								if(node.get(Constants.MAPPED_COLUMN_NAME)!=null && !node.get(Constants.MAPPED_COLUMN_NAME).asText().equals(""))

								{

									empIdColumn = node.get(Constants.MAPPED_COLUMN_NAME).asText();

									empIdMap.put("empIdColumn", empIdColumn);

								}

							}

								

						}

					}



					if (flag) {

						JsonNode dataNode = entittiesNode.get(i).get(Constants.DATA);

						if (dataNode != null) {

							for (int k = 0; k < dataNode.size(); k++) {



								if (dataNode.get(k).get(Constants.EMPLOYEE_EEID) != null

										&& !dataNode.get(k).get(Constants.EMPLOYEE_EEID).asText().equals("")) {

									String eeId = dataNode.get(k).get(Constants.EMPLOYEE_EEID).asText();

									if (!eeidSet.add(eeId)) {

										eeidSet.clear(); 

										empIdMap.put(Constants.ISUNIQUE,Constants.DUPLICATE);

										return empIdMap;

									}

								} else {

									 empIdMap.put(Constants.ISUNIQUE,Constants.MISSING);

									 return empIdMap;

								}

							}

						}

						if(dataAsJson!=null)

						{

							empIdMap = DataUtility.checkUniqueEEIDForRequestDataJson(dataAsJson,empIdMap);

							return empIdMap;

						}

					}

				}

			}

		

		return empIdMap;



	}



	// PE : 2449 to prepare auto-correct validation object in case of missing EEID..

	public ObjectNode prepareEeidAutoCorrectObject(String code, HashMap<String, JsonNode> dataMap, JsonNode dataNode,

			 HashMap<String, String> columnMappingMap) throws Exception {

		

		ObjectNode obj = null;  

		

		if (dataMap != null && ((!JsonUtils.isNullOrBlankOrNullNode(dataMap.get(Constants.EMPLOYEE_EEID))

				&& !JsonUtils.isBlankTextNode(dataMap.get(Constants.EMPLOYEE_EEID)) ) 

				|| (!JsonUtils.isNullOrBlankOrNullNode(dataMap.get(columnMappingMap.get(code)))

						&& !JsonUtils.isBlankTextNode(dataMap.get(columnMappingMap.get(code))) )))

				 {

			return obj;

		} else {

			String eeId = generateUniqueEeid();

			

				obj = APIExpressionEvaluator.prepareOutputJson(null, null, eeId, ValidationTypes.EEIDAUTOCORRECT, code,

						null, null, null);

				((ObjectNode) (dataNode)).put(code, eeId);

		}

		

		return obj;

	}



	public static synchronized String generateUniqueEeid() throws Exception {

		

		Date d1 = new Date();

		SimpleDateFormat sdf = new SimpleDateFormat(Constants.EEID_FORMAT2);

		

		String tmpNanoTime = String.valueOf(System.nanoTime());		// Duplicate EEID occurred at millisec-level. Found in PE-8616.

		return sdf.format(d1)+ tmpNanoTime.substring(tmpNanoTime.length()-5, tmpNanoTime.length());



	}

	

	public static ArrayNode removeMlNode(JsonNode jsonNode) throws Exception {

		JsonNode mlNode = null;

		JsonNode entityNode = null;

		Iterator<Entry<String, JsonNode>> mlItr = null;

		ArrayNode mlJsonDataArray = (ArrayNode) jsonNode.get(Constants.ENTITIES);

		Map.Entry<String, JsonNode> entry = null;



		for (int i = 0; i < mlJsonDataArray.size(); i++) {



			entityNode = mlJsonDataArray.get(i);

			mlNode = entityNode.get(Constants.DATA);



			for (int k = 0; k < mlNode.size(); k++) {



				mlItr = mlNode.get(k).fields();

				while (mlItr.hasNext()) {

					entry = mlItr.next();

					if (entry.getKey().contains(Constants.UNDERSCORE_REPLACE) || entry.getKey().contains(Constants.UNDERSCORE_ACFLAG)) {

						((ObjectNode) mlNode.get(k)).remove(entry.getKey());

					}

				}

			}

		}

		return mlJsonDataArray;

	}

	public void performL0Validation(String entitiesJson, String csvJsonString, boolean isAggregateRequest,

			HashMap<String, String> xslJsonMap, String FirstSheetName) throws Exception

	{

		

			JsonNode sectionStructureNode = null;

			JsonNode contextDataNode = null;

			JsonNode csvNode = null;

			JsonNode entityNode = null;

			long startTime=0;

			long stopTime=0;

			Set<String> xslJsonSheets = null;

			String code = "";

			JsonNode validationNode = null;

			String returnValue="";

			Pattern pattern = Pattern.compile(Constants.EXP_PATTERN);

			boolean columnMissflag = true;

		

		

			Map<String,String> mappedColumnMap = new HashMap<String,String>();

			// 8395 JP

			String field=null;

			ArrayNode entitiesNode = mapper.readValue(entitiesJson, ArrayNode.class);

			if (xslJsonMap != null) {

				xslJsonSheets = new HashSet<String>(xslJsonMap.keySet());

			}

			for (int i = 0; i < entitiesNode.size(); i++) {

				entityNode = entitiesNode.get(i);

				if(entitiesNode.get(i).get(Constants.DATA)!=null && entitiesNode.get(i).get(Constants.DATA).size()>0)

				{

				break;	

				}

				sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

				contextDataNode = entityNode.get(Constants.CONTEXTDATAKEY);

				if (csvJsonString != null && !csvJsonString.equals("")) {

					csvNode = mapper.readValue(csvJsonString, JsonNode.class);

				} else if (xslJsonMap != null && xslJsonMap.size() > 0) {

					String coName_CtryCode = contextDataNode.get(Constants.COMPANYNAME).asText() + "_"

							+ contextDataNode.get(Constants.CTRYCDOE).asText();

					if (FirstSheetName != null) {

						csvNode = mapper.readValue(xslJsonMap.get(FirstSheetName), JsonNode.class);

					} else if (xslJsonMap.get(coName_CtryCode) != null) {

						if (xslJsonSheets.contains(coName_CtryCode)) {

							csvNode = mapper.readValue(xslJsonMap.get(coName_CtryCode), JsonNode.class);

						}

					} else if (xslJsonMap.get(coName_CtryCode) == null) {

						/*throw new CustomStatusException(Constants.HTTPSTATUS_400,

								String.format(Constants.ERROR_SHEET_NOT_FOUND, coName_CtryCode));*/

						paramValueList =  new ArrayList<String>();

						paramValueList.add(""+coName_CtryCode);

						throw new CustomStatusException(Constants.HTTPSTATUS_400,Cache.getPropertyFromError(ErrorCacheConstant.ERR_035),

								Cache.getPropertyFromError(ErrorCacheConstant.ERR_035+Constants.UNDERSCORE_PARAMS),paramValueList);

					}

				}

				if (!JsonUtils.isNullOrBlankOrNullNode(csvNode) 

						&& csvNode.get(Constants.DATA).size() > 0 ) {

					sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

					ArrayList<JsonNode> columnNodeList = new ArrayList<>();

					for (int j = 0; j < sectionStructureNode.get(Constants.COLUMNS).size(); j++) {

						columnNodeList.add(sectionStructureNode.get(Constants.COLUMNS).get(j));

						//Code added for 8395

						/*if(null != columnNodeList.get(j).get(Constants.CODE) && null != columnNodeList.get(j).get(Constants.MAPPED_COLUMN_NAME)){

							mappedColumnMap.put(columnNodeList.get(j).get(Constants.CODE).asText(), columnNodeList.get(j).get(Constants.MAPPED_COLUMN_NAME).asText());

						}

						if(null != columnNodeList.get(j).get(Constants.CODE)) {

							metaDataColumnsSet.add(columnNodeList.get(j).get(Constants.CODE).asText());

						}*/

						//8395 JP

						if (null != columnNodeList.get(j).get(Constants.CODE)) {

							mappedColumnMap.put(columnNodeList.get(j).get(Constants.CODE).asText(),columnNodeList.get(j).get(Constants.CODE).asText());

							if (null != columnNodeList.get(j).get(Constants.MAPPED_COLUMN_NAME)) {

								mappedColumnMap.put(columnNodeList.get(j).get(Constants.CODE).asText(),

										columnNodeList.get(j).get(Constants.MAPPED_COLUMN_NAME).asText());

							}

						}

						

					}

						if (columnNodeList != null && columnNodeList.size() > 0) {



							for (JsonNode node : columnNodeList) {

								if (node.get(Constants.CODE) != null)

									code = node.get(Constants.CODE).asText();

								if (node.get(Constants.VALIDATIONS) != null

										&& node.get(Constants.VALIDATIONS).size() > 0) {

									for (int k = 0; k < node.get(Constants.VALIDATIONS).size(); k++) {

										validationNode = node.get(Constants.VALIDATIONS).get(k);

										columnMissflag = true;

										if (validationNode != null && validationNode.get("category") != null

												&& validationNode.get("category").asText().equalsIgnoreCase("file_acceptance")) {

											

											if (validationNode.get(Constants.EXPRESSION) != null && validationNode

													.get(Constants.EXPRESSION).asText().contains("ISBLANK")) {

												Matcher matcher = pattern.matcher(validationNode.get(Constants.EXPRESSION).asText()

																.replaceAll(Constants.CURLY_BRACES, ""));

												List<String> fields = new ArrayList<String>();

												while (matcher.find()) {

													// 8395 JP

													field=matcher.group().replace("this.", "").trim();

													if(mappedColumnMap.containsKey(field)) {

														fields.add(field);

													}

												}

									/*	for (int fieldSize = 0; fieldSize < fields.size(); fieldSize++) {

											// if(csvNode.get(Constants.DATA).get(0).has(fields.get(fieldSize))){

											if (csvNode.get(Constants.DATA).get(0)

													.has(mappedColumnMap.get(fields.get(fieldSize)))) {

												// if (csvNode.get(Constants.DATA).get(0).get(fields.get(fieldSize)) !=

												// null) {

												columnMissflag = false;

												break;

											}

										}*/

												//if (!columnMissflag) {

													 startTime = System.currentTimeMillis();

													//Code added for 8395

													columnMissflag = checkforBlank(fields, csvNode,mappedColumnMap);

													 stopTime = System.currentTimeMillis();

													 //logger.info(String.format("Time taken for checkforBlank() in FormValidator..%d", (stopTime-startTime)));

												//}

											if (columnMissflag) {

												

												paramValueList = new ArrayList<>();

												paramValueList.add(validationNode.get("_id").asText());

												paramValueList.add(code);

												throw new CustomStatusException(Constants.HTTPSTATUS_400,

														Cache.getPropertyFromError(ErrorCacheConstant.ERR_060),

														Cache.getPropertyFromError(ErrorCacheConstant.ERR_060 + Constants.UNDERSCORE_PARAMS),

														paramValueList);

											}

											}



										}



									}

								}

							}

						}



					}

				}

		//return returnValue;

	}

	public boolean checkforBlank(List<String> fields,JsonNode csvNode,Map<String,String> mappedColumn)throws Exception

	

	{

		boolean returnValue = true;

		JsonNode dataNode = csvNode.get(Constants.DATA);

		for (String field : fields) {

			if (returnValue == false)

				break;

				JsonNode dataNodeValue = null;

				for (int i = 0; i < dataNode.size(); i++) {

					dataNodeValue = dataNode.get(i);

					//Code added for 8395

					if ((dataNodeValue != null && !JsonUtils.isNullOrBlankOrNullNode(dataNodeValue.get(field))

					&& dataNodeValue.get(field).asText() != null && !dataNodeValue.get(field).asText().trim().equals("")) ||

							((dataNodeValue != null && !JsonUtils.isNullOrBlankOrNullNode(dataNodeValue.get(mappedColumn.get(field)))

					&& dataNodeValue.get(mappedColumn.get(field)).asText() != null && !dataNodeValue.get(mappedColumn.get(field)).asText().trim().equals("")))

					/*(mappedColumn.get(dataNode.get(i)) != null)*/) {

						returnValue = false;

						break;

					}

				}

		}

		return returnValue;

	}



	// PE-7241 Changes

	public static ArrayList<ObjectNode> validatePredefinedValidation(JsonNode dataNode, JsonNode columnNode,

			JsonNode validationNode, HashMap<String, JsonNode> dataMap, String mappedColumnName, String errorType)

			throws Exception {

		ArrayList<ObjectNode> emailObjList = new ArrayList<ObjectNode>();	

		// Get Regex depending upon error Type

		Pattern COMMON_REGEX_PATTERN = null;

		switch (errorType) {

		case Constants.EMAIL:

			COMMON_REGEX_PATTERN = VALID_EMAIL_ADDRESS_REGEX;

			break;

		case Constants.PHONE:

			COMMON_REGEX_PATTERN = VALID_PHONE_NUMBER_REGEX;

			break;

		case Constants.REQUIRED:

			break;

		default:

			break;

		}

		boolean checkCode = false;

 		// Mapped Columns matching

		if(columnNode.has(Constants.CODE) && !JsonUtils.isNullOrBlankOrNullNode(columnNode.get(Constants.CODE))){

			checkCode = true;

		} 

		

		// Dimensions

		if (columnNode.has(Constants.DIMENSIONS) && !JsonUtils.isNullNode(columnNode.get(Constants.DIMENSIONS))) {

			JsonNode dimensionNode = dataNode.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName);

			// PE-8463 NULL Handling

			if (!JsonUtils.isNullOrBlankOrNullNode(dimensionNode)) {

				dimensionNode = dimensionNode.deepCopy();

			}

			else {

				// PE-8814 - No data for dimension

				dimensionNode = createNullNodesForDimension(columnNode.get(Constants.CODE).asText(),

						columnNode.get(Constants.DIMENSIONS));

				//PE-8814 

			}

			if(!JsonUtils.isNullOrBlankOrNullNode(dimensionNode)){

			// PE-8463 NULL Handling

			for (JsonNode dimensionAttribute : columnNode.get(Constants.DIMENSIONS)) {

				if (dimensionNode.has(dimensionAttribute.asText())) {

					switch (dimensionNode.get(dimensionAttribute.asText()).getNodeType()) {

					case OBJECT:

						// No object Expected

						return emailObjList;

					// For Checkbox Array

					case ARRAY:

						for (final JsonNode checkBoxObj : dimensionNode.get(dimensionAttribute.asText())) {

							String inputValue = checkInputNodeType(checkBoxObj);

							ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

									inputValue, dimensionAttribute, columnNode, validationNode, dataMap,checkCode,

									errorType);

							if (objNode.size() > 0) {

								emailObjList.add(objNode);

							}

						}

						break;

					case STRING:

					case BINARY:

					case BOOLEAN:

					case MISSING:

					case NULL:

					case NUMBER:

					case POJO:

						// For Normal Values / CheckBox String

						if (!JsonUtils.isNullOrBlankOrNullNode(dimensionNode.get(dimensionAttribute.asText()))) {

							String[] splitValues = dimensionNode.get(dimensionAttribute.asText()).asText().split(",");

							for (String splitValue : splitValues) {

								/*emailObjList.add(new APIExpressionEvaluator().prepareOutputJson(validationNode, columnNode,

								null, ValidationTypes.expression, null, null, null, null));*/

								ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

										splitValue, dimensionAttribute, columnNode, validationNode, dataMap,checkCode, errorType);

								if (objNode.size() > 0) {

									emailObjList.add(objNode);

								}

							}

							break;

						}else{

							ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

									null, dimensionAttribute, columnNode, validationNode, dataMap,checkCode, errorType);

							if (objNode.size() > 0) {

								emailObjList.add(objNode);

							}

						}

					}

				}

				// Dimension not found and Validation is required

				else if (errorType.equals(Constants.REQUIRED)) {

					ObjectNode objNode = APIExpressionEvaluator.prepareOutputJson(validationNode, dimensionAttribute, null,

							ValidationTypes.PREDEFINEDVALIDATIONS,

							checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName, null, null, null);

					if (objNode.size() > 0) {

						emailObjList.add(objNode);

					}

				}

				

			}

		}

		} else if ((columnNode.has(Constants.QUESTION_TYPE)

				&& (!JsonUtils.isNullNode(columnNode.get(Constants.QUESTION_TYPE))

						&& columnNode.get(Constants.QUESTION_TYPE).asText().equals(Constants.CHECKBOXES)))

				&& (!JsonUtils.isNullNode(

						dataNode.get(checkCode ? 

								columnNode.get(Constants.CODE).asText() :

									mappedColumnName)))) {

			JsonNode checkBoxData = dataNode.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName);

			if (!JsonUtils.isNullOrBlankOrNullNode(checkBoxData)) {

				switch (checkBoxData.getNodeType()) {

				case ARRAY:

					if(checkBoxData.size()==0) {

						ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

								null, null, columnNode, validationNode, dataMap, checkCode, errorType);

						if (objNode.size() > 0) {

							emailObjList.add(objNode);

						}

						

					}else {

						for (final JsonNode checkBoxObj : checkBoxData) {

							String inputValue = checkInputNodeType(checkBoxObj);

							ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

									inputValue, null, columnNode, validationNode, dataMap, checkCode, errorType);

							if (objNode.size() > 0) {

								emailObjList.add(objNode);

							}

						}

					}

					break;

				case STRING:

				case BINARY:

				case BOOLEAN:

				case MISSING:

				case NULL:

				case NUMBER:

				case POJO:

					// String Value (Comma Separated)

					if (!JsonUtils.isNullOrBlankOrNullNode(

							dataNode.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName))) {

						String[] splitValues = dataNode

								.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName).asText()

								.split(",");

						for (String splitValue : splitValues) {



							/*emailObjList.add(new APIExpressionEvaluator().prepareOutputJson(validationNode, columnNode,

									null, ValidationTypes.expression, null, null, null, null));*/



							ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

									splitValue, null, columnNode, validationNode, dataMap, checkCode, errorType);

							if (objNode.size() > 0) {

								emailObjList.add(objNode);

							}

						}

					} else {

						ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN, dataNode

								.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName).asText(),

								null, columnNode, validationNode, dataMap, checkCode, errorType);

						if (objNode.size() > 0) {

							emailObjList.add(objNode);

						}

					}

					break;

				default:

					break;

				}

			}

		} else {

			/*emailObjList.add(new APIExpressionEvaluator().prepareOutputJson(validationNode, columnNode,

						null, ValidationTypes.expression, null, null, null, null));*/

			ObjectNode objNode = buildValidationObjectForPredefinedValidation(COMMON_REGEX_PATTERN,

					(!JsonUtils.isNullOrBlankOrNullNode(dataNode.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName)) ? dataNode.get(checkCode ? columnNode.get(Constants.CODE).asText() : mappedColumnName).asText() : null), null,

					columnNode, validationNode, dataMap, checkCode, errorType);

			if (objNode.size() > 0) {

				emailObjList.add(objNode);

			}

		}

		return emailObjList; 

	}

 

	private static JsonNode createNullNodesForDimension(String code, JsonNode dimensions) {

		ObjectNode dimensionsObject = mapper.createObjectNode();

		for (JsonNode dimensionNode : dimensions) {

			dimensionsObject.set(dimensionNode.asText(),NullNode.getInstance());

		}

		return mapper.createObjectNode().set(code, dimensionsObject);

	}



	static String checkInputNodeType(JsonNode node) {

		String inputValue = node.asText();

		switch (node.getNodeType()) {

		case STRING:

		case BINARY:

		case BOOLEAN:

			break;

		case MISSING:

		case NULL:

			inputValue = null;

			break;

		case NUMBER:

		case POJO:

			break;

		default:

			break;

		}

		return inputValue;

	}

	

	static ObjectNode buildValidationObjectForPredefinedValidation(Pattern COMMON_REGEX_PATTERN, String inputValue,

			JsonNode dimensionAttribute, JsonNode columnNode, JsonNode validationNode, HashMap<String, JsonNode> dataMap, boolean checkCode, String errorType)

			throws Exception {

		ObjectNode errorObjNode = mapper.createObjectNode();		

		switch (errorType) {

		case Constants.EMAIL:

		case Constants.PHONE:

			// (null == inputValue) 

			if (!StringUtils.isBlank(inputValue) && (null != COMMON_REGEX_PATTERN)

					&& !COMMON_REGEX_PATTERN.matcher(inputValue).find()) {

				if ((null != dimensionAttribute) && !JsonUtils.isNullNode(dimensionAttribute)) {

					((ObjectNode) validationNode).put("dimension", dimensionAttribute.asText());

				}

				switch (validationNode.get(Constants.ERROR_TYPE).asText().toUpperCase() ) {

				case Constants.ENHANCEMENT:

				case Constants.ALERT:

				case Constants.ERROR:

					errorObjNode = APIExpressionEvaluator.prepareOutputJson(validationNode,

							columnNode, null, ValidationTypes.EXPRESSION, null, null, null, null);

					break;

				case Constants.AUTOCORRECT:

					/*((ObjectNode) validationNode).set("OriginalValue", dimensionNode.get(jsonNode.asText()));

					((ObjectNode) (dimensionNode)).put(jsonNode.asText(), result);

					((ObjectNode) validationNode).put("dimension", jsonNode.asText());

					emailObjList.add(prepareOutputJson(validationNode, columnNode, result, ValidationTypes.expression, null,

							null, null, null));*/

					break;

				default :

					break;

				}

			}

			break;

		case Constants.REQUIRED:

			/*ObjectNode requiredObjNode = new APIExpressionEvaluator().checkMandatoryFields(dataMap, code,

					columnNode.get(Constants.DATA_TYPE).asText(), mappedColumnName);*/

			//return new APIExpressionEvaluator().checkRequiredField(inputValue, code, columnNode.get(Constants.DATA_TYPE).asText());

 

			if (StringUtils.isBlank(inputValue)) {

				String code = null;

				if (checkCode) {

					code = columnNode.get(Constants.CODE).asText();

				} else {

					code = columnNode.get(Constants.MAPPED_COLUMN_NAME).asText();

				}

				errorObjNode = APIExpressionEvaluator.prepareOutputJson(validationNode, dimensionAttribute, null,

						ValidationTypes.PREDEFINEDVALIDATIONS, code, inputValue, null, null);

			}

			break;

		default:

			break;

		}

	return errorObjNode;

	}

	

	public JsonNode preparyCompanyWiseData(JsonNode dataNode,JsonNode contextDataNode)

	{

		String comppany_Column=null;

		String country_Column=null;

		JsonNode mappedCompany=null;

		JsonNode node=null;

		JsonNode companyDataNode = mapper.createObjectNode();;

		boolean found=false;

		ArrayNode arr = mapper.createArrayNode();

		HashMap<String,String> nodeMap=null;

		List<HashMap<String,String>> companyList= new ArrayList<HashMap<String,String>>();

		if(!JsonUtils.isNullOrBlankOrNullNode(contextDataNode))

		{

			if (!JsonUtils.isNullOrBlankOrNullNode(contextDataNode.get(Constants.ENTITY_NAME_COLUMN_CODE)) 

					&& JsonUtils.isStringExist(contextDataNode.get(Constants.ENTITY_NAME_COLUMN_CODE).asText())) {

				comppany_Column=contextDataNode.get(Constants.ENTITY_NAME_COLUMN_CODE).asText();

				

				if (!JsonUtils.isNullOrBlankOrNullNode(contextDataNode.get(Constants.MAPPED_COMPANY))) {

					mappedCompany=contextDataNode.get(Constants.MAPPED_COMPANY);

					Iterator<JsonNode> it=mappedCompany.iterator();

					while(it.hasNext())

					{

						node=(JsonNode)it.next();

						nodeMap=new HashMap<String,String>();

						nodeMap.put("name",(!JsonUtils.isNullOrBlankOrNullNode(node.get("name"))? node.get("name").asText():""));

						nodeMap.put(Constants.COUNTRY,(!JsonUtils.isNullOrBlankOrNullNode(node.get(Constants.COUNTRY))? node.get(Constants.COUNTRY).asText():"")); // check null poiter here

						companyList.add(nodeMap);

					}

					

				}

				

				

			}

			if (!JsonUtils.isNullOrBlankOrNullNode(contextDataNode.get(Constants.ENTITY_COUNTRY_COLUMN_CODE)) 

					&& JsonUtils.isStringExist(contextDataNode.get(Constants.ENTITY_COUNTRY_COLUMN_CODE).asText())) {

				country_Column=contextDataNode.get(Constants.ENTITY_COUNTRY_COLUMN_CODE).asText();

			}

		}

		if(!JsonUtils.isNullOrBlankOrNullNode(dataNode) && JsonUtils.isStringExist(comppany_Column) && !JsonUtils.isNullOrBlankOrNullNode(mappedCompany))

		{

			for(JsonNode data:dataNode)

			{

				found=false;

				if(!JsonUtils.isNullOrBlankOrNullNode(data.get(comppany_Column)))

					{

					for(HashMap<String,String> map:companyList)

					{

						if(!map.get(Constants.COUNTRY).equals(""))

						{

							if(data.get(country_Column).asText().equals(map.get(Constants.COUNTRY)) && data.get(comppany_Column).asText().equals(map.get("name")))

							{

								found=true;

								break;

							}

						}

						else if(data.get(comppany_Column).asText().equals(map.get("name")))

						{

							found=true;

							break;

						}

					}

					if(found)

					{

						arr.add(data);

					}

					}

			}

			((ObjectNode) (companyDataNode)).set(Constants.DATA, arr);

		}

		else

		{

			return ((ObjectNode) (companyDataNode)).set(Constants.DATA, dataNode);

		}

		return companyDataNode;

	}

	

	public HashMap<String, JsonNode> updateDataMapWithEmptyDimension(HashMap<String, JsonNode> dataMap ,HashMap<String, JsonNode> dimMap)

	{

		JsonNode value=null;

		String code="";

		String key="";

		if(dimMap!=null && dimMap.size()>0)

		{

			for (Map.Entry<String, JsonNode> entry : dimMap.entrySet())

	        {

	           code = entry.getKey();

	           value = entry.getValue();

	            for(JsonNode dim : value)

	            {

	            key=code+"."+dim.asText();

	            if(JsonUtils.isNullOrBlankOrNullNode(dataMap.get(key)))

	            {

	            	dataMap.put(key, JsonNodeFactory.instance.nullNode());

	            }

	            }

	            //use key and value

	        }

		}

		return dataMap;

	}



	/**

	 * 

	 * @param entitiesJson

	 * @param csvJsonString

	 * @param isAggregateRequest

	 * @param xslJsonMap

	 * @param firstSheetName

	 * @param isMultiPartRequest

	 * @param isEEIDMissingInAsyncL0

	 * @return

	 * @throws IOException 

	 * @throws CustomStatusException 

	 * @throws Exception

	 */

	public JsonNode preparingL1Input(String entitiesJson, String csvJsonString, boolean isAggregateRequest,

			Map<String, String> xslJsonMap, String firstSheetName, boolean isMultiPartRequest,Object isEEIDMissingInAsyncL0) throws IOException, CustomStatusException {

		JsonNode resultNode = null;

		ArrayNode entityArr = null;

 		Set<String> xslJsonSheets = null;

		JsonNode resultEntiryNode = null;

 		JsonNode sectionStructureNode = null;

		JsonNode dataNode = null;

		JsonNode otherSecDataNode = null;

		JsonNode contextDataNode = null;

		JsonNode csvNode = null;

		String uniqueIdColumnCode = "";

		JsonNode entityNode = null;

		resultNode = mapper.createObjectNode();

 		entityArr = mapper.createArrayNode();

 		Iterator<String> fieldIter=null;

		ArrayNode entitiesNode = mapper.readValue(entitiesJson, ArrayNode.class);

 		StringBuffer coName_CtryCode = new StringBuffer();

		String countryCode = null;

		ArrayList<String> fieldList=new ArrayList<>();

		JsonNode companyDataNode = null;

		ArrayNode objNode=null;

		String fieldName=null;

		String code="";

		String mappedCode="";

		// For .xlsx data

		if (xslJsonMap != null) {

			xslJsonSheets = new HashSet<String>(xslJsonMap.keySet());

		}

		JsonNode colNode = null;

		HashMap<String,String> metaDataCodes=new HashMap<>();

		for (int i = 0; i < entitiesNode.size(); i++) {

			countryCode = "";

 			entityNode = entitiesNode.get(i);

			resultEntiryNode = mapper.createObjectNode();

			sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

			dataNode = entityNode.get(Constants.DATA);

			otherSecDataNode = entityNode.get(Constants.OTHERSECTIONDATAKEY);

			contextDataNode = entityNode.get(Constants.CONTEXTDATAKEY);

			colNode = sectionStructureNode.get(Constants.COLUMNS);

			// PE: 8976 Start

			for (JsonNode col : colNode) {

				if (!JsonUtils.isNullOrBlankOrNullNode(col.get(Constants.MAPPED_COLUMN_NAME))) {

					mappedCode = col.get(Constants.MAPPED_COLUMN_NAME).asText();

				}

				if (!JsonUtils.isNullOrBlankOrNullNode(col.get("code"))) {

					code = col.get("code").asText();

				}

				if(mappedCode.length() > 0 )

				{

					if (!JsonUtils.isNullOrBlankOrNullNode(col.get("skip"))) {

						metaDataCodes.put(mappedCode, "skipEnabled");

					} else if (code.length() > 0) {

						metaDataCodes.put(mappedCode, code);

					}

				}



				



			}

			// PE: 8976 End

			// Added uniqueIdColumnCode for PE: 8315 . Following line is

			// assigned for merging.

			uniqueIdColumnCode = contextDataNode.get("uniqueIdColumnCode").asText();



			if (!contextDataNode.isNull() && contextDataNode.has(Constants.CTRYCDOE)) {

				countryCode = contextDataNode.get(Constants.CTRYCDOE).asText();

			}

			if (csvJsonString != null && !csvJsonString.equals("")) {

				csvNode = mapper.readValue(csvJsonString, JsonNode.class);

			} else if (xslJsonMap != null && xslJsonMap.size() > 0) {

				coName_CtryCode.delete(0, coName_CtryCode.length());

				coName_CtryCode.append(contextDataNode.get(Constants.COMPANYNAME).asText());

				coName_CtryCode.append("_");

				coName_CtryCode.append(countryCode);



				// Default condition - Need to check usage of FirstSheetName !!

				if (firstSheetName != null) {

					csvNode = mapper.readValue(xslJsonMap.get(firstSheetName), JsonNode.class);



					// If sheet is found

				} else if (xslJsonMap.get(coName_CtryCode.toString()) != null) {



					// If sheet not previously used

					if (xslJsonSheets.contains(coName_CtryCode.toString())) {

						csvNode = mapper.readValue(xslJsonMap.get(coName_CtryCode.toString()), JsonNode.class);

						xslJsonSheets.remove(coName_CtryCode.toString());



						// If sheet previously used

					} else {

						paramValueList = new ArrayList<String>();

						paramValueList.add("" + contextDataNode.get(Constants.COMPANYNAME).asText());

						paramValueList.add("" + countryCode );

						throw new CustomStatusException(Constants.HTTPSTATUS_400,

								Cache.getPropertyFromError(ErrorCacheConstant.ERR_034),

								Cache.getPropertyFromError(ErrorCacheConstant.ERR_034 + Constants.UNDERSCORE_PARAMS),

								paramValueList);

					}



					// If sheet not found

				} else if (xslJsonMap.get(coName_CtryCode.toString()) == null) {



					paramValueList = new ArrayList<String>();

					paramValueList.add("" + coName_CtryCode.toString());

					throw new CustomStatusException(Constants.HTTPSTATUS_400,

							Cache.getPropertyFromError(ErrorCacheConstant.ERR_035),

							Cache.getPropertyFromError(ErrorCacheConstant.ERR_035 + Constants.UNDERSCORE_PARAMS), paramValueList);

				}

			}

			if (csvNode != null) {

				companyDataNode = preparyCompanyWiseData(csvNode.get("data"), contextDataNode);

				dataNode = findAndMerge(dataNode, (ArrayNode) companyDataNode.get("data"), uniqueIdColumnCode);

			}



			// L0 Validations Ends Here



			// Returning Input Json for L1 Validation

			// PE 8976 Start : To send column code in the L0response 

			if (metaDataCodes.size() > 0) {

				fieldIter = dataNode.get(0).fieldNames();

				while (fieldIter != null && fieldIter.hasNext()) {

					fieldName = fieldIter.next();

					if (null != metaDataCodes.get(fieldName) && !metaDataCodes.get(fieldName).equals("skipEnabled")) {

						fieldList.add(fieldName);

					}

				}

				if (metaDataCodes.keySet().containsAll(fieldList)) {

					objNode = mapper.createArrayNode();

					JsonNode node = null;

					for (JsonNode data : dataNode) {

						node = mapper.createObjectNode();

						for (String field : fieldList) {

							if (metaDataCodes.get(field) != null) {

								((ObjectNode) node).set(metaDataCodes.get(field), (data.get(field)!=null)?data.get(field):data.get(metaDataCodes.get(field)));

							}

						}

						objNode.add(node);

					}

				}

			}

			// PE 8976 End

			

			//PE-8933 - Starts

			if (null != isEEIDMissingInAsyncL0) {

				if ((boolean) isEEIDMissingInAsyncL0) {

					((ObjectNode) (contextDataNode)).put(Constants.MISSING_EEID, true);

				} else {

					((ObjectNode) (contextDataNode)).put(Constants.MISSING_EEID, false);	

				}

				((ObjectNode) (entityNode)).set(Constants.CONTEXTDATAKEY, contextDataNode);

			}

			//PE-8933 - Ends

			

			resultEntiryNode = entityNode;

			if(null!=objNode && objNode.size()>0)

			{

				((ObjectNode) (resultEntiryNode)).set(Constants.DATA, objNode);	

			}

			else  

			{

				((ObjectNode) (resultEntiryNode)).set(Constants.DATA, dataNode);	

			}

			

			entityArr.add(resultEntiryNode);

			metaDataCodes.clear();

		}

		resultNode = mapper.createObjectNode().set(Constants.ENTITIES, entityArr);

		return resultNode;

	}

	

	public JsonNode perform_L1_L2Validations(JsonNode entitiesNodeResponse, boolean isAggregateRequest,

			boolean isMultiPartRequest) throws Exception {



		JsonNode dataNode = null;

		HashMap<String, String> columnMappingMap = null;

		JsonNode sectionStructureNode = null;

		HashMap<String, JsonNode> dimMap = new HashMap<String, JsonNode>();

		String countryCode = null;

		JsonNode otherSecDataNode = null;

		JsonNode contextDataNode = null;

		JsonNode resultNode = null;

		JsonNode mlNode = null;

		ArrayNode entityArr = null;

		ArrayNode arr = null;

		JsonNode entityNode = null;

		JsonNode resultEntiryNode = null;

		Map.Entry<String, JsonNode> entry = null;

		Map.Entry<String, JsonNode> entry1 = null;

		List<String> otherAndContextKeys = new ArrayList<>();

		String tempArrayName = null;

		entityArr = mapper.createArrayNode();

		ArrayNode entitiesNode = (ArrayNode) entitiesNodeResponse.get(Constants.ENTITIES);

		

		boolean mlAutoCoorect=false;

		JsonNode mLdataNode = null;

		

		// Making csv out of JSON single level

		for (int i = 0; i < entitiesNode.size(); i++) {

			

			//Creating a ExpressionEvalutionDriver at a request level(Per Entity (Eg. Employee section))

			ExpressionEvalutionDriver expressionEvalutionDriver = new ExpressionEvalutionDriver(); 

			logger.debug(

					"perform_L1_L2Validations >> ExpressionEvalutionDriver Cache Initialized");

			

			countryCode = "";

			arr = mapper.createArrayNode();

			entityNode = entitiesNode.get(i);

			resultEntiryNode = mapper.createObjectNode();

			sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

			dataNode = entityNode.get(Constants.DATA);

			otherSecDataNode = entityNode.get(Constants.OTHERSECTIONDATAKEY);

			contextDataNode = entityNode.get(Constants.CONTEXTDATAKEY);

			

			// Before L1 for File upload ML Auto-correction API call will be executed.

			// PE-6254 ML Integration works only for File upload case.

			// Check flag from config file located in API_HOME 

			if ( isMultiPartRequest  

					&& "Y".equalsIgnoreCase(Cache.getMLProperty(CacheConstats.IS_AUTOCORRECTION).trim()) ) {

				mlNode =  MachineLearningRestClient.callMLRestAPI(entityNode);

				if( null != mlNode) {

					dataNode = mlNode;

					mlAutoCoorect=true;

				}

			}

			//PE-6254 ML End.

			if (null != contextDataNode && contextDataNode.has(Constants.CTRYCDOE)) {

				countryCode = contextDataNode.get(Constants.CTRYCDOE).asText();

			}

			

			HashMap<String, JsonNode> dataMap = new HashMap<>();



			contextAndOtherSectionDataToMap(dataMap, otherSecDataNode, contextDataNode, otherAndContextKeys, entry,

					entry1, tempArrayName, isMultiPartRequest);

					//PE-8552

			HashMap<String, JsonNode> contextDataAndOtherDataMap = new HashMap<String, JsonNode>(dataMap);



			ArrayList<JsonNode> columnNodeList = new ArrayList<JsonNode>();

			columnMappingMap = new HashMap<String, String>();

			APIExpressionEvaluator apiExpressionEvaluator = new APIExpressionEvaluator(expressionEvalutionDriver);



			String uuid = setColumnMappingAndDataForAggregation(apiExpressionEvaluator, columnNodeList,

					otherAndContextKeys, columnMappingMap, sectionStructureNode, dataMap, dimMap, dataNode,

					countryCode);



			// Added by Nikhil

			// Start PE-7056 Expression should not executed if columns are not

			// present in MetaData.

			List<String> metaDataColumnsOnly = null;

			metaDataColumnsOnly = DataUtility.extractColumnsFromMetaData(apiExpressionEvaluator);

			if (!otherAndContextKeys.isEmpty()) {

				metaDataColumnsOnly.addAll(otherAndContextKeys);

			}

			// End PE-7056



			Iterator<Map.Entry<String, JsonNode>> iterator = null;

			for (int k = 0; k < dataNode.size(); k++) {

				if(mlAutoCoorect)

				{

				mLdataNode = dataNode.get(k);

				}

				iterator = dataNode.get(k).fields();

				while (iterator.hasNext()) {

					entry = iterator.next();

					//8552

 					JsonNode value = JsonUtils.isNullNode(entry.getValue()) ? null : entry.getValue();

					if (entry.getValue().getNodeType() != JsonNodeType.OBJECT) {

						// PE-6254 ML Integration works start

						if(mlAutoCoorect)

						{

							if(entry.getKey().trim().contains(Constants.UNDERSCORE_ACFLAG)

									|| entry.getKey().trim().contains(Constants.UNDERSCORE_REPLACE))

							{

								continue;

							}

							else

							{	

								if(mLdataNode.get(entry.getKey()+Constants.UNDERSCORE_ACFLAG).asText().equalsIgnoreCase("R"))

								{

									dataMap.put(entry.getKey().trim(), mLdataNode.get(entry.getKey()+Constants.UNDERSCORE_REPLACE));

								}

								else

								{

									dataMap.put(entry.getKey().trim(), value);	

								}

							}

						}

						// PE-6254 ML Integration works end

						else

						{

							dataMap.put(entry.getKey().trim(), value);	

						}

					} else {

						dataMap = parseJsonObjectMap(entry.getKey(), value, dataMap, null,

								mapper.valueToTree(columnNodeList), null); // PE-7605 (2)

					}

				}

				dataMap = updateDataMapWithEmptyDimension(dataMap, dimMap);

				// logger.debug(String.format("dataMap..%s", dataMap));

				resultNode = validate(dataMap, columnNodeList, dataNode.get(k), apiExpressionEvaluator, false,

						columnMappingMap, dimMap, metaDataColumnsOnly,mlAutoCoorect);

				arr.add(resultNode);

				// logger.debug(String.format("resultNode..%s", resultNode));

 				//tempMetaDataColumn.removeAll(dataNodeColumnsOnly);

				/*if (tempMetaDataColumn != null && tempMetaDataColumn.size() > 0) {

					for (String key : tempMetaDataColumn) {

						dataMap.remove(key);

					}

				}*/

				dataMap = new HashMap<String, JsonNode>(contextDataAndOtherDataMap);

 			}



			((ObjectNode) (resultEntiryNode)).set(Constants.DATA, arr);



			if (null != contextDataNode)

				((ObjectNode) (resultEntiryNode)).set(Constants.CONTEXTDATAKEY, contextDataNode);

			entityArr.add(resultEntiryNode);



			// removing csv from expression evaluator

			// added by Nikhil

			apiExpressionEvaluator.getExpressionEvaluatorDriver().removeCsvData(uuid);

			 

			logger.debug(

					"perform_L1_L2Validations >> ExpressionEvalutionDriver Cache Destroyed");

		}

		resultNode = mapper.createObjectNode().set(Constants.ENTITIES, entityArr);

		// logger.info(String.format("%s %s", Constants.EXIT ,Constants.LOG_PARSEANDVALIDATE));

		return resultNode;

	}

	

	private String setColumnMappingAndDataForAggregation(APIExpressionEvaluator apiExpressionEvaluator, ArrayList<JsonNode> columnNodeList,

			List<String> otherAndContextKeys, HashMap<String, String> columnMappingMap, JsonNode sectionStructureNode,

			HashMap<String, JsonNode> dataMap, HashMap<String, JsonNode> dimMap, JsonNode dataNode,

			String countryCode) throws UnsupportedEncodingException, JSONException {

		

		if (sectionStructureNode.has(Constants.COLUMNS)) {

			for (int j = 0; j < sectionStructureNode.get(Constants.COLUMNS).size(); j++) {

				columnNodeList.add(sectionStructureNode.get(Constants.COLUMNS).get(j));

				if (!JsonUtils.isNullOrBlankOrNullNode(

						sectionStructureNode.get(Constants.COLUMNS).get(j).get(Constants.MAPPED_COLUMN_NAME))

						&& !sectionStructureNode.get(Constants.COLUMNS).get(j).get(Constants.MAPPED_COLUMN_NAME).equals(""))

					if (!JsonUtils

							.isNullOrBlankOrNullNode(sectionStructureNode.get(Constants.COLUMNS).get(j).get("code")))

						columnMappingMap.put(sectionStructureNode.get(Constants.COLUMNS).get(j).get("code").asText(),

								sectionStructureNode.get(Constants.COLUMNS).get(j).get(Constants.MAPPED_COLUMN_NAME).asText());

			}

		}



		if (!JsonUtils.isNullOrBlankOrNullNode(dataNode)) {

			JSONArray myArr = new JSONArray(dataNode.toString());

			Set<String> keyList = new HashSet<>();

			for (int j = 0; j < myArr.length(); j++) {

				JSONObject json = myArr.getJSONObject(j);

				Iterator<String> keys = json.keys();

				while (keys.hasNext()) {

					keyList.add(keys.next());

				}

			}

			logger.info("FormValidator >> setColumnMappingAndDataForAggregation() >> CSVParser.parseFile : start");

			long startTImeForCSVParseFile = System.currentTimeMillis();

			CSVParser.parseFile(new ByteArrayInputStream(

					(JsonUtils.rowToString(new JSONArray(keyList)) + JsonUtils.toString(new JSONArray(keyList), myArr))

							.toString().getBytes("UTF-8")));

			logger.info("FormValidator >> setColumnMappingAndDataForAggregation() >> CSVParser.parseFile : end. Took : "

					+ (System.currentTimeMillis() - startTImeForCSVParseFile) + " msec");

		}



		// Set Column - data type mapping - PE

		apiExpressionEvaluator.setColumnDataTypeMapping(new HashMap<String, String>());

		APIExpressionEvaluator.df.setMaximumFractionDigits(16);

		APIExpressionEvaluator.df.setMinimumFractionDigits(1);



		/**

		 * @author Vivek Kasadwar

		 */

		for (JsonNode columnNode : columnNodeList) {

			if ((!JsonUtils.isNullOrBlankOrNullNode(columnNode.get(Constants.CODE))

					&& !StringUtils.isBlank(columnNode.get(Constants.CODE).asText()))

					&& (null != columnNode.get(Constants.DATA_TYPE))) {

				apiExpressionEvaluator.getColumnDataTypeMapping().put(columnNode.get(Constants.CODE).asText(),

						columnNode.get(Constants.DATA_TYPE).asText());

				

				// PE-9734

				

				if (columnNode.get(Constants.CODE).asText().equalsIgnoreCase(Constants.COLUMN_EMP047)) {

					 autoCorrectionUsingDB(dataMap, dataNode, columnNode);

				}

				

				/**

				 * @author Akhileshwar

				 * @throws Exception

				 *             PE-7045

				 */

				if (!JsonUtils.isNullOrBlankOrNullNode(columnNode.get(Constants.DIMENSIONS))) {

					for (JsonNode dimensioNode : columnNode.get(Constants.DIMENSIONS)) {

						apiExpressionEvaluator.getColumnDataTypeMapping().put(

								columnNode.get(Constants.CODE).asText() + Constants.DOT + dimensioNode.asText(),

								columnNode.get(Constants.DATA_TYPE).asText());

					}

					dimMap.put(columnNode.get(Constants.CODE).asText(), columnNode.get(Constants.DIMENSIONS));

				}

			}

		}



		HashMap<String, Object> otherParams = null;

		if (!StringUtils.isEmpty(countryCode)) {

			otherParams = new HashMap<String, Object>();

			otherParams.put("countryCode", countryCode);

		}

		String uuid = apiExpressionEvaluator.getExpressionEvaluatorDriver().setCsvData(CSVParser.dataForAggregation, otherParams);

		return uuid;

	}



	void contextAndOtherSectionDataToMap(HashMap<String, JsonNode> dataMap, JsonNode otherSecDataNode,

			JsonNode contextDataNode, List<String> otherAndContextKeys, Entry<String, JsonNode> entry,

			Entry<String, JsonNode> entry1, String tempArrayName, boolean isMultiPartRequest) throws Exception {



		// Other Section data iterator

		if (!JsonUtils.isNullOrBlankOrNullNode(otherSecDataNode) && !otherSecDataNode.toString().equals("")) {

			Iterator<Map.Entry<String, JsonNode>> osdIt = otherSecDataNode.fields();

			while (osdIt.hasNext()) {

				entry = osdIt.next();

				if (entry.getValue().getNodeType() != JsonNodeType.OBJECT) {

					dataMap.put(Constants.OTHERSECTIONDATAKEY + "." + entry.getKey().trim(), entry.getValue());

					// PE-7056

					otherAndContextKeys.add(Constants.OTHERSECTIONDATAKEY + "." + entry.getKey().trim());

				} else {

					dataMap = parseJsonObjectMap(entry.getKey().trim(), entry.getValue(), dataMap,

							Constants.OTHERSECTIONDATAKEY, null, otherAndContextKeys);

				}

			}

			entry = null;

		}



		// Context data node iterator

		// Populate dataMap with values from "contextDataNode"

		if (!JsonUtils.isNullOrBlankOrNullNode(contextDataNode) && !contextDataNode.toString().equals("")) {

			Iterator<Map.Entry<String, JsonNode>> contextIt = contextDataNode.fields();

			while (contextIt.hasNext()) {

				entry = contextIt.next();

				if (entry.getValue().getNodeType() != JsonNodeType.OBJECT

						&& entry.getValue().getNodeType() != JsonNodeType.ARRAY) {

					dataMap.put(Constants.CONTEXTDATAKEY + "." + entry.getKey().trim(), entry.getValue());

					// PE-7056

					otherAndContextKeys.add(Constants.CONTEXTDATAKEY + "." + entry.getKey().trim());



				} else if (entry.getValue().getNodeType() == JsonNodeType.ARRAY && (entry.getValue().size() > 0)) {

					Iterator<Map.Entry<String, JsonNode>> arrIt = contextDataNode.fields();

					tempArrayName = entry.getKey();

					// PE - 7257

					if (entry.getValue().getNodeType() == JsonNodeType.ARRAY && isMultiPartRequest) {

						dataMap.put(Constants.CONTEXTDATAKEY + "." + entry.getKey().trim(), entry.getValue());

						// PE-7056

						otherAndContextKeys.add(Constants.CONTEXTDATAKEY + "." + entry.getKey().trim());

					} else {

						if (entry.getValue().size() > 0) {

							arrIt = entry.getValue().get(0).fields();

							while (arrIt.hasNext()) {

								entry1 = arrIt.next();

								dataMap.put(

										Constants.CONTEXTDATAKEY + "." + tempArrayName + "." + entry1.getKey().trim(),

										entry1.getValue());

								// PE-7056

								otherAndContextKeys.add(

										Constants.CONTEXTDATAKEY + "." + tempArrayName + "." + entry1.getKey().trim());

							}

						}

					}



				} else {

					dataMap = parseJsonObjectMap(entry.getKey(), entry.getValue(), dataMap, Constants.CONTEXTDATAKEY,

							null, otherAndContextKeys);



				}

			}

			entry = null;

		}

	}

		

	public JsonNode performValidateAggregateRequest(JsonNode entitiesNodeResponse, boolean isAggregateRequest, boolean isMultiPartRequest) throws Exception {



		JsonNode dataNode = null;

		HashMap<String, String> columnMappingMap = null;

		JsonNode sectionStructureNode = null;

		HashMap<String, JsonNode> dimMap = new HashMap<String, JsonNode>();

		String countryCode = null;

		JsonNode otherSecDataNode = null;

		JsonNode contextDataNode = null;

		JsonNode resultNode = null;

		ArrayNode entityArr = null;

 		JsonNode entityNode = null;

		JsonNode resultEntiryNode = null;

		Map.Entry<String, JsonNode> entry = null;

		Map.Entry<String, JsonNode> entry1 = null;

		List<String> otherAndContextKeys = new ArrayList<>();

		String tempArrayName = null;

		entityArr = mapper.createArrayNode();



		ArrayNode entitiesNode = (ArrayNode) entitiesNodeResponse.get(Constants.ENTITIES);



		// Making csv out of JSON single level

		for (int i = 0; i < entitiesNode.size(); i++) {

			countryCode = "";

 			entityNode = entitiesNode.get(i);

			resultEntiryNode = mapper.createObjectNode();

			

			if (entityNode.has(Constants.SECTION_STRUCTURE)) {

				sectionStructureNode = entityNode.get(Constants.SECTION_STRUCTURE);

			}

			if (entityNode.has(Constants.DATA)) {

				dataNode = entityNode.get(Constants.DATA);

			}

			if (entityNode.has(Constants.OTHERSECTIONDATAKEY)) {

				otherSecDataNode = entityNode.get(Constants.OTHERSECTIONDATAKEY);

			}

			if (entityNode.has(Constants.CONTEXTDATAKEY)) {

				contextDataNode = entityNode.get(Constants.CONTEXTDATAKEY);

			}



			if (null != contextDataNode && contextDataNode.has(Constants.CTRYCDOE)) {

				countryCode = contextDataNode.get(Constants.CTRYCDOE).asText();

			}

			

			HashMap<String, JsonNode> dataMap = new HashMap<>();



			contextAndOtherSectionDataToMap(dataMap, otherSecDataNode, contextDataNode, otherAndContextKeys, entry,

					entry1, tempArrayName, isMultiPartRequest);



			ArrayList<JsonNode> columnNodeList = new ArrayList<JsonNode>();

			columnMappingMap = new HashMap<>();

			

			//Creating a ExpressionEvalutionDriver at a request level(Per Entity (Eg. Employee section))

			ExpressionEvalutionDriver expressionEvalutionDriver = new ExpressionEvalutionDriver();

			

			logger.debug("performValidateAggregateRequest >> expressionEvalutionDriver Cache : {}",

					expressionEvalutionDriver.getAggregateFunctionCache());



			APIExpressionEvaluator apiExpressionEvaluator = new APIExpressionEvaluator(expressionEvalutionDriver);



			String uuid = setColumnMappingAndDataForAggregation(apiExpressionEvaluator, columnNodeList,

					otherAndContextKeys, columnMappingMap, sectionStructureNode, dataMap, dimMap, dataNode,

					countryCode);



			// Added by Nikhil

			// Start PE-7056 Expression should not executed if columns are not

			// present in MetaData.

			List<String> metaDataColumnsOnly = null;

			metaDataColumnsOnly = DataUtility.extractColumnsFromMetaData(apiExpressionEvaluator);

			if (!otherAndContextKeys.isEmpty()) {

				metaDataColumnsOnly.addAll(otherAndContextKeys);

			}

			// End PE-7056



			// PE : 2449 to update datamap with entity data which was

			// missing in case of aggregate



			Iterator<Map.Entry<String, JsonNode>> iterator = null;

			

			if(dataNode.isArray() && dataNode.size()!=0) {	// Remove NullPointer when there is no object in data-node.

				iterator = dataNode.get(0).fields();

				while (iterator.hasNext()) {

					entry = iterator.next();

					JsonNode value = JsonUtils.isNullNode(entry.getValue()) ? null : entry.getValue();

					if (entry.getValue().getNodeType() != JsonNodeType.OBJECT) {

						dataMap.put(entry.getKey().trim(), value);

					} else {

						dataMap = parseJsonObjectMap(entry.getKey(), value, dataMap, null, null, null);

					}

				}

			}

			resultNode = validate(dataMap, columnNodeList, dataNode.get(0), apiExpressionEvaluator, true,

					columnMappingMap, dimMap, metaDataColumnsOnly,false);



			((ObjectNode) (resultEntiryNode)).set(Constants.VALIDATION_RESULTS, resultNode);

			if (null != contextDataNode)

				((ObjectNode) (resultEntiryNode)).set(Constants.CONTEXTDATAKEY, contextDataNode);

			entityArr.add(resultEntiryNode);

			apiExpressionEvaluator.getExpressionEvaluatorDriver().removeCsvData(uuid);



		}

		

		resultNode = mapper.createObjectNode().set(Constants.ENTITIES, entityArr);

		// logger.info(String.format("%s %s", Constants.EXIT ,Constants.LOG_PARSEANDVALIDATE));

		return resultNode;

	}

	// PE-6254 ML Integration works start

	public ObjectNode prepareMLAutocorrectObject(String field,String validationType,JsonNode originalvalue)

	{

		ObjectNode validation = mapper.createObjectNode();

		validation.put(Constants.FIELD, field);

		validation.put(Constants.ERROR_TYPE, Constants.AUTOCORRECT);

		// validation.put(Constants.MESSAGE, result);

		validation.put(Constants.MESSAGE_KEY,

				in.lnt.utility.general.Cache.getPropertyFromError(ErrorCacheConstant.ERR_059));

		if(!JsonUtils.isNullOrBlankOrNullNode((originalvalue)))

		validation.put(Constants.ORIGINAL_VALUE,originalvalue.asText());

		else

		validation.put(Constants.ORIGINAL_VALUE,"");

		return validation;

	}

	public void removeDataNodeACFields(JsonNode dataNode)

	{

		Iterator<Map.Entry<String, JsonNode>> iterator = null;

		iterator = dataNode.fields();

		//dataNode.

		List<String> fieldsToDelete=new ArrayList<String>();

		Map.Entry<String, JsonNode> entry = null;

		while (iterator.hasNext()) {

			entry = iterator.next();

			if(entry.getKey().trim().contains(Constants.UNDERSCORE_ACFLAG) || entry.getKey().trim().contains(Constants.UNDERSCORE_REPLACE))

			{}

			else{

				fieldsToDelete.add(entry.getKey());

			}

		}

		((ObjectNode)dataNode).retain(fieldsToDelete);

	}

	// PE-6254 ML Integration works end

	



	/**

	 * @param dataMap

	 * @param dataNode

	 * @param columnNode

	 * @return

	 */

	// PE-9734

	public void autoCorrectionUsingDB(Map<String, JsonNode> dataMap, JsonNode dataNode, JsonNode columnNode) {

		String replacedOption = null;

		String originalValue = null;

		StringBuilder runTimeQuery = new StringBuilder();

		for (JsonNode node : dataNode) {

			runTimeQuery.setLength(0);

			runTimeQuery = new StringBuilder(" SELECT REPLACEMENT FROM region_lookup WHERE  \"MATCH\"  = ");



			runTimeQuery.append("'");

			if (!JsonUtils.isNullOrBlankOrNullNode(node.get(Constants.COLUMN_EMP047))) {

				originalValue = node.get(Constants.COLUMN_EMP047).asText();

				runTimeQuery.append(originalValue);

			} else {

				continue;

			}

			runTimeQuery.append("'");



			if (!JsonUtils.isNullOrBlankOrNullNode(dataMap.get(Constants.COTEXT_COUNTRY_CODE))) {

				runTimeQuery.append(" AND CTX_CTRY_CODE = ");

				runTimeQuery.append("'");

				runTimeQuery.append(dataMap.get(Constants.COTEXT_COUNTRY_CODE).asText());

				runTimeQuery.append("'");

			}



			try {

				replacedOption = DBUtils.getDataFromDerbyTable(runTimeQuery.toString());

			} catch (SQLException ex) {

				logger.error("Exception occured in getRangeDetails..{}", ex);

				replacedOption = null;

			}



			if (null != replacedOption && !StringUtils.equalsIgnoreCase(replacedOption, originalValue)) {

				((ObjectNode) node).put(Constants.COLUMN_EMP047 + Constants.UNDERSCORE_REPLACE,

						FormValidator_CSC.getOptionFromDropDown(columnNode, replacedOption));

				((ObjectNode) node).put(Constants.COLUMN_EMP047 + Constants.UNDERSCORE_ACFLAG, "R");

			} else {

				((ObjectNode) node).put(Constants.COLUMN_EMP047 + Constants.UNDERSCORE_REPLACE, originalValue);

				((ObjectNode) node).put(Constants.COLUMN_EMP047 + Constants.UNDERSCORE_ACFLAG, "O");

			}

		}

	}



	

}