/AppLib/CE.InsightsDW.ImportData.ConsoleApp/ProfileInputFile.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Xml;
using System.Xml.Linq;
using System.Xml.Schema;
using Aclara.UFx.StatusManagement;
using CE.InsightsDW.ImportData.Infastructure;
using CE.InsightsDW.ImportData.Infastructure.DataAccess;
using CE.InsightsDW.Model;

// ReSharper disable once CheckNamespace
namespace CE.InsightsDW.ImportData
{
    public class ProfileInputFile : IProcessFile
    {
        private readonly int _clientId;
        private readonly IConfig _config;

        private const string PremiseNode_Customerid = "CustomerId";
        private const string PremiseNode_Accountid = "AccountId";
        private const string PremiseNode_PremiseId = "PremiseId";
        private const string AttributeNode_Source = "Source";
        private const string AttributeNode_ModifiedDate = "ModifiedDate";
        private const string AttributeNode_CreatedDate = "CreateDate";
        private const string AttributeNode_Key = "AttributeKey";
        private const string AttributeNode_Value = "AttributeValue";

        private Infastructure.InsightsDW _daInsightsDw;
        private InsightsMetadata _daMetadata;

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="options"></param>
        /// <param name="config"></param>
        public ProfileInputFile(Options options, IConfig config)
        {
            _clientId = Convert.ToInt32(options.ClientId);
            _config = config;
        }

        /// <summary>
        /// Process the Profile file
        /// </summary>
        /// <param name="theFile">The file to be processed</param>
        /// <param name="emailMessageContents">Contents of error email message</param>
        /// <param name="logMessageContents">Messages for the log file</param>
        /// <param name="statusList">The Status</param>
        /// <returns>The tracking id</returns>
        public string ProcessFile(FileInfo theFile, ICollection<string> emailMessageContents, ICollection<LoggerMessageModel> logMessageContents, 
            CountsForReportingModel captureCountsForReporting, out StatusList statusList)
        {

            var trackingid = string.Empty;
            statusList = new StatusList();
            var nodeCounter = 0;
            var totalNodeCounter = 0;
            var processedNodeCounter = 0;
            var attributeTable = new List<Holding_PremiseAttribute>();

            _daInsightsDw = new Infastructure.InsightsDW(_config.ShardName);
            _daMetadata = new InsightsMetadata();

            try
            {
                DateTime trackingIdDate;
                Helpers.SetTrackingData(theFile.Name, out trackingid, out trackingIdDate);
                var xsdFile = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" +
                                  _config.InputXSD;


                // This logic sets-up a custom iteration using linq and yield.  Each element is traversed forward-only, greatly saving resources.
                var getCustomerElements = InputFileXMLStream.StreamData(theFile.FullName).Select(el => el);

                var schemas = new XmlSchemaSet();
                schemas.Add(Constants.FileNamespace, XmlReader.Create(new StreamReader(xsdFile)));

                foreach (var element in getCustomerElements)
                {
                    //Validate the node
                    var errors = false;

                    var errStatusList = new StatusList();

                    var doc = new XDocument(new XElement(Constants.RootNode));
                    var xElement = doc.Element(Constants.RootNode);
                    xElement?.Add(element);

                    doc.Validate(schemas, (o, e) =>
                    {
                        errStatusList.Add(new Status(e.Message, StatusTypes.StatusSeverity.Error));

                        var elementData = string.Concat(element.Elements());

                        _daInsightsDw.LogErrorsByInputType(
                            Helpers.ExtractElements(elementData, DWImportConstants.RunType.Profile, _clientId,
                                trackingid), e.Message, theFile.Name, DWImportConstants.RunType.Profile,
                            _config.ImportSpecificLoggingOn);

                        logMessageContents.Add(new LoggerMessageModel
                        {
                            LogLevel = NLog.LogLevel.Error,
                            Exception = new InsightsDWImportException(
                                $"Validation Error for file {theFile.Name} data: {elementData}",
                                e.Exception)
                        });

                        errors = true;

                    }, true);


                    if (nodeCounter == 0)
                    {
                        //Create the datatable for storage
                        attributeTable = new List<Holding_PremiseAttribute>(); 
                    }

                    // Main Call
                    if (!errors)
                    {
                        try
                        {

                            var xmlNode = InputFileXMLStream.GetXmlNode(element);
                            ProcessNode(trackingid, trackingIdDate, xmlNode, attributeTable);
                            nodeCounter++;

                        }
                        catch (Exception eX)
                        {
                            errStatusList.Add(new Status(eX.Message, StatusTypes.StatusSeverity.Error));

                            var elementData = string.Concat(element.Elements());

                            _daInsightsDw.LogErrorsByInputType(
                                Helpers.ExtractElements(elementData, DWImportConstants.RunType.Profile, _clientId, trackingid),
                                eX.Message, theFile.Name, DWImportConstants.RunType.Profile, _config.ImportSpecificLoggingOn);

                            logMessageContents.Add(new LoggerMessageModel
                            {
                                LogLevel = NLog.LogLevel.Error,
                                Exception =
                                    new InsightsDWImportException(
                                        $"Processing Error for file {theFile.Name} data: " + eX.Message)
                            });

                            errors = true;
                        }
                    }

                    if (errors)
                    {
                        statusList.AddRange(errStatusList);
                        errors = false;
                        errStatusList.Clear();
                    }

                    totalNodeCounter++;

                    // Write out data if reached max batch size
                    if (attributeTable.Count >= _config.MaxBatchSize)
                    {
                        processedNodeCounter += nodeCounter;
                        var results = _daInsightsDw.WritePremiseAttributeData(attributeTable);
                        nodeCounter = 0;

                        if (results != attributeTable.Count)
                        {
                            var logMessage1 = "Not all Profile data was inserted. Only " + results + " out of  " + attributeTable.Count + " saved to database.";
                            statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage1, StatusTypes.StatusSeverity.Error));
                            logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage1 });
                            emailMessageContents.Add(logMessage1);
                        }
                        attributeTable.Clear();

                    }
                }

                // End of loop. Write what is left.
                if (attributeTable.Count > 0)
                {
                    processedNodeCounter += nodeCounter;

                    //Less than max rows, so write it out
                    var results = _daInsightsDw.WritePremiseAttributeData(attributeTable); 

                    if (results != attributeTable.Count)
                    {
                        var logMessage1 = "Not all Profile data was inserted. Only " + results + " out of  " + attributeTable.Count + " saved to database.";
                        statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage1, StatusTypes.StatusSeverity.Error));
                        logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage1 });
                        emailMessageContents.Add(logMessage1);
                    }
                }

                var logMessage = ">>>> Processed " + processedNodeCounter + " Profiles out of " + totalNodeCounter + " <<<<"; 

                statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage, StatusTypes.StatusSeverity.Informational));
                logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage });
                emailMessageContents.Add(logMessage);

                captureCountsForReporting.Count = processedNodeCounter;
                captureCountsForReporting.Comment = logMessage;

            }
            catch (Exception ex)
            {
                statusList.Add(new Status(ex, StatusTypes.StatusSeverity.Error));
                logMessageContents.Add(new LoggerMessageModel
                {
                    LogLevel = NLog.LogLevel.Error,
                    Exception = null,
                    MessageToLog = ex.Message
                });
            }

            return trackingid;
        }

        /// <summary>
        /// Process the XML Node
        /// </summary>
        /// <param name="trackingid">The tracking Id</param>
        /// <param name="trackingdate">The tracking date</param>
        /// <param name="rootNode">The xml node</param>
        /// <param name="premiseAttributeTable">The data table</param>
        private void ProcessNode(string trackingid, DateTime trackingdate, XmlNode rootNode,
            ICollection<Holding_PremiseAttribute> premiseAttributeTable)
        {
            //For some reason, customer is coming in as a child node, so just get all children
            var cNode = rootNode.FirstChild;

            //Load the Premise Attribute Table
            if (!cNode.HasChildNodes)
            {
                return;
            }

            foreach (XmlElement accountNode in cNode.ChildNodes)
            {
                if (!accountNode.HasChildNodes)
                {
                    continue;
                }

                foreach (XmlElement premiseNode in accountNode.ChildNodes)
                {
                    foreach (XmlElement attributeNode in premiseNode.ChildNodes)
                    {
                        var premiseAttribute = new Holding_PremiseAttribute
                        {
                            ClientID = _clientId,
                            CustomerID = Helpers.GetText(cNode, PremiseNode_Customerid),
                            AccountID = Helpers.GetText(accountNode, PremiseNode_Accountid),
                            PremiseID = Helpers.GetText(premiseNode, PremiseNode_PremiseId),
                            TrackingID = trackingid,
                            TrackingDate = trackingdate,
                            OptionValue = Helpers.GetText(attributeNode, AttributeNode_Value),
                            AttributeKey = Helpers.GetText(attributeNode, AttributeNode_Key),
                            UpdateDate = Convert.ToDateTime(Helpers.GetText(attributeNode, AttributeNode_ModifiedDate)),
                            SourceId = _daInsightsDw.ValidateAndGetSource(Helpers.GetText(attributeNode, AttributeNode_Source))
                        };

                        // Check if valid Profile Attribute
                        if (!_daMetadata.IsValidClientProfileAttribute(premiseAttribute.AttributeKey))
                        {
                            throw new ApplicationException("Invalid Premise Attribute Key  : " + premiseAttribute.AttributeKey);
                        }

                        var strDateTime = Helpers.GetText(attributeNode, AttributeNode_CreatedDate);
                        if (strDateTime.Length == 0)
                        {
                            premiseAttribute.CreateDate = null;
                        }
                        else
                        {
                            premiseAttribute.CreateDate = Convert.ToDateTime(strDateTime);
                        }

                        premiseAttributeTable.Add(premiseAttribute);
                    }
                }
            }
        }
    }
}