/AppLib/CE.InsightsBulk.LoadData/ActionItemFile.cs

using CE.InsightsBulk.Infrastructure;
using CE.InsightsBulk.Infrastructure.DataAccess;
using CE.InsightsBulk.Infrastructure.Interfaces;
using CE.InsightsBulk.Infrastructure.SharedClasses;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Xml;
using System.Xml.Linq;
using System.Xml.Schema;
using Aclara.UFx.StatusManagement;
using CE.InsightsBulk.Model;

namespace CE.InsightsBulk.LoadData
{
    public class ActionItemFile : IBulkFile
    {
        private readonly int _clientId;
        private readonly IConfig _config;
        private readonly ILogger _log;

        private const string PremiseNode_Customerid = "CustomerId";
        private const string PremiseNode_Accountid = "AccountId";
        private const string PremiseNode_PremiseId = "PremiseId";
        private const string ActionItemNode_ActionKey = "ActionKey";
        private const string ActionItemNode_SubActionKey = "SubActionKey";
        private const string ActionItemNode_Status = "Status";
        private const string ActionItemNode_StatusDate = "StatusDate";
        private const string ActionItemNode_Source = "Source";
        private const string ActionKeyData_ActionData = "ActionData";
        private const string ActionKeyData_ActionValue = "ActionValue";

        /// <summary>
        /// Constructor
        /// </summary>
        /// <param name="o"></param>
        /// <param name="config"></param>
        /// <param name="log"></param>
        public ActionItemFile(Options o, IConfig config, ILogger log)
        {
            _clientId = Convert.ToInt32(o.ClientId);
            _config = config;
            _log = log;
        }

        /// <summary>
        /// Process the Action Item file
        /// </summary>
        /// <param name="theFile">The file to be processed</param>
        /// <param name="statusList">The Status</param>
        /// <returns>The tracking id</returns>
        public string ProcessFile(FileInfo theFile, out StatusList statusList)
        {
            var trackingid = "";
            statusList = new StatusList();
            var nodeCounter = 0;
            var totalNodeCounter = 0;
            var processedNodeCounter = 0;

            var attributeTable = new List<ActionItem>();
            var da = new BulkDataAccess();
            var daMeta = new MetaDataAccess();

            try
            {
                statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + " Processing File " + theFile.FullName,
                    StatusTypes.StatusSeverity.Informational));

                _log.Info(trackingid, $"Processing File{theFile.FullName} ");

                DateTime trackingIdDate;
                Helpers.SetTrackingData(theFile.Name, out trackingid, out trackingIdDate);
                var xsdFile = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" +
                                  _config.InputXSD;

                // This logic sets-up a custom iteration using linq and yield.  Each element is traversed forward-only, greatly saving resources.
                var getCustomerElements = InputFileXMLStream.StreamData(theFile.FullName).Select(el => el);

                var schemas = new XmlSchemaSet();
                schemas.Add(Constants.FileNamespace, XmlReader.Create(new StreamReader(xsdFile)));

                foreach (var element in getCustomerElements)
                {
                    //Validate the node
                    var errors = false;
                    var errStatusList = new StatusList();
                    var doc = new XDocument(new XElement(Constants.RootNode));
                    var xElement = doc.Element(Constants.RootNode);
                    xElement?.Add(element);

                    doc.Validate(schemas, (o, e) =>
                    {
                        errStatusList.Add(new Status(e.Message, StatusTypes.StatusSeverity.Error));

                        string elementData = string.Concat(element.Elements());

                        daMeta.LogErrorsByInputType(
                            Helpers.ExtractElements(elementData, BulkLoadConstants.RunType.ActionItem, _clientId,
                                trackingid), e.Message, BulkLoadConstants.RunType.ActionItem,
                            _config.ImportSpecificLoggingOn);

                        _log.ErrorException(trackingid,
                            new InsightsBulkLoadException(
                                $"Validation Error for file {theFile.FullName} data: {elementData}", e.Exception));

                        errors = true;

                    }, true);

                    if (nodeCounter == 0)
                    {
                        //Create the datatable for storage
                        attributeTable = new List<ActionItem>();
                    }

                    // Main Call
                    if (!errors)
                    {
                        nodeCounter++;
                        var xmlNode = element.GetXmlNode();
                        ProcessNode(trackingid, trackingIdDate, xmlNode, attributeTable);
                    }
                    else
                    {
                        statusList.AddRange(errStatusList);
                        errors = false;
                        errStatusList.Clear();
                    }

                    totalNodeCounter++;
                    if (attributeTable.Count >= _config.MaxBatchSize)
                    {
                        //WriteOutdata
                        processedNodeCounter += nodeCounter;
                        var results = da.WriteActionKeyData(attributeTable, ref statusList);
                        nodeCounter = 0;
                        if (results != attributeTable.Count)
                        {
                            statusList.Add(new Status("Not all data was inserted", StatusTypes.StatusSeverity.Error));
                        }
                        attributeTable.Clear();

                    }
                }

                if (attributeTable.Count > 0)
                {
                    processedNodeCounter += nodeCounter;
                    var results = da.WriteActionKeyData(attributeTable, ref statusList);

                    if (results != attributeTable.Count)
                    {
                        statusList.Add(new Status("Not all data was inserted", StatusTypes.StatusSeverity.Error));
                        _log.ErrorException(trackingid, new InsightsBulkLoadException("Not all data was inserted"));
                    }
                }

                statusList.Add(
                    new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + " Processed " + processedNodeCounter + " customers out of " +
                        totalNodeCounter, StatusTypes.StatusSeverity.Informational));

                _log.Info(trackingid,$"Processed {processedNodeCounter} customers out of {totalNodeCounter}");

            }
            catch (Exception ex)
            {
                statusList.Add(new Status(ex, StatusTypes.StatusSeverity.Error));
                _log.ErrorException(trackingid, ex);

            }

            return trackingid;
        }

        /// <summary>
        /// Process the XML Node
        /// </summary>
        /// <param name="trackingid">The tracking Id</param>
        /// <param name="trackingdate">The tracking date</param>
        /// <param name="rootNode">The xml node</param>
        /// <param name="actionItemTable">The data table</param>
        private void ProcessNode(string trackingid, DateTime trackingdate, XmlNode rootNode,
            ICollection<ActionItem> actionItemTable)
        {
            //For some reason, customer is coming in as a child node, so just get all children
            var cNode = rootNode.FirstChild;
            if (!cNode.HasChildNodes)
            {
                return;
            }

            //Load the Premise Attribute Table
            foreach (XmlElement accountNode in cNode.ChildNodes)
            {
                if (!accountNode.HasChildNodes)
                {
                    continue;
                }

                foreach (XmlElement premiseNode in accountNode.ChildNodes)
                {
                    foreach (XmlElement actionNode in premiseNode.ChildNodes)
                    {
                        foreach (XmlElement actionDataNode in actionNode.ChildNodes)
                        {
                            var actionItem = new ActionItem
                            {
                                ClientId = _clientId,
                                CustomerId = Helpers.GetText(cNode, PremiseNode_Customerid),
                                AccountID = Helpers.GetText(accountNode, PremiseNode_Accountid),
                                PremiseId = Helpers.GetText(premiseNode, PremiseNode_PremiseId),
                                ActionKey = Helpers.GetText(actionNode, ActionItemNode_ActionKey),
                                SubActionKey = Helpers.GetText(actionNode, ActionItemNode_SubActionKey),
                                ActionData = Helpers.GetText(actionDataNode, ActionKeyData_ActionData),
                                ActionDataValue = Helpers.GetText(actionDataNode, ActionKeyData_ActionValue),
                                TrackingID = trackingid,
                                TrackingDate = trackingdate,
                                StatusId = Helpers.ConvertStatusToId(Helpers.GetText(actionNode, ActionItemNode_Status)),
                                SourceId = Helpers.ConvertSourceToId(Helpers.GetText(actionNode, ActionItemNode_Source))
                            };

                            var strDateTime = Helpers.GetText(actionNode, ActionItemNode_StatusDate);
                            actionItem.StatusDate = Convert.ToDateTime(strDateTime.Length == 0 ? "1900-01-01" : strDateTime);

                            actionItemTable.Add(actionItem);
                        }
                    }
                }
            }
        }
    }
}