/AppLib/CE.InsightsDW.ImportData.ConsoleApp/ProfileInputFile.cs
https://bitbucket.org/jeffmccommas/acex · C# · 291 lines · 211 code · 49 blank · 31 comment · 15 complexity · 349338083e36c640d2381b55ec48e653 MD5 · raw file
- using System;
- using System.Collections.Generic;
- using System.Globalization;
- using System.IO;
- using System.Linq;
- using System.Reflection;
- using System.Xml;
- using System.Xml.Linq;
- using System.Xml.Schema;
- using Aclara.UFx.StatusManagement;
- using CE.InsightsDW.ImportData.Infastructure;
- using CE.InsightsDW.ImportData.Infastructure.DataAccess;
- using CE.InsightsDW.Model;
- // ReSharper disable once CheckNamespace
- namespace CE.InsightsDW.ImportData
- {
- public class ProfileInputFile : IProcessFile
- {
- private readonly int _clientId;
- private readonly IConfig _config;
- private const string PremiseNode_Customerid = "CustomerId";
- private const string PremiseNode_Accountid = "AccountId";
- private const string PremiseNode_PremiseId = "PremiseId";
- private const string AttributeNode_Source = "Source";
- private const string AttributeNode_ModifiedDate = "ModifiedDate";
- private const string AttributeNode_CreatedDate = "CreateDate";
- private const string AttributeNode_Key = "AttributeKey";
- private const string AttributeNode_Value = "AttributeValue";
- private Infastructure.InsightsDW _daInsightsDw;
- private InsightsMetadata _daMetadata;
- /// <summary>
- /// Constructor
- /// </summary>
- /// <param name="options"></param>
- /// <param name="config"></param>
- public ProfileInputFile(Options options, IConfig config)
- {
- _clientId = Convert.ToInt32(options.ClientId);
- _config = config;
- }
- /// <summary>
- /// Process the Profile file
- /// </summary>
- /// <param name="theFile">The file to be processed</param>
- /// <param name="emailMessageContents">Contents of error email message</param>
- /// <param name="logMessageContents">Messages for the log file</param>
- /// <param name="statusList">The Status</param>
- /// <returns>The tracking id</returns>
- public string ProcessFile(FileInfo theFile, ICollection<string> emailMessageContents, ICollection<LoggerMessageModel> logMessageContents,
- CountsForReportingModel captureCountsForReporting, out StatusList statusList)
- {
- var trackingid = string.Empty;
- statusList = new StatusList();
- var nodeCounter = 0;
- var totalNodeCounter = 0;
- var processedNodeCounter = 0;
- var attributeTable = new List<Holding_PremiseAttribute>();
- _daInsightsDw = new Infastructure.InsightsDW(_config.ShardName);
- _daMetadata = new InsightsMetadata();
- try
- {
- DateTime trackingIdDate;
- Helpers.SetTrackingData(theFile.Name, out trackingid, out trackingIdDate);
- var xsdFile = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\" +
- _config.InputXSD;
- // This logic sets-up a custom iteration using linq and yield. Each element is traversed forward-only, greatly saving resources.
- var getCustomerElements = InputFileXMLStream.StreamData(theFile.FullName).Select(el => el);
- var schemas = new XmlSchemaSet();
- schemas.Add(Constants.FileNamespace, XmlReader.Create(new StreamReader(xsdFile)));
- foreach (var element in getCustomerElements)
- {
- //Validate the node
- var errors = false;
- var errStatusList = new StatusList();
- var doc = new XDocument(new XElement(Constants.RootNode));
- var xElement = doc.Element(Constants.RootNode);
- xElement?.Add(element);
- doc.Validate(schemas, (o, e) =>
- {
- errStatusList.Add(new Status(e.Message, StatusTypes.StatusSeverity.Error));
- var elementData = string.Concat(element.Elements());
- _daInsightsDw.LogErrorsByInputType(
- Helpers.ExtractElements(elementData, DWImportConstants.RunType.Profile, _clientId,
- trackingid), e.Message, theFile.Name, DWImportConstants.RunType.Profile,
- _config.ImportSpecificLoggingOn);
- logMessageContents.Add(new LoggerMessageModel
- {
- LogLevel = NLog.LogLevel.Error,
- Exception = new InsightsDWImportException(
- $"Validation Error for file {theFile.Name} data: {elementData}",
- e.Exception)
- });
- errors = true;
- }, true);
- if (nodeCounter == 0)
- {
- //Create the datatable for storage
- attributeTable = new List<Holding_PremiseAttribute>();
- }
- // Main Call
- if (!errors)
- {
- try
- {
- var xmlNode = InputFileXMLStream.GetXmlNode(element);
- ProcessNode(trackingid, trackingIdDate, xmlNode, attributeTable);
- nodeCounter++;
- }
- catch (Exception eX)
- {
- errStatusList.Add(new Status(eX.Message, StatusTypes.StatusSeverity.Error));
- var elementData = string.Concat(element.Elements());
- _daInsightsDw.LogErrorsByInputType(
- Helpers.ExtractElements(elementData, DWImportConstants.RunType.Profile, _clientId, trackingid),
- eX.Message, theFile.Name, DWImportConstants.RunType.Profile, _config.ImportSpecificLoggingOn);
- logMessageContents.Add(new LoggerMessageModel
- {
- LogLevel = NLog.LogLevel.Error,
- Exception =
- new InsightsDWImportException(
- $"Processing Error for file {theFile.Name} data: " + eX.Message)
- });
- errors = true;
- }
- }
- if (errors)
- {
- statusList.AddRange(errStatusList);
- errors = false;
- errStatusList.Clear();
- }
- totalNodeCounter++;
- // Write out data if reached max batch size
- if (attributeTable.Count >= _config.MaxBatchSize)
- {
- processedNodeCounter += nodeCounter;
- var results = _daInsightsDw.WritePremiseAttributeData(attributeTable);
- nodeCounter = 0;
- if (results != attributeTable.Count)
- {
- var logMessage1 = "Not all Profile data was inserted. Only " + results + " out of " + attributeTable.Count + " saved to database.";
- statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage1, StatusTypes.StatusSeverity.Error));
- logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage1 });
- emailMessageContents.Add(logMessage1);
- }
- attributeTable.Clear();
- }
- }
- // End of loop. Write what is left.
- if (attributeTable.Count > 0)
- {
- processedNodeCounter += nodeCounter;
- //Less than max rows, so write it out
- var results = _daInsightsDw.WritePremiseAttributeData(attributeTable);
- if (results != attributeTable.Count)
- {
- var logMessage1 = "Not all Profile data was inserted. Only " + results + " out of " + attributeTable.Count + " saved to database.";
- statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage1, StatusTypes.StatusSeverity.Error));
- logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage1 });
- emailMessageContents.Add(logMessage1);
- }
- }
- var logMessage = ">>>> Processed " + processedNodeCounter + " Profiles out of " + totalNodeCounter + " <<<<";
- statusList.Add(new Status(DateTime.Now.ToString(CultureInfo.InvariantCulture) + logMessage, StatusTypes.StatusSeverity.Informational));
- logMessageContents.Add(new LoggerMessageModel { LogLevel = NLog.LogLevel.Info, MessageToLog = logMessage });
- emailMessageContents.Add(logMessage);
- captureCountsForReporting.Count = processedNodeCounter;
- captureCountsForReporting.Comment = logMessage;
- }
- catch (Exception ex)
- {
- statusList.Add(new Status(ex, StatusTypes.StatusSeverity.Error));
- logMessageContents.Add(new LoggerMessageModel
- {
- LogLevel = NLog.LogLevel.Error,
- Exception = null,
- MessageToLog = ex.Message
- });
- }
- return trackingid;
- }
- /// <summary>
- /// Process the XML Node
- /// </summary>
- /// <param name="trackingid">The tracking Id</param>
- /// <param name="trackingdate">The tracking date</param>
- /// <param name="rootNode">The xml node</param>
- /// <param name="premiseAttributeTable">The data table</param>
- private void ProcessNode(string trackingid, DateTime trackingdate, XmlNode rootNode,
- ICollection<Holding_PremiseAttribute> premiseAttributeTable)
- {
- //For some reason, customer is coming in as a child node, so just get all children
- var cNode = rootNode.FirstChild;
- //Load the Premise Attribute Table
- if (!cNode.HasChildNodes)
- {
- return;
- }
- foreach (XmlElement accountNode in cNode.ChildNodes)
- {
- if (!accountNode.HasChildNodes)
- {
- continue;
- }
- foreach (XmlElement premiseNode in accountNode.ChildNodes)
- {
- foreach (XmlElement attributeNode in premiseNode.ChildNodes)
- {
- var premiseAttribute = new Holding_PremiseAttribute
- {
- ClientID = _clientId,
- CustomerID = Helpers.GetText(cNode, PremiseNode_Customerid),
- AccountID = Helpers.GetText(accountNode, PremiseNode_Accountid),
- PremiseID = Helpers.GetText(premiseNode, PremiseNode_PremiseId),
- TrackingID = trackingid,
- TrackingDate = trackingdate,
- OptionValue = Helpers.GetText(attributeNode, AttributeNode_Value),
- AttributeKey = Helpers.GetText(attributeNode, AttributeNode_Key),
- UpdateDate = Convert.ToDateTime(Helpers.GetText(attributeNode, AttributeNode_ModifiedDate)),
- SourceId = _daInsightsDw.ValidateAndGetSource(Helpers.GetText(attributeNode, AttributeNode_Source))
- };
- // Check if valid Profile Attribute
- if (!_daMetadata.IsValidClientProfileAttribute(premiseAttribute.AttributeKey))
- {
- throw new ApplicationException("Invalid Premise Attribute Key : " + premiseAttribute.AttributeKey);
- }
- var strDateTime = Helpers.GetText(attributeNode, AttributeNode_CreatedDate);
- if (strDateTime.Length == 0)
- {
- premiseAttribute.CreateDate = null;
- }
- else
- {
- premiseAttribute.CreateDate = Convert.ToDateTime(strDateTime);
- }
- premiseAttributeTable.Add(premiseAttribute);
- }
- }
- }
- }
- }
- }