/Rhino.Etl.Core/EtlProcess.cs

http://github.com/ayende/rhino-etl · C# · 180 lines · 107 code · 20 blank · 53 comment · 2 complexity · 75d357afd6ca53b6e565de17caf7e22d MD5 · raw file

  1. using System.Configuration;
  2. using Rhino.Etl.Core.Infrastructure;
  3. namespace Rhino.Etl.Core
  4. {
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Data;
  8. using Operations;
  9. using Pipelines;
  10. /// <summary>
  11. /// A single etl process
  12. /// </summary>
  13. public abstract class EtlProcess : EtlProcessBase<EtlProcess>, IDisposable
  14. {
  15. private IPipelineExecuter pipelineExecuter = new ThreadPoolPipelineExecuter();
  16. /// <summary>
  17. /// Gets the pipeline executer.
  18. /// </summary>
  19. /// <value>The pipeline executer.</value>
  20. public IPipelineExecuter PipelineExecuter
  21. {
  22. get { return pipelineExecuter; }
  23. set
  24. {
  25. Info("Setting PipelineExecutor to {0}", value.GetType().ToString());
  26. pipelineExecuter = value;
  27. }
  28. }
  29. /// <summary>
  30. /// Gets a new partial process that we can work with
  31. /// </summary>
  32. protected static PartialProcessOperation Partial
  33. {
  34. get
  35. {
  36. PartialProcessOperation operation = new PartialProcessOperation();
  37. return operation;
  38. }
  39. }
  40. #region IDisposable Members
  41. ///<summary>
  42. ///Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
  43. ///</summary>
  44. ///<filterpriority>2</filterpriority>
  45. public void Dispose()
  46. {
  47. foreach (IOperation operation in operations)
  48. {
  49. operation.Dispose();
  50. }
  51. }
  52. #endregion
  53. /// <summary>
  54. /// Initializes this instance.
  55. /// </summary>
  56. protected abstract void Initialize();
  57. /// <summary>
  58. /// Executes this process
  59. /// </summary>
  60. public void Execute()
  61. {
  62. Initialize();
  63. MergeLastOperationsToOperations();
  64. RegisterToOperationsEvents();
  65. Trace("Starting to execute {0}", Name);
  66. PipelineExecuter.Execute(Name, operations, TranslateRows);
  67. PostProcessing();
  68. }
  69. /// <summary>
  70. /// Translate the rows from one representation to another
  71. /// </summary>
  72. public virtual IEnumerable<Row> TranslateRows(IEnumerable<Row> rows)
  73. {
  74. return rows;
  75. }
  76. private void RegisterToOperationsEvents()
  77. {
  78. foreach (IOperation operation in operations)
  79. {
  80. operation.OnRowProcessed += OnRowProcessed;
  81. operation.OnFinishedProcessing += OnFinishedProcessing;
  82. }
  83. }
  84. /// <summary>
  85. /// Called when this process has finished processing.
  86. /// </summary>
  87. /// <param name="op">The op.</param>
  88. protected virtual void OnFinishedProcessing(IOperation op)
  89. {
  90. Trace("Finished {0}: {1}", op.Name, op.Statistics);
  91. }
  92. /// <summary>
  93. /// Allow derived class to deal with custom logic after all the internal steps have been executed
  94. /// </summary>
  95. protected virtual void PostProcessing()
  96. {
  97. }
  98. /// <summary>
  99. /// Called when a row is processed.
  100. /// </summary>
  101. /// <param name="op">The operation.</param>
  102. /// <param name="dictionary">The dictionary.</param>
  103. protected virtual void OnRowProcessed(IOperation op, Row dictionary)
  104. {
  105. if (op.Statistics.OutputtedRows % 1000 == 0)
  106. Info("Processed {0} rows in {1}", op.Statistics.OutputtedRows, op.Name);
  107. else
  108. Debug("Processed {0} rows in {1}", op.Statistics.OutputtedRows, op.Name);
  109. }
  110. /// <summary>
  111. /// Executes the command and return a scalar
  112. /// </summary>
  113. /// <typeparam name="T"></typeparam>
  114. /// <param name="connectionName">Name of the connection.</param>
  115. /// <param name="commandText">The command text.</param>
  116. /// <returns></returns>
  117. protected static T ExecuteScalar<T>(string connectionName, string commandText)
  118. {
  119. return ExecuteScalar<T>(ConfigurationManager.ConnectionStrings[connectionName], commandText);
  120. }
  121. /// <summary>
  122. /// Executes the command and return a scalar
  123. /// </summary>
  124. /// <typeparam name="T"></typeparam>
  125. /// <param name="connectionStringSettings">The connection string settings node to use</param>
  126. /// <param name="commandText">The command text.</param>
  127. /// <returns></returns>
  128. protected static T ExecuteScalar<T>(ConnectionStringSettings connectionStringSettings, string commandText)
  129. {
  130. return Use.Transaction<T>(connectionStringSettings, delegate(IDbCommand cmd)
  131. {
  132. cmd.CommandText = commandText;
  133. object scalar = cmd.ExecuteScalar();
  134. return (T)(scalar ?? default(T));
  135. });
  136. }
  137. /// <summary>
  138. /// Gets all errors that occured during the execution of this process
  139. /// </summary>
  140. /// <returns></returns>
  141. public IEnumerable<Exception> GetAllErrors()
  142. {
  143. foreach (Exception error in Errors)
  144. {
  145. yield return error;
  146. }
  147. foreach (Exception error in pipelineExecuter.GetAllErrors())
  148. {
  149. yield return error;
  150. }
  151. foreach (IOperation operation in operations)
  152. {
  153. foreach (Exception exception in operation.GetAllErrors())
  154. {
  155. yield return exception;
  156. }
  157. }
  158. }
  159. }
  160. }