/Alpha/Source/Libraries/Tests/GSF.SortedTreeStore.Test/Engine/ArchiveReaderSequential_Test.cs
C# | 528 lines | 444 code | 72 blank | 12 comment | 30 complexity | 9f4aaa2cc94168b9361bf820034f8452 MD5 | raw file
Possible License(s): MPL-2.0-no-copyleft-exception, EPL-1.0
- using System;
- using System.Collections.Generic;
- using System.Linq;
- using System.Text;
- using System.Threading.Tasks;
- using GSF.IO.Unmanaged;
- using GSF.SortedTreeStore.Engine.Reader;
- using GSF.SortedTreeStore.Filters;
- using NUnit.Framework;
- using openHistorian;
- using GSF.SortedTreeStore.Storage;
- using openHistorian.Collections;
- using GSF.SortedTreeStore.Tree;
- using GSF.SortedTreeStore.Tree.TreeNodes;
- using openHistorian.Data.Query;
- using GSF.SortedTreeStore.Net.Compression;
-
- namespace GSF.SortedTreeStore.Engine
- {
- [TestFixture]
- public class ArchiveReaderSequential_Test
- {
- [Test]
- public void TestOneFile()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var master = CreateTable();
- var table1 = CreateTable();
- AddData(master, 100, 100, 100);
- AddData(table1, 100, 100, 100);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
-
- using (var masterRead = master.BeginRead())
- {
- var masterScan = masterRead.GetTreeScanner();
- masterScan.SeekToStart();
- var masterScanSequential = masterScan.TestSequential();
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var scanner = sequencer.Read().TestSequential();
-
- int count = 0;
- while (scanner.Read())
- {
- count++;
- if (!masterScanSequential.Read())
- throw new Exception();
-
- if (!scanner.CurrentKey.IsEqualTo(masterScan.CurrentKey))
- throw new Exception();
-
- if (!scanner.CurrentValue.IsEqualTo(masterScan.CurrentValue))
- throw new Exception();
- }
- if (masterScan.Read())
- throw new Exception();
-
- }
- list.Dispose();
- master.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void TestTwoFiles()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var master = CreateTable();
- var table1 = CreateTable();
- var table2 = CreateTable();
- AddData(master, 100, 100, 100);
- AddData(table1, 100, 100, 100);
- AddData(master, 101, 100, 100);
- AddData(table2, 101, 100, 100);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- editor.Add(table2, false);
- }
-
- using (var masterRead = master.BeginRead())
- {
- var masterScan = masterRead.GetTreeScanner();
- masterScan.SeekToStart();
- var masterScanSequential = masterScan.TestSequential();
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var scanner = sequencer.Read().TestSequential();
-
- while (scanner.Read())
- {
- if (!masterScanSequential.Read())
- throw new Exception();
-
- if (!scanner.CurrentKey.IsEqualTo(masterScan.CurrentKey))
- throw new Exception();
-
- if (!scanner.CurrentValue.IsEqualTo(masterScan.CurrentValue))
- throw new Exception();
- }
- if (masterScan.Read())
- throw new Exception();
- }
- master.Dispose();
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void TestTwoIdenticalFiles()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var master = CreateTable();
- var table1 = CreateTable();
- var table2 = CreateTable();
- AddData(master, 100, 100, 100);
- AddData(table1, 100, 100, 100);
- AddData(table2, 100, 100, 100);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- editor.Add(table2, false);
- }
-
- using (var masterRead = master.BeginRead())
- {
- var masterScan = masterRead.GetTreeScanner();
- masterScan.SeekToStart();
- var masterScanSequential = masterScan.TestSequential();
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var scanner = sequencer.Read().TestSequential();
-
- while (scanner.Read())
- {
- if (!masterScanSequential.Read())
- throw new Exception();
-
- if (!scanner.CurrentKey.IsEqualTo(masterScan.CurrentKey))
- throw new Exception();
-
- if (!scanner.CurrentValue.IsEqualTo(masterScan.CurrentValue))
- throw new Exception();
- }
- if (masterScan.Read())
- throw new Exception();
- }
- list.Dispose();
- master.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
-
- SortedTreeTable<HistorianKey, HistorianValue> CreateTable()
- {
- var file = SortedTreeFile.CreateInMemory();
- var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(SortedTree.FixedSizeNode);
- return table;
- }
-
- void AddData(SortedTreeTable<HistorianKey, HistorianValue> table, ulong start, ulong step, ulong count)
- {
- using (var edit = table.BeginEdit())
- {
- var key = new HistorianKey();
- var value = new HistorianValue();
-
- for (ulong v = start; v < start + step * count; v += step)
- {
- key.SetMin();
- key.PointID = v;
- edit.AddPoint(key, value);
- }
- edit.Commit();
- }
- }
- void AddData(SortedTreeTable<HistorianKey, HistorianValue> table, DateTime startTime, TimeSpan stepTime, int countTime, ulong startPoint, ulong stepPoint, ulong countPoint)
- {
- using (var edit = table.BeginEdit())
- {
- var key = new HistorianKey();
- var value = new HistorianValue();
- key.SetMin();
- var stepTimeTicks = (ulong)stepTime.Ticks;
- var stopTime = (ulong)(startTime.Ticks + countTime * stepTime.Ticks);
- for (ulong t = (ulong)startTime.Ticks; t < stopTime; t += stepTimeTicks)
- {
- for (ulong v = startPoint; v < startPoint + stepPoint * countPoint; v += stepPoint)
- {
- key.Timestamp = t;
- key.PointID = v;
- edit.AddPoint(key, value);
- }
- }
- edit.Commit();
- }
- }
- void AddDataTerminal(SortedTreeTable<HistorianKey, HistorianValue> table, ulong pointID, DateTime startTime, TimeSpan stepTime, ulong startValue, ulong stepValue, int count)
- {
- using (var edit = table.BeginEdit())
- {
- var key = new HistorianKey();
- var value = new HistorianValue();
- key.SetMin();
- var t = (ulong)startTime.Ticks;
- var v = startValue;
-
- while (count > 0)
- {
- count--;
- key.Timestamp = t;
- key.PointID = pointID;
- value.Value1 = v;
-
- edit.AddPoint(key, value);
- t += (ulong)stepTime.Ticks;
- v += stepValue;
- }
-
- edit.Commit();
- }
- }
-
- [Test]
- public void BenchmarkRawFile()
- {
- MemoryPoolTest.TestMemoryLeak();
- const int Max = 1000000;
-
- var master = CreateTable();
- AddData(master, 100, 100, Max);
-
-
- DebugStopwatch sw = new DebugStopwatch();
- using (var masterRead = master.BeginRead())
- {
- double sec = sw.TimeEvent(() =>
- {
- var scanner = masterRead.GetTreeScanner();
- scanner.SeekToStart();
- while (scanner.Read())
- {
- }
- });
- Console.WriteLine(Max / sec / 1000000);
-
- }
- master.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void BenchmarkOneFile()
- {
- MemoryPoolTest.TestMemoryLeak();
- const int Max = 1000000;
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var table1 = CreateTable();
- AddData(table1, 100, 100, Max);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
-
- DebugStopwatch sw = new DebugStopwatch();
-
- double sec = sw.TimeEvent(() =>
- {
- var scanner = sequencer.Read();
- while (scanner.Read())
- {
- }
- });
- Console.WriteLine(Max / sec / 1000000);
- table1.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void BenchmarkTwoFiles()
- {
- MemoryPoolTest.TestMemoryLeak();
- const int Max = 1000000;
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var table1 = CreateTable();
- var table2 = CreateTable();
- AddData(table1, 100, 100, Max / 2);
- AddData(table2, 101, 100, Max / 2);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- editor.Add(table2, false);
- }
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
-
- DebugStopwatch sw = new DebugStopwatch();
-
- double sec = sw.TimeEvent(() =>
- {
- var scanner = sequencer.Read();
- while (scanner.Read())
- {
- }
- });
- Console.WriteLine(Max / sec / 1000000);
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void BenchmarkThreeFiles()
- {
- MemoryPoolTest.TestMemoryLeak();
- const int Max = 1000000;
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
-
- var table1 = CreateTable();
- var table2 = CreateTable();
- var table3 = CreateTable();
- AddData(table1, 100, 100, Max / 3);
- AddData(table2, 101, 100, Max / 3);
- AddData(table3, 102, 100, Max / 3);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- editor.Add(table2, false);
- editor.Add(table3, false);
- }
-
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
-
- DebugStopwatch sw = new DebugStopwatch();
-
- double sec = sw.TimeEvent(() =>
- {
- var scanner = sequencer.Read();
- while (scanner.Read())
- {
- }
- });
- Console.WriteLine(Max / sec / 1000000);
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void BenchmarkRealisticSamples()
- {
-
- MemoryPoolTest.TestMemoryLeak();
- const int Max = 1000000;
- const int FileCount = 1000;
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
- DateTime start = DateTime.Now.Date;
-
- for (int x = 0; x < FileCount; x++)
- {
- var table1 = CreateTable();
- AddData(table1, start.AddMinutes(2 * x), new TimeSpan(TimeSpan.TicksPerSecond), 60, 100, 1, Max / 60 / FileCount);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
- }
-
- var filter = TimestampFilter.CreateFromIntervalData<HistorianKey>(start, start.AddMinutes(2 * FileCount), new TimeSpan(TimeSpan.TicksPerSecond * 2), new TimeSpan(TimeSpan.TicksPerMillisecond));
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
-
- DebugStopwatch sw = new DebugStopwatch();
- int xi = 0;
- double sec = sw.TimeEvent(() =>
- {
- var scanner = sequencer.Read(filter);
- while (scanner.Read())
- {
- xi++;
- }
- });
- Console.WriteLine(Max / sec / 1000000);
- //TreeKeyMethodsBase<HistorianKey>.WriteToConsole();
- //TreeValueMethodsBase<HistorianValue>.WriteToConsole();
-
- //Console.WriteLine("KeyMethodsBase calls");
- //for (int x = 0; x < 23; x++)
- //{
- // Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString());
- //}
- //Console.WriteLine("ValueMethodsBase calls");
- //for (int x = 0; x < 5; x++)
- //{
- // Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString());
- //}
- list.Dispose();
-
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void ConsoleTest1()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
- DateTime start = DateTime.Now.Date;
-
- for (int x = 0; x < 3; x++)
- {
- var table1 = CreateTable();
- AddDataTerminal(table1, (ulong)x, start, new TimeSpan(TimeSpan.TicksPerSecond), (ulong)(1000 * x), 1, 60 * 60);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
- }
-
- var filter = TimestampFilter.CreateFromIntervalData<HistorianKey>(start, start.AddMinutes(10), new TimeSpan(TimeSpan.TicksPerSecond * 1), new TimeSpan(TimeSpan.TicksPerMillisecond));
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var frames = sequencer.Read(filter).GetFrames();
- WriteToConsole(frames);
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void ConsoleTest2()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
- DateTime start = DateTime.Now.Date;
-
- for (int x = 0; x < 3; x++)
- {
- var table1 = CreateTable();
- AddDataTerminal(table1, (ulong)x, start, new TimeSpan(TimeSpan.TicksPerSecond), (ulong)(1000 * x), 1, 60 * 60);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
- }
-
- var filter = TimestampFilter.CreateFromIntervalData<HistorianKey>(start.AddMinutes(-100), start.AddMinutes(10), new TimeSpan(TimeSpan.TicksPerSecond * 60), new TimeSpan(TimeSpan.TicksPerSecond));
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var frames = sequencer.Read(filter).GetFrames();
- WriteToConsole(frames);
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- [Test]
- public void ConsoleTest3()
- {
- MemoryPoolTest.TestMemoryLeak();
- ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>();
- DateTime start = DateTime.Now.Date;
-
- for (int x = 0; x < 3; x++)
- {
- var table1 = CreateTable();
- AddDataTerminal(table1, (ulong)x, start, new TimeSpan(TimeSpan.TicksPerSecond), (ulong)(1000 * x), 1, 60 * 60);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
- }
- for (int x = 0; x < 3; x++)
- {
- var table1 = CreateTable();
- AddDataTerminal(table1, (ulong)x, start, new TimeSpan(TimeSpan.TicksPerSecond), (ulong)(1000 * x), 1, 60 * 60);
- using (var editor = list.AcquireEditLock())
- {
- editor.Add(table1, false);
- }
- }
-
- var filter = TimestampFilter.CreateFromIntervalData<HistorianKey>(start, start.AddMinutes(10), new TimeSpan(TimeSpan.TicksPerSecond * 60), new TimeSpan(TimeSpan.TicksPerSecond));
- var sequencer = new SortedTreeEngineReaderSequential<HistorianKey, HistorianValue>(list);
- var frames = sequencer.Read(filter).GetFrames();
- WriteToConsole(frames);
- list.Dispose();
- MemoryPoolTest.TestMemoryLeak();
- }
-
- void WriteToConsole(SortedList<DateTime, FrameData> frames)
- {
- StringBuilder sb = new StringBuilder();
-
- ulong?[] data = new ulong?[10];
-
- foreach (var frame in frames)
- {
- Array.Clear(data, 0, data.Length);
- foreach (var sample in frame.Value.Points)
- {
- data[sample.Key] = sample.Value.Value1;
- }
-
- sb.Append(frame.Key.ToString());
- sb.Append('\t');
- foreach (var value in data)
- {
- if (value.HasValue)
- {
- sb.Append(value.Value);
- }
- sb.Append('\t');
- }
-
- Console.WriteLine(sb.ToString());
- sb.Clear();
- }
- }
-
-
- }
- }