PageRenderTime 47ms CodeModel.GetById 21ms RepoModel.GetById 1ms app.codeStats 0ms

/Tools/ABB.SrcML.Tools.DataTester/Program.cs

https://github.com/nkcsgexi/SrcML.NET
C# | 255 lines | 216 code | 37 blank | 2 comment | 18 complexity | ab610d6d022b83f16298b14744a0502e MD5 | raw file
  1. using ABB.SrcML.Data;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Diagnostics;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Text;
  8. using System.Threading;
  9. using System.Threading.Tasks;
  10. using Newtonsoft.Json;
  11. using Newtonsoft.Json.Linq;
  12. namespace ABB.SrcML.Tools.DataTester {
  13. class Program {
  14. static void Main(string[] args) {
  15. var projects = ReadMapping(@"C:\Workspace\source-srcmldata-mapping.txt");
  16. foreach(var project in projects) {
  17. GenerateData(project.Key, project.Value, @"c:\Workspace\SrcMLData");
  18. }
  19. }
  20. private static Dictionary<string, string> ReadMapping(string mappingFilePath) {
  21. var pairs = from line in File.ReadAllLines(mappingFilePath)
  22. let parts = line.Split('|')
  23. where parts.Length == 2
  24. select new { Key = parts[0], Value = parts[1] };
  25. var mapping = new Dictionary<string, string>(pairs.Count());
  26. foreach(var pair in pairs) {
  27. mapping.Add(pair.Key, pair.Value);
  28. }
  29. return mapping;
  30. }
  31. private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) {
  32. Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() {
  33. { Language.CPlusPlus, new CPlusPlusCodeParser() },
  34. { Language.Java, new JavaCodeParser() },
  35. { Language.CSharp, new CSharpCodeParser() }
  36. };
  37. string fileLogPath = Path.Combine(dataPath, "parse.log");
  38. string callLogPath = Path.Combine(dataPath, "methodcalls.log");
  39. string csvPath = Path.Combine(csvDirectory, "timing.csv");
  40. string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
  41. if(!Directory.Exists(sourcePath)) {
  42. Console.Error.WriteLine("{0} does not exist", sourcePath);
  43. return;
  44. }
  45. if(File.Exists(callLogPath)) {
  46. File.Delete(callLogPath);
  47. }
  48. if(File.Exists(fileLogPath)) {
  49. File.Delete(fileLogPath);
  50. }
  51. var archive = new SrcMLArchive(dataPath);
  52. archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
  53. archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
  54. archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
  55. archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
  56. AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
  57. ManualResetEvent mre = new ManualResetEvent(false);
  58. Stopwatch timer = new Stopwatch();
  59. bool startupCompleted = false;
  60. monitor.IsReadyChanged += (o, e) => {
  61. if(e.ReadyState) {
  62. timer.Stop();
  63. startupCompleted = true;
  64. mre.Set();
  65. }
  66. };
  67. timer.Start();
  68. monitor.Startup();
  69. string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
  70. int spinner_index = -1;
  71. while(!startupCompleted) {
  72. spinner_index = (++spinner_index) % 3;
  73. Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
  74. startupCompleted = mre.WaitOne(5000);
  75. }
  76. timer.Stop();
  77. Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
  78. Scope globalScope = null;
  79. timer.Reset();
  80. int numberOfFailures = 0;
  81. int numberOfSuccesses = 0;
  82. int numberOfFiles = 0;
  83. Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
  84. if(!File.Exists(csvPath)) {
  85. File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
  86. }
  87. using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
  88. timer.Start();
  89. foreach(var unit in archive.FileUnits) {
  90. var fileName = SrcMLElement.GetFileNameForUnit(unit);
  91. var language = SrcMLElement.GetLanguageForUnit(unit);
  92. try {
  93. var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
  94. if(null == globalScope) {
  95. globalScope = scopeForUnit;
  96. } else {
  97. globalScope = globalScope.Merge(scopeForUnit);
  98. }
  99. timer.Stop();
  100. fileLog.WriteLine("Parsing {0} PASSED", fileName);
  101. numberOfSuccesses++;
  102. } catch(Exception e) {
  103. timer.Stop();
  104. fileLog.WriteLine("Parsing {0} FAILED", fileName);
  105. fileLog.WriteLine(e.StackTrace);
  106. var key = e.StackTrace.Split('\n')[0].Trim();
  107. if(!errors.ContainsKey(key)) {
  108. errors[key] = new List<string>();
  109. }
  110. errors[key].Add(fileName);
  111. numberOfFailures++;
  112. }
  113. finally {
  114. if(++numberOfFiles % 50 == 0) {
  115. Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
  116. csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
  117. }
  118. timer.Start();
  119. }
  120. }
  121. }
  122. timer.Stop();
  123. Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
  124. Console.WriteLine("\nSummary");
  125. Console.WriteLine("===================");
  126. Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
  127. Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
  128. Console.WriteLine("{0} to generate data", timer.Elapsed);
  129. Console.WriteLine("See parse log at {0}", fileLogPath);
  130. OutputCallGraphByType(globalScope, jsonPath);
  131. PrintScopeReport(globalScope, sourcePath, csvDirectory);
  132. PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
  133. }
  134. private static void PrintScopeReport(Scope globalScope, string sourcePath, string csvDirectory) {
  135. var csvPath = Path.Combine(csvDirectory, "scopes.csv");
  136. Console.WriteLine("\nScope Report");
  137. Console.WriteLine("===============");
  138. var allScopes = VariableScopeIterator.Visit(globalScope);
  139. int numScopes = allScopes.Count();
  140. int numNamedScopes = allScopes.OfType<NamedScope>().Count();
  141. int numNamespaces = allScopes.OfType<NamespaceDefinition>().Count();
  142. int numTypes = allScopes.OfType<TypeDefinition>().Count();
  143. int numMethods = allScopes.OfType<MethodDefinition>().Count();
  144. Console.WriteLine("{0,10:N0} scopes", numScopes);
  145. Console.WriteLine("{0,10:N0} named scopes", numNamedScopes);
  146. Console.WriteLine("{0,10:N0} namespaces", numNamespaces);
  147. Console.WriteLine("{0,10:N0} types", numTypes);
  148. Console.WriteLine("{0,10:N0} methods", numMethods);
  149. if(!File.Exists(csvPath)) {
  150. File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Scopes", "Named Scopes", "Namespaces", "Types", "Methods"), Environment.NewLine));
  151. }
  152. File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numScopes, numNamedScopes, numNamespaces, numTypes, numMethods), Environment.NewLine));
  153. }
  154. private static void PrintMethodCallReport(Scope globalScope, string sourcePath, string csvDirectory, string callLogPath) {
  155. var csvPath = Path.Combine(csvDirectory, "methodcalls.csv");
  156. Console.WriteLine("\nMethod Call Report");
  157. Console.WriteLine("===============");
  158. var methodCalls = from scope in VariableScopeIterator.Visit(globalScope)
  159. from call in scope.MethodCalls
  160. select call;
  161. int numMethodCalls = 0;
  162. int numMatchedMethodCalls = 0;
  163. Stopwatch sw = new Stopwatch();
  164. using(var callLog = new StreamWriter(callLogPath)) {
  165. foreach(var call in methodCalls) {
  166. sw.Start();
  167. var match = call.FindMatches().FirstOrDefault();
  168. sw.Stop();
  169. numMethodCalls++;
  170. if(null != match) {
  171. numMatchedMethodCalls++;
  172. callLog.WriteLine("{0} ({1}:{2}) -> {3} ({4}:{5})", call.Name, call.Location.SourceFileName, call.Location.StartingLineNumber, match.Name, match.PrimaryLocation.SourceFileName, match.PrimaryLocation.StartingLineNumber);
  173. }
  174. }
  175. }
  176. Console.WriteLine("{0,10:N0} method calls", numMethodCalls);
  177. Console.WriteLine("{0,10:N0} matched method calls ({1,8:P2})", numMatchedMethodCalls, ((float)numMatchedMethodCalls) / numMethodCalls);
  178. Console.WriteLine("{0,10:N0} matches / millisecond ({1,7:N0} ms elapsed)", ((float)numMethodCalls) / sw.ElapsedMilliseconds, sw.ElapsedMilliseconds);
  179. Console.WriteLine("See matched method calls in {0}", callLogPath);
  180. if(!File.Exists(csvPath)) {
  181. File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Method Calls", "Matched Method Calls", "Time (ms)"), Environment.NewLine));
  182. }
  183. File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numMethodCalls, numMatchedMethodCalls, sw.ElapsedMilliseconds), Environment.NewLine));
  184. }
  185. private static void OutputCallGraphByType(Scope globalScope, string jsonPath) {
  186. using(var writer = new JsonTextWriter(new StreamWriter(jsonPath))) {
  187. writer.WriteStartArray();
  188. foreach(var typeDefinition in globalScope.GetDescendantScopesAndSelf<TypeDefinition>()) {
  189. writer.WriteStartObject();
  190. writer.WritePropertyName("name");
  191. writer.WriteValue(typeDefinition.GetFullName());
  192. var calls = from scope in typeDefinition.GetDescendantScopesAndSelf()
  193. from call in scope.MethodCalls
  194. select call;
  195. writer.WritePropertyName("size");
  196. writer.WriteValue(calls.Count());
  197. // find the parent type of all the calls
  198. var callMatches = from call in calls
  199. let match = call.FindMatches().FirstOrDefault()
  200. where match != null
  201. let parentOfMatch = match.GetFirstParent<TypeDefinition>()
  202. where parentOfMatch != null
  203. select parentOfMatch.GetFullName();
  204. // output the calls property and array
  205. writer.WritePropertyName("calls");
  206. writer.WriteStartArray();
  207. foreach (var call in callMatches)
  208. {
  209. writer.WriteValue(call);
  210. }
  211. writer.WriteEndArray();
  212. writer.WriteEndObject();
  213. }
  214. writer.WriteEndArray();
  215. }
  216. }
  217. }
  218. }