SrcML.NET /Tools/ABB.SrcML.Tools.DataTester/Program.cs

Language C# Lines 256
MD5 Hash ab610d6d022b83f16298b14744a0502e
Repository https://github.com/nkcsgexi/SrcML.NET.git View Raw File
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
using ABB.SrcML.Data;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace ABB.SrcML.Tools.DataTester {
    class Program {
        static void Main(string[] args) {

            var projects = ReadMapping(@"C:\Workspace\source-srcmldata-mapping.txt");
            foreach(var project in projects) {
                GenerateData(project.Key, project.Value, @"c:\Workspace\SrcMLData");
            }
        }

        private static Dictionary<string, string> ReadMapping(string mappingFilePath) {
            var pairs = from line in File.ReadAllLines(mappingFilePath)
                        let parts = line.Split('|')
                        where parts.Length == 2
                        select new { Key = parts[0], Value = parts[1] };
            var mapping = new Dictionary<string, string>(pairs.Count());
            foreach(var pair in pairs) {
                mapping.Add(pair.Key, pair.Value);
            }
            return mapping;
        }

        private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) {
            Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() {
                { Language.CPlusPlus, new CPlusPlusCodeParser() },
                { Language.Java, new JavaCodeParser() },
                { Language.CSharp, new CSharpCodeParser() }
            };

            string fileLogPath = Path.Combine(dataPath, "parse.log");
            string callLogPath = Path.Combine(dataPath, "methodcalls.log");
            string csvPath = Path.Combine(csvDirectory, "timing.csv");
            string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
            
            if(!Directory.Exists(sourcePath)) {
                Console.Error.WriteLine("{0} does not exist", sourcePath);
                return;
            }

            if(File.Exists(callLogPath)) {
                File.Delete(callLogPath);
            }
            if(File.Exists(fileLogPath)) {
                File.Delete(fileLogPath);
            }

            var archive = new SrcMLArchive(dataPath);
            archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
            archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
            archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
            archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;

            AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);

            ManualResetEvent mre = new ManualResetEvent(false);
            Stopwatch timer = new Stopwatch();
            bool startupCompleted = false;

            monitor.IsReadyChanged += (o, e) => {
                if(e.ReadyState) {
                    timer.Stop();
                    startupCompleted = true;
                    mre.Set();
                }
            };

            timer.Start();
            monitor.Startup();
            string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
            int spinner_index = -1;
            while(!startupCompleted) {
                spinner_index = (++spinner_index) % 3;
                Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
                startupCompleted = mre.WaitOne(5000);
            }
            timer.Stop();
            Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);

            Scope globalScope = null;
            timer.Reset();

            int numberOfFailures = 0;
            int numberOfSuccesses = 0;
            int numberOfFiles = 0;
            Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();

            
            if(!File.Exists(csvPath)) {
                File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
            }
            using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
                timer.Start();
                foreach(var unit in archive.FileUnits) {
                    var fileName = SrcMLElement.GetFileNameForUnit(unit);
                    var language = SrcMLElement.GetLanguageForUnit(unit);

                    try {
                        var scopeForUnit = CodeParser[language].ParseFileUnit(unit);

                        if(null == globalScope) {
                            globalScope = scopeForUnit;
                        } else {
                            globalScope = globalScope.Merge(scopeForUnit);
                        }
                        timer.Stop();
                        fileLog.WriteLine("Parsing {0} PASSED", fileName);
                        numberOfSuccesses++;
                    } catch(Exception e) {
                        timer.Stop();
                        fileLog.WriteLine("Parsing {0} FAILED", fileName);
                        fileLog.WriteLine(e.StackTrace);
                        var key = e.StackTrace.Split('\n')[0].Trim();
                        if(!errors.ContainsKey(key)) {
                            errors[key] = new List<string>();
                        }
                        errors[key].Add(fileName);

                        numberOfFailures++;
                    }
                    finally {
                        if(++numberOfFiles % 50 == 0) {
                            Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
                            csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
                        }
                        timer.Start();
                    }
                }
            }
            timer.Stop();
            Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);

            Console.WriteLine("\nSummary");
            Console.WriteLine("===================");

            Console.WriteLine("{0,10:N0} failures  ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
            Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
            Console.WriteLine("{0} to generate data", timer.Elapsed);
            Console.WriteLine("See parse log at {0}", fileLogPath);
            
            OutputCallGraphByType(globalScope, jsonPath);

            PrintScopeReport(globalScope, sourcePath, csvDirectory);
            PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
        }
        private static void PrintScopeReport(Scope globalScope, string sourcePath, string csvDirectory) {
            var csvPath = Path.Combine(csvDirectory, "scopes.csv");
            Console.WriteLine("\nScope Report");
            Console.WriteLine("===============");

            var allScopes = VariableScopeIterator.Visit(globalScope);
            int numScopes = allScopes.Count();
            int numNamedScopes = allScopes.OfType<NamedScope>().Count();
            int numNamespaces = allScopes.OfType<NamespaceDefinition>().Count();
            int numTypes = allScopes.OfType<TypeDefinition>().Count();
            int numMethods = allScopes.OfType<MethodDefinition>().Count();

            Console.WriteLine("{0,10:N0} scopes", numScopes);
            
            Console.WriteLine("{0,10:N0} named scopes", numNamedScopes);
            
            Console.WriteLine("{0,10:N0} namespaces", numNamespaces);
            Console.WriteLine("{0,10:N0} types", numTypes);
            Console.WriteLine("{0,10:N0} methods", numMethods);
            if(!File.Exists(csvPath)) {
                File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Scopes", "Named Scopes", "Namespaces", "Types", "Methods"), Environment.NewLine));
            }
            File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numScopes, numNamedScopes, numNamespaces, numTypes, numMethods), Environment.NewLine));
        }

        private static void PrintMethodCallReport(Scope globalScope, string sourcePath, string csvDirectory, string callLogPath) {
            var csvPath = Path.Combine(csvDirectory, "methodcalls.csv");
            Console.WriteLine("\nMethod Call Report");
            Console.WriteLine("===============");
            var methodCalls = from scope in VariableScopeIterator.Visit(globalScope)
                              from call in scope.MethodCalls
                              select call;

            int numMethodCalls = 0;
            int numMatchedMethodCalls = 0;
            Stopwatch sw = new Stopwatch();

            using(var callLog = new StreamWriter(callLogPath)) {
                foreach(var call in methodCalls) {
                    sw.Start();
                    var match = call.FindMatches().FirstOrDefault();
                    sw.Stop();
                    numMethodCalls++;
                    if(null != match) {
                        numMatchedMethodCalls++;
                        callLog.WriteLine("{0} ({1}:{2}) -> {3} ({4}:{5})", call.Name, call.Location.SourceFileName, call.Location.StartingLineNumber, match.Name, match.PrimaryLocation.SourceFileName, match.PrimaryLocation.StartingLineNumber);
                    }
                }
            }

            Console.WriteLine("{0,10:N0} method calls", numMethodCalls);
            Console.WriteLine("{0,10:N0} matched method calls ({1,8:P2})", numMatchedMethodCalls, ((float)numMatchedMethodCalls) / numMethodCalls);
            Console.WriteLine("{0,10:N0} matches / millisecond ({1,7:N0} ms elapsed)", ((float)numMethodCalls) / sw.ElapsedMilliseconds, sw.ElapsedMilliseconds);
            Console.WriteLine("See matched method calls in {0}", callLogPath);

            if(!File.Exists(csvPath)) {
                File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Method Calls", "Matched Method Calls", "Time (ms)"), Environment.NewLine));
            }
            File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numMethodCalls, numMatchedMethodCalls, sw.ElapsedMilliseconds), Environment.NewLine));
        }

        private static void OutputCallGraphByType(Scope globalScope, string jsonPath) {
            using(var writer = new JsonTextWriter(new StreamWriter(jsonPath))) {
                writer.WriteStartArray();
                foreach(var typeDefinition in globalScope.GetDescendantScopesAndSelf<TypeDefinition>()) {
                    writer.WriteStartObject();
                    writer.WritePropertyName("name");
                    writer.WriteValue(typeDefinition.GetFullName());
                    
                    
                    var calls = from scope in typeDefinition.GetDescendantScopesAndSelf()
                                from call in scope.MethodCalls
                                select call;
                    
                    writer.WritePropertyName("size");
                    writer.WriteValue(calls.Count());

                    // find the parent type of all the calls
                    var callMatches = from call in calls
                                      let match = call.FindMatches().FirstOrDefault()
                                      where match != null
                                      let parentOfMatch = match.GetFirstParent<TypeDefinition>()
                                      where parentOfMatch != null
                                      select parentOfMatch.GetFullName();
                    // output the calls property and array
                    writer.WritePropertyName("calls");
                    writer.WriteStartArray();
                    foreach (var call in callMatches)
	                {
                        writer.WriteValue(call);
	                }
                    writer.WriteEndArray();
                    writer.WriteEndObject();
                }
                writer.WriteEndArray();
            }
        }
    }
}
Back to Top