PageRenderTime 28ms CodeModel.GetById 10ms app.highlight 12ms RepoModel.GetById 1ms app.codeStats 1ms

/Tools/ABB.SrcML.Tools.DataTester/Program.cs

https://github.com/nkcsgexi/SrcML.NET
C# | 255 lines | 216 code | 37 blank | 2 comment | 18 complexity | ab610d6d022b83f16298b14744a0502e MD5 | raw file
  1using ABB.SrcML.Data;
  2using System;
  3using System.Collections.Generic;
  4using System.Diagnostics;
  5using System.IO;
  6using System.Linq;
  7using System.Text;
  8using System.Threading;
  9using System.Threading.Tasks;
 10using Newtonsoft.Json;
 11using Newtonsoft.Json.Linq;
 12
 13namespace ABB.SrcML.Tools.DataTester {
 14    class Program {
 15        static void Main(string[] args) {
 16
 17            var projects = ReadMapping(@"C:\Workspace\source-srcmldata-mapping.txt");
 18            foreach(var project in projects) {
 19                GenerateData(project.Key, project.Value, @"c:\Workspace\SrcMLData");
 20            }
 21        }
 22
 23        private static Dictionary<string, string> ReadMapping(string mappingFilePath) {
 24            var pairs = from line in File.ReadAllLines(mappingFilePath)
 25                        let parts = line.Split('|')
 26                        where parts.Length == 2
 27                        select new { Key = parts[0], Value = parts[1] };
 28            var mapping = new Dictionary<string, string>(pairs.Count());
 29            foreach(var pair in pairs) {
 30                mapping.Add(pair.Key, pair.Value);
 31            }
 32            return mapping;
 33        }
 34
 35        private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) {
 36            Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() {
 37                { Language.CPlusPlus, new CPlusPlusCodeParser() },
 38                { Language.Java, new JavaCodeParser() },
 39                { Language.CSharp, new CSharpCodeParser() }
 40            };
 41
 42            string fileLogPath = Path.Combine(dataPath, "parse.log");
 43            string callLogPath = Path.Combine(dataPath, "methodcalls.log");
 44            string csvPath = Path.Combine(csvDirectory, "timing.csv");
 45            string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
 46            
 47            if(!Directory.Exists(sourcePath)) {
 48                Console.Error.WriteLine("{0} does not exist", sourcePath);
 49                return;
 50            }
 51
 52            if(File.Exists(callLogPath)) {
 53                File.Delete(callLogPath);
 54            }
 55            if(File.Exists(fileLogPath)) {
 56                File.Delete(fileLogPath);
 57            }
 58
 59            var archive = new SrcMLArchive(dataPath);
 60            archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
 61            archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
 62            archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
 63            archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
 64
 65            AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
 66
 67            ManualResetEvent mre = new ManualResetEvent(false);
 68            Stopwatch timer = new Stopwatch();
 69            bool startupCompleted = false;
 70
 71            monitor.IsReadyChanged += (o, e) => {
 72                if(e.ReadyState) {
 73                    timer.Stop();
 74                    startupCompleted = true;
 75                    mre.Set();
 76                }
 77            };
 78
 79            timer.Start();
 80            monitor.Startup();
 81            string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
 82            int spinner_index = -1;
 83            while(!startupCompleted) {
 84                spinner_index = (++spinner_index) % 3;
 85                Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
 86                startupCompleted = mre.WaitOne(5000);
 87            }
 88            timer.Stop();
 89            Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
 90
 91            Scope globalScope = null;
 92            timer.Reset();
 93
 94            int numberOfFailures = 0;
 95            int numberOfSuccesses = 0;
 96            int numberOfFiles = 0;
 97            Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
 98
 99            
100            if(!File.Exists(csvPath)) {
101                File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
102            }
103            using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
104                timer.Start();
105                foreach(var unit in archive.FileUnits) {
106                    var fileName = SrcMLElement.GetFileNameForUnit(unit);
107                    var language = SrcMLElement.GetLanguageForUnit(unit);
108
109                    try {
110                        var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
111
112                        if(null == globalScope) {
113                            globalScope = scopeForUnit;
114                        } else {
115                            globalScope = globalScope.Merge(scopeForUnit);
116                        }
117                        timer.Stop();
118                        fileLog.WriteLine("Parsing {0} PASSED", fileName);
119                        numberOfSuccesses++;
120                    } catch(Exception e) {
121                        timer.Stop();
122                        fileLog.WriteLine("Parsing {0} FAILED", fileName);
123                        fileLog.WriteLine(e.StackTrace);
124                        var key = e.StackTrace.Split('\n')[0].Trim();
125                        if(!errors.ContainsKey(key)) {
126                            errors[key] = new List<string>();
127                        }
128                        errors[key].Add(fileName);
129
130                        numberOfFailures++;
131                    }
132                    finally {
133                        if(++numberOfFiles % 50 == 0) {
134                            Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
135                            csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
136                        }
137                        timer.Start();
138                    }
139                }
140            }
141            timer.Stop();
142            Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
143
144            Console.WriteLine("\nSummary");
145            Console.WriteLine("===================");
146
147            Console.WriteLine("{0,10:N0} failures  ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
148            Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
149            Console.WriteLine("{0} to generate data", timer.Elapsed);
150            Console.WriteLine("See parse log at {0}", fileLogPath);
151            
152            OutputCallGraphByType(globalScope, jsonPath);
153
154            PrintScopeReport(globalScope, sourcePath, csvDirectory);
155            PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
156        }
157        private static void PrintScopeReport(Scope globalScope, string sourcePath, string csvDirectory) {
158            var csvPath = Path.Combine(csvDirectory, "scopes.csv");
159            Console.WriteLine("\nScope Report");
160            Console.WriteLine("===============");
161
162            var allScopes = VariableScopeIterator.Visit(globalScope);
163            int numScopes = allScopes.Count();
164            int numNamedScopes = allScopes.OfType<NamedScope>().Count();
165            int numNamespaces = allScopes.OfType<NamespaceDefinition>().Count();
166            int numTypes = allScopes.OfType<TypeDefinition>().Count();
167            int numMethods = allScopes.OfType<MethodDefinition>().Count();
168
169            Console.WriteLine("{0,10:N0} scopes", numScopes);
170            
171            Console.WriteLine("{0,10:N0} named scopes", numNamedScopes);
172            
173            Console.WriteLine("{0,10:N0} namespaces", numNamespaces);
174            Console.WriteLine("{0,10:N0} types", numTypes);
175            Console.WriteLine("{0,10:N0} methods", numMethods);
176            if(!File.Exists(csvPath)) {
177                File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Scopes", "Named Scopes", "Namespaces", "Types", "Methods"), Environment.NewLine));
178            }
179            File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numScopes, numNamedScopes, numNamespaces, numTypes, numMethods), Environment.NewLine));
180        }
181
182        private static void PrintMethodCallReport(Scope globalScope, string sourcePath, string csvDirectory, string callLogPath) {
183            var csvPath = Path.Combine(csvDirectory, "methodcalls.csv");
184            Console.WriteLine("\nMethod Call Report");
185            Console.WriteLine("===============");
186            var methodCalls = from scope in VariableScopeIterator.Visit(globalScope)
187                              from call in scope.MethodCalls
188                              select call;
189
190            int numMethodCalls = 0;
191            int numMatchedMethodCalls = 0;
192            Stopwatch sw = new Stopwatch();
193
194            using(var callLog = new StreamWriter(callLogPath)) {
195                foreach(var call in methodCalls) {
196                    sw.Start();
197                    var match = call.FindMatches().FirstOrDefault();
198                    sw.Stop();
199                    numMethodCalls++;
200                    if(null != match) {
201                        numMatchedMethodCalls++;
202                        callLog.WriteLine("{0} ({1}:{2}) -> {3} ({4}:{5})", call.Name, call.Location.SourceFileName, call.Location.StartingLineNumber, match.Name, match.PrimaryLocation.SourceFileName, match.PrimaryLocation.StartingLineNumber);
203                    }
204                }
205            }
206
207            Console.WriteLine("{0,10:N0} method calls", numMethodCalls);
208            Console.WriteLine("{0,10:N0} matched method calls ({1,8:P2})", numMatchedMethodCalls, ((float)numMatchedMethodCalls) / numMethodCalls);
209            Console.WriteLine("{0,10:N0} matches / millisecond ({1,7:N0} ms elapsed)", ((float)numMethodCalls) / sw.ElapsedMilliseconds, sw.ElapsedMilliseconds);
210            Console.WriteLine("See matched method calls in {0}", callLogPath);
211
212            if(!File.Exists(csvPath)) {
213                File.WriteAllText(csvPath, String.Format("{0}{1}", String.Join(",", "Project", "Method Calls", "Matched Method Calls", "Time (ms)"), Environment.NewLine));
214            }
215            File.AppendAllText(csvPath, String.Format("{0}{1}", String.Join(",", sourcePath, numMethodCalls, numMatchedMethodCalls, sw.ElapsedMilliseconds), Environment.NewLine));
216        }
217
218        private static void OutputCallGraphByType(Scope globalScope, string jsonPath) {
219            using(var writer = new JsonTextWriter(new StreamWriter(jsonPath))) {
220                writer.WriteStartArray();
221                foreach(var typeDefinition in globalScope.GetDescendantScopesAndSelf<TypeDefinition>()) {
222                    writer.WriteStartObject();
223                    writer.WritePropertyName("name");
224                    writer.WriteValue(typeDefinition.GetFullName());
225                    
226                    
227                    var calls = from scope in typeDefinition.GetDescendantScopesAndSelf()
228                                from call in scope.MethodCalls
229                                select call;
230                    
231                    writer.WritePropertyName("size");
232                    writer.WriteValue(calls.Count());
233
234                    // find the parent type of all the calls
235                    var callMatches = from call in calls
236                                      let match = call.FindMatches().FirstOrDefault()
237                                      where match != null
238                                      let parentOfMatch = match.GetFirstParent<TypeDefinition>()
239                                      where parentOfMatch != null
240                                      select parentOfMatch.GetFullName();
241                    // output the calls property and array
242                    writer.WritePropertyName("calls");
243                    writer.WriteStartArray();
244                    foreach (var call in callMatches)
245	                {
246                        writer.WriteValue(call);
247	                }
248                    writer.WriteEndArray();
249                    writer.WriteEndObject();
250                }
251                writer.WriteEndArray();
252            }
253        }
254    }
255}