PageRenderTime 51ms CodeModel.GetById 32ms app.highlight 16ms RepoModel.GetById 0ms app.codeStats 0ms

/src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs

https://github.com/eusebiu/SharpDevelop
C# | 388 lines | 336 code | 44 blank | 8 comment | 31 complexity | 4535ec5565df41dcfefce366ffd78223 MD5 | raw file
  1// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt)
  2// This code is distributed under the GNU LGPL (for details please see \doc\license.txt)
  3
  4using System;
  5using System.Collections.Generic;
  6using System.Diagnostics;
  7using System.IO;
  8using System.Linq;
  9using System.Text;
 10using System.Text.RegularExpressions;
 11
 12namespace NRefactoryASTGenerator
 13{
 14	static class KeywordGenerator
 15	{
 16		static readonly string baseDir = "../../../Project/Src/Lexer/";
 17		static readonly string testBaseDir = "../../../Test/Lexer/";
 18		static readonly string parserBaseDir = "../../../Project/Src/Parser/";
 19		
 20		public static void Generate()
 21		{
 22			Generate("CSharp");
 23			Generate("VBNet");
 24		}
 25
 26		static void Generate(string language)
 27		{
 28			try {
 29				Dictionary<string, string> properties = new Dictionary<string, string>();
 30				Dictionary<string, string[]> sets = new Dictionary<string, string[]>();
 31				List<string> keywords = new List<string>();
 32				List<string> terminals = new List<string>();
 33				Dictionary<string, string> specialChars = new Dictionary<string, string>();
 34
 35				ReadFile(properties, sets, keywords, terminals, specialChars, language);
 36
 37				GenerateFiles(properties, sets, keywords, terminals, specialChars, language);
 38			} catch (Exception e) {
 39				Debug.Print(e.ToString());
 40			}
 41		}
 42		
 43		static void GenerateFiles(Dictionary<string, string> properties, Dictionary<string, string[]> sets,
 44		                          List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars,
 45		                          string language)
 46		{
 47			GenerateKeywords(properties, keywords, language);
 48			GenerateTokens(properties, sets, keywords, terminals, specialChars, language);
 49			GenerateTests(keywords, specialChars, language);
 50			GenerateKeywordSection(keywords, terminals, specialChars, language);
 51		}
 52		
 53		static void GenerateKeywordSection(List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars, string language)
 54		{
 55			string sourceDir = Path.Combine(parserBaseDir, language, (language == "CSharp" ? "cs" : language) + ".atg");
 56			StringBuilder builder = new StringBuilder();
 57			
 58			builder.AppendLine("/* START AUTOGENERATED TOKENS SECTION */");
 59			builder.AppendLine("TOKENS");
 60			builder.AppendLine("\t/* ----- terminal classes ----- */");
 61			builder.AppendLine("\t/* EOF is 0 */");
 62			
 63			foreach (string terminal in terminals) {
 64				if (terminal == "EOF")
 65					continue;
 66				if (terminal == "Identifier") {
 67					builder.AppendLine("\tident");
 68					continue;
 69				}
 70				builder.AppendLine("\t" + terminal);
 71			}
 72			
 73			builder.AppendLine();
 74			builder.AppendLine("\t/* ----- special character ----- */");
 75			foreach (string specialChar in specialChars.Values) {
 76				builder.AppendLine("\t" + specialChar);
 77			}
 78			
 79			builder.AppendLine();
 80			builder.AppendLine("\t/* ----- keywords ----- */");
 81			foreach (string keyword in keywords) {
 82				builder.AppendLine("\t\"" + keyword + "\"");
 83			}
 84			
 85			builder.AppendLine("/* END AUTOGENERATED TOKENS SECTION */");
 86			
 87			string[] generatedLines = builder.ToString().Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
 88			string[] lines = File.ReadAllLines(sourceDir);
 89			
 90			var newContent = lines
 91				.TakeWhile(l => l != "/* START AUTOGENERATED TOKENS SECTION */")
 92				.Concat(generatedLines)
 93				.Concat(lines.SkipWhile(l => l != "/* END AUTOGENERATED TOKENS SECTION */").Skip(2))
 94				.ToArray();
 95
 96			File.WriteAllLines(sourceDir, newContent);
 97		}
 98
 99		static void GenerateTests(List<string> keywords, Dictionary<string, string> specialChars, string language)
100		{
101			string sourceDir = Path.Combine(testBaseDir, language, "LexerTests.cs");
102			using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
103				writer.WriteLine("// this file was autogenerated by a tool.");
104				writer.WriteLine("using System;");
105				writer.WriteLine("using System.IO;");
106				writer.WriteLine("using NUnit.Framework;");
107				writer.WriteLine("using ICSharpCode.NRefactory.Parser;");
108				writer.WriteLine("using ICSharpCode.NRefactory.Parser.{0};", language == "VBNet" ? "VB" : language);
109				writer.WriteLine("using ICSharpCode.NRefactory.PrettyPrinter;");
110				writer.WriteLine();
111				writer.WriteLine("namespace ICSharpCode.NRefactory.Tests.Lexer.{0}", language == "VBNet" ? "VB" : language);
112				writer.WriteLine("{");
113				writer.WriteLine("\t[TestFixture]");
114				writer.WriteLine("\tpublic sealed class LexerTests");
115				writer.WriteLine("\t{");
116				writer.WriteLine("\t\tILexer GenerateLexer(StringReader sr)");
117				writer.WriteLine("\t\t{");
118				writer.WriteLine("\t\t\treturn ParserFactory.CreateLexer(SupportedLanguage.{0}, sr);", language);
119				writer.WriteLine("\t\t}");
120				for (int i = 0; i < specialChars.Values.Count; i++) {
121					writer.WriteLine();
122					writer.WriteLine("\t\t[Test]");
123					writer.WriteLine("\t\tpublic void Test{0}()", specialChars.Keys.ElementAt(i));
124					writer.WriteLine("\t\t{");
125					writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader({0}));", specialChars.Values.ElementAt(i));
126					writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", specialChars.Keys.ElementAt(i));
127					writer.WriteLine("\t\t}");
128				}
129				foreach (string keyword in keywords) {
130					if (keyword == "Rem")
131						continue;
132					writer.WriteLine();
133					writer.WriteLine("\t\t[Test]");
134					writer.WriteLine("\t\tpublic void Test{0}()", UpperCaseFirst(keyword));
135					writer.WriteLine("\t\t{");
136					writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader(\"{0}\"));", keyword);
137					writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", UpperCaseFirst(keyword));
138					writer.WriteLine("\t\t}");
139				}
140				writer.WriteLine("\t}");
141				writer.WriteLine("}");
142			}
143		}
144
145		static void GenerateTokens(Dictionary<string, string> properties, Dictionary<string, string[]> sets, List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars, string language)
146		{
147			string sourceDir = Path.Combine(baseDir, language, "Tokens.cs");
148			using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
149				writer.WriteLine("// this file was autogenerated by a tool.");
150				writer.WriteLine("using System;");
151				writer.WriteLine("using System.Collections;");
152				writer.WriteLine();
153				writer.WriteLine("namespace {0}", properties["Namespace"]);
154				writer.WriteLine("{");
155				writer.WriteLine("\tpublic static class Tokens");
156				writer.WriteLine("\t{");
157				writer.WriteLine("\t\t// ----- terminal classes -----");
158				int tokenValue = 0;
159				foreach (string terminal in terminals)
160					writer.WriteToken(terminal, ref tokenValue);
161				writer.WriteLine();
162				writer.WriteLine("\t\t// ----- special character -----");
163				foreach (string specialChar in specialChars.Keys)
164					writer.WriteToken(specialChar, ref tokenValue);
165				writer.WriteLine();
166				writer.WriteLine("\t\t// ----- keywords -----");
167				foreach (string keyword in keywords)
168					writer.WriteToken(keyword, ref tokenValue);
169				writer.WriteLine();
170				writer.WriteLine("\t\tpublic const int MaxToken = {0};", tokenValue);
171				if (sets.Any()) {
172					writer.WriteLine("\t\tstatic BitArray NewSet(params int[] values)");
173					writer.WriteLine("\t\t{");
174					writer.WriteLine("\t\t\tBitArray bitArray = new BitArray(MaxToken);");
175					writer.WriteLine("\t\t\tforeach (int val in values) {");
176					writer.WriteLine("\t\t\tbitArray[val] = true;");
177					writer.WriteLine("\t\t\t}");
178					writer.WriteLine("\t\t\treturn bitArray;");
179					writer.WriteLine("\t\t}");
180					foreach (var pair in sets) {
181						StringBuilder builder = new StringBuilder();
182						PrintList(pair.Value, builder, sets, specialChars);
183						writer.WriteLine("\t\tpublic static BitArray {0} = NewSet({1});", pair.Key, builder.ToString());
184					}
185					writer.WriteLine();
186				}
187
188				// write token number --> string function.
189				writer.WriteLine("\t\tstatic string[] tokenList = new string[] {");
190
191				writer.WriteLine("\t\t\t// ----- terminal classes -----");
192				foreach (string terminal in terminals)
193					writer.WriteLine("\t\t\t\"<{0}>\",", terminal);
194
195				writer.WriteLine("\t\t\t// ----- special character -----");
196				foreach (string specialChar in specialChars.Values)
197					writer.WriteLine("\t\t\t{0},", specialChar);
198
199				writer.WriteLine("\t\t\t// ----- keywords -----");
200				foreach (string keyword in keywords)
201					writer.WriteLine("\t\t\t\"{0}\",", keyword);
202
203				writer.WriteLine("\t\t};");
204
205				writer.WriteLine("\t\tpublic static string GetTokenString(int token)");
206				writer.WriteLine("\t\t{");
207				writer.WriteLine("\t\t\tif (token >= 0 && token < tokenList.Length) {");
208				writer.WriteLine("\t\t\t\treturn tokenList[token];");
209				writer.WriteLine("\t\t\t}");
210				writer.WriteLine("\t\t\tthrow new System.NotSupportedException(\"Unknown token:\" + token);");
211				writer.WriteLine("\t\t}");
212
213				writer.WriteLine("\t}");
214				writer.WriteLine("}");
215			}
216		}
217
218		static void PrintList(string[] value, StringBuilder builder, Dictionary<string, string[]> sets, Dictionary<string, string> specialChars)
219		{
220			for (int i = 0; i < value.Length; i++) {
221				string item = value[i];
222				if (Regex.IsMatch(item, "\\\"(\\w+)\\\"")) // keywords
223					builder.Append(UpperCaseFirst(item.Trim('"', ' ', '\t')));
224				else if (Regex.IsMatch(item, "\\\"(\\W+)\\\"")) // special chars
225					builder.Append(specialChars.Keys.ElementAt(specialChars.Values.FindIndex(it => item == it)));
226				else if (Regex.IsMatch(item, "@(\\w+)")) // other list
227					PrintList(sets[item.Substring(1)], builder, sets, specialChars);
228				else
229					builder.Append(item);
230				if (i + 1 < value.Length)
231					builder.Append(", ");
232			}
233		}
234		
235		static void GenerateKeywords(Dictionary<string, string> properties, List<string> keywords, string language)
236		{
237			string sourceDir = Path.Combine(baseDir, language, "Keywords.cs");
238			using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
239				writer.WriteLine("// this file was autogenerated by a tool.");
240				writer.WriteLine("using System;");
241				writer.WriteLine();
242				writer.WriteLine("namespace {0}", properties["Namespace"]);
243				writer.WriteLine("{");
244				writer.WriteLine("\tpublic static class Keywords");
245				writer.WriteLine("\t{");
246				writer.WriteLine("\t\tstatic readonly string[] keywordList = {");
247				for (int i = 0; i < keywords.Count; i++) {
248					writer.Write("\t\t\t\"{0}\"", properties["UpperCaseKeywords"] == "True" ? keywords[i].ToUpperInvariant() : keywords[i]);
249					if (i + 1 < keywords.Count)
250						writer.Write(",");
251					writer.WriteLine();
252				}
253				writer.WriteLine("\t\t};");
254				writer.WriteLine("\t\t");
255				writer.WriteLine("\t\tstatic LookupTable keywords = new LookupTable({0});", properties["UpperCaseKeywords"] == "True" ? "false" : "true");
256				writer.WriteLine("\t\t");
257				writer.WriteLine("\t\tstatic Keywords()");
258				writer.WriteLine("\t\t{");
259				writer.WriteLine("\t\t\tfor (int i = 0; i < keywordList.Length; ++i) {");
260				writer.WriteLine("\t\t\t\tkeywords[keywordList[i]] = i + Tokens.{0};", UpperCaseFirst(keywords[0]));
261				writer.WriteLine("\t\t\t}");
262				writer.WriteLine("\t\t}");
263				writer.WriteLine("\t\t");
264				writer.WriteLine("\t\tpublic static int GetToken(string keyword)");
265				writer.WriteLine("\t\t{");
266				writer.WriteLine("\t\t\treturn keywords[keyword];");
267				writer.WriteLine("\t\t}");
268				writer.WriteLine("\t\t");
269				writer.WriteLine("\t\tpublic static bool IsNonIdentifierKeyword(string word)");
270				writer.WriteLine("\t\t{");
271				writer.WriteLine("\t\t\tint token = GetToken(word);");
272				writer.WriteLine("\t\t\tif (token < 0)");
273				writer.WriteLine("\t\t\t\treturn false;");
274				writer.WriteLine("\t\t\treturn !Tokens.IdentifierTokens[token];");
275				writer.WriteLine("\t\t}");
276
277				writer.WriteLine("\t}");
278				writer.WriteLine("}");
279
280				writer.Close();
281			}
282		}
283
284		#region input
285		static void ReadFile(Dictionary<string, string> properties, Dictionary<string, string[]> sets,
286		                     List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars,
287		                     string language)
288		{
289			string sourceDir = Path.Combine(baseDir, language, "KeywordList.txt");
290			
291			using (StreamReader reader = new StreamReader(new FileStream(sourceDir, FileMode.Open))) {
292				string line = reader.ReadLine();
293				while (line != null) {
294					ReadProperty(properties, line);
295					ReadKeyword(keywords, line);
296					ReadSet(sets, line);
297					ReadTerminalSymbol(terminals, line);
298					ReadSpecialChar(specialChars, line);
299					line = reader.ReadLine();
300				}
301				reader.Close();
302			}
303		}
304
305		static void ReadProperty(Dictionary<string, string> properties, string line)
306		{
307			// properties form: $PROPERTY = "VALUE"
308			Match match = Regex.Match(line, @"^\s*\$(\w+)\s*=\s*(\S+)\s*$");
309			
310			if (match.Success) {
311				properties.Add(match.Groups[1].Value, match.Groups[2].Value);
312			}
313		}
314
315		static void ReadKeyword(List<string> keywords, string line)
316		{
317			// special keywords form: "VALUE"
318			Match match = Regex.Match(line, "^\\s*\\\"(\\S+)\\s*\\\"\\s*$");
319			
320			if (match.Success) {
321				keywords.Add(match.Groups[1].Value);
322			}
323		}
324
325		static void ReadSet(Dictionary<string, string[]> sets, string line)
326		{
327			// sets form: NAME(comma separated list)
328			Match match = Regex.Match(line, @"^\s*(\w+)\s*\((.*)\)\s*$");
329			
330			if (match.Success) {
331				sets.Add(
332					match.Groups[1].Value,
333					match.Groups[2].Value.Split(new[] {", "}, StringSplitOptions.None)
334				);
335			}
336		}
337
338		static void ReadTerminalSymbol(List<string> terminals, string line)
339		{
340			// special terminal classes form: name
341			Match match = Regex.Match(line, @"^\s*(\w+)\s*$");
342			
343			if (match.Success) {
344				terminals.Add(match.Groups[1].Value);
345			}
346		}
347
348		static void ReadSpecialChar(Dictionary<string, string> specialChars, string line)
349		{
350			// special characters form: name = "VALUE"
351			Match match = Regex.Match(line, @"^\s*(\w+)\s*=\s*(\S+)\s*$");
352			
353			if (match.Success) {
354				specialChars.Add(match.Groups[1].Value, match.Groups[2].Value);
355			}
356		}
357		#endregion
358		
359		#region helpers
360		static string UpperCaseFirst(string keyword)
361		{
362			return char.ToUpperInvariant(keyword[0]) + keyword.Substring(1);
363		}
364		
365		static void WriteToken(this StreamWriter writer, string tokenName, ref int tokenValue)
366		{
367			string formattedName = UpperCaseFirst(tokenName).PadRight(20);
368			if (tokenName == "GetType" || tokenName.ToLowerInvariant() == "equals")
369				writer.WriteLine("\t\tnew public const int {0} = {1};", formattedName, tokenValue);
370			else
371				writer.WriteLine("\t\tpublic const int {0} = {1};", formattedName, tokenValue);
372			tokenValue++;
373		}
374		
375		static int FindIndex<T>(this IEnumerable<T> items, Func<T, bool> f)
376		{
377			int index = -1;
378			foreach (T item in items) {
379				index++;
380				if (f(item))
381					return index;
382			}
383			
384			return -1;
385		}
386		#endregion
387	}
388}