PageRenderTime 105ms CodeModel.GetById 3ms app.highlight 87ms RepoModel.GetById 2ms app.codeStats 0ms

/src/main/java/com/laytonsmith/core/MethodScriptCompiler.java

https://github.com/sk89q/commandhelper
Java | 2003 lines | 1597 code | 63 blank | 343 comment | 601 complexity | 019a85dcb2f0738f382eb8cdf0e1d9cc MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1package com.laytonsmith.core;
   2
   3import com.laytonsmith.annotations.breakable;
   4import com.laytonsmith.annotations.nolinking;
   5import com.laytonsmith.annotations.unbreakable;
   6import com.laytonsmith.core.Optimizable.OptimizationOption;
   7import com.laytonsmith.core.compiler.FileOptions;
   8import com.laytonsmith.core.compiler.KeywordList;
   9import com.laytonsmith.core.constructs.CDouble;
  10import com.laytonsmith.core.constructs.CFunction;
  11import com.laytonsmith.core.constructs.CIdentifier;
  12import com.laytonsmith.core.constructs.CInt;
  13import com.laytonsmith.core.constructs.CKeyword;
  14import com.laytonsmith.core.constructs.CLabel;
  15import com.laytonsmith.core.constructs.CNull;
  16import com.laytonsmith.core.constructs.CPreIdentifier;
  17import com.laytonsmith.core.constructs.CSlice;
  18import com.laytonsmith.core.constructs.CString;
  19import com.laytonsmith.core.constructs.CSymbol;
  20import com.laytonsmith.core.constructs.CVoid;
  21import com.laytonsmith.core.constructs.Construct;
  22import com.laytonsmith.core.constructs.IVariable;
  23import com.laytonsmith.core.constructs.Target;
  24import com.laytonsmith.core.constructs.Token;
  25import com.laytonsmith.core.constructs.Token.TType;
  26import com.laytonsmith.core.constructs.Variable;
  27import com.laytonsmith.core.environments.Environment;
  28import com.laytonsmith.core.environments.GlobalEnv;
  29import com.laytonsmith.core.exceptions.ConfigCompileException;
  30import com.laytonsmith.core.exceptions.ConfigCompileGroupException;
  31import com.laytonsmith.core.exceptions.ConfigRuntimeException;
  32import com.laytonsmith.core.exceptions.ProgramFlowManipulationException;
  33import com.laytonsmith.core.functions.Compiler;
  34import com.laytonsmith.core.functions.DataHandling;
  35import com.laytonsmith.core.functions.Function;
  36import com.laytonsmith.core.functions.FunctionBase;
  37import com.laytonsmith.core.functions.FunctionList;
  38import com.laytonsmith.core.functions.IncludeCache;
  39import com.laytonsmith.persistence.DataSourceException;
  40import java.io.File;
  41import java.io.IOException;
  42import java.net.URISyntaxException;
  43import java.util.ArrayList;
  44import java.util.Arrays;
  45import java.util.EmptyStackException;
  46import java.util.EnumSet;
  47import java.util.HashMap;
  48import java.util.HashSet;
  49import java.util.List;
  50import java.util.Map;
  51import java.util.Set;
  52import java.util.Stack;
  53import java.util.concurrent.atomic.AtomicInteger;
  54import java.util.regex.Pattern;
  55
  56/**
  57 * The MethodScriptCompiler class handles the various stages of compilation and
  58 * provides helper methods for execution of the compiled trees.
  59 */
  60public final class MethodScriptCompiler {
  61
  62	private final static EnumSet<Optimizable.OptimizationOption> NO_OPTIMIZATIONS = EnumSet.noneOf(Optimizable.OptimizationOption.class);
  63
  64	private final static FileOptions fileOptions = new FileOptions(new HashMap<String, String>());
  65
  66	private MethodScriptCompiler() {
  67	}
  68
  69	private static final Pattern VAR_PATTERN = Pattern.compile("\\$[a-zA-Z0-9_]+");
  70	private static final Pattern IVAR_PATTERN = Pattern.compile("@[a-zA-Z0-9_]+");
  71
  72	/**
  73	 * Lexes the script, and turns it into a token stream. This looks through the script
  74	 * character by character.
  75	 * @param script The script to lex
  76	 * @param file The file this script came from, or potentially null if the code is from
  77	 * a dynamic source
  78	 * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure
  79	 * MethodScript is defined as code that doesn't have command alias wrappers.
  80	 * @return A stream of tokens
  81	 * @throws ConfigCompileException If compilation fails due to bad syntax
  82	 */
  83	@SuppressWarnings("UnnecessaryContinue")
  84	public static List<Token> lex(String script, File file, boolean inPureMScript) throws ConfigCompileException {
  85		if(script.isEmpty()){
  86			return new ArrayList<>();
  87		}
  88		if((int)script.charAt(0) == 65279){
  89			// Remove the UTF-8 Byte Order Mark, if present.
  90			script = script.substring(1);
  91		}
  92		script = script.replaceAll("\r\n", "\n");
  93		script = script + "\n";
  94		Set<String> keywords = KeywordList.getKeywordNames();
  95		List<Token> token_list = new ArrayList<>();
  96		//Set our state variables
  97		boolean state_in_quote = false;
  98		int quoteLineNumberStart = 1;
  99		boolean in_smart_quote = false;
 100		int smartQuoteLineNumberStart = 1;
 101		boolean in_comment = false;
 102		int commentLineNumberStart = 1;
 103		boolean comment_is_block = false;
 104		boolean in_opt_var = false;
 105		boolean inCommand = (!inPureMScript);
 106		boolean inMultiline = false;
 107		StringBuilder buf = new StringBuilder();
 108		int line_num = 1;
 109		int column = 1;
 110		int lastColumn = 0;
 111		Target target = Target.UNKNOWN;
 112		//first we lex
 113		for (int i = 0; i < script.length(); i++) {
 114			Character c = script.charAt(i);
 115			Character c2 = null;
 116			Character c3 = null;
 117			if (i < script.length() - 1) {
 118				c2 = script.charAt(i + 1);
 119			}
 120			if (i < script.length() - 2) {
 121				c3 = script.charAt(i + 2);
 122			}
 123
 124			column += i - lastColumn;
 125			lastColumn = i;
 126			if (c == '\n') {
 127				line_num++;
 128				column = 1;
 129				if(!inMultiline && !inPureMScript){
 130					inCommand = true;
 131				}
 132			}
 133			target = new Target(line_num, file, column);
 134
 135			//Comment handling. If we're inside a string, bypass this though
 136			if (!state_in_quote && !in_smart_quote) {
 137				//Block comments start
 138				if (c == '/' && c2 == '*' && !in_comment) {
 139					in_comment = true;
 140					comment_is_block = true;
 141					commentLineNumberStart = line_num;
 142					i++;
 143					continue;
 144				}
 145				//Line comment start
 146				if (c == '#' && !in_comment) {
 147					in_comment = true;
 148					comment_is_block = false;
 149					continue;
 150				}
 151				//Double slash line comment start
 152				if(c == '/' && c2 == '/' && !in_comment){
 153					in_comment = true;
 154					comment_is_block = false;
 155					i++;
 156					continue;
 157				}
 158				//Block comment end
 159				if (c == '*' && c2 == '/' && in_comment && comment_is_block) {
 160					if (in_comment && comment_is_block) {
 161						in_comment = false;
 162						comment_is_block = false;
 163						i++;
 164						continue;
 165					} else if (!in_comment) {
 166						throw new ConfigCompileException("Unexpected block comment end", target);
 167					} //else they put it in a line comment, which is fine
 168				}
 169				//Line comment end
 170				if (c == '\n' && in_comment && !comment_is_block) {
 171					in_comment = false;
 172					continue;
 173				}
 174			}
 175			//Currently, if they are in a comment, we completely throw this away. Eventually block
 176			//comments that were started with /** will be kept and applied to the next identifier, but for the time
 177			//being, nothing.
 178			if (in_comment) {
 179				continue;
 180			}
 181			if(c == '+' && c2 == '=' && !state_in_quote){
 182				if(buf.length() > 0){
 183					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 184					buf = new StringBuilder();
 185				}
 186				token_list.add(new Token(TType.PLUS_ASSIGNMENT, "+=", target));
 187				i++;
 188				continue;
 189			}
 190			if(c == '-' && c2 == '=' && !state_in_quote){
 191				if(buf.length() > 0){
 192					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 193					buf = new StringBuilder();
 194				}
 195				token_list.add(new Token(TType.MINUS_ASSIGNMENT, "-=", target));
 196				i++;
 197				continue;
 198			}
 199			if(c == '*' && c2 == '=' && !state_in_quote){
 200				if(buf.length() > 0){
 201					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 202					buf = new StringBuilder();
 203				}
 204				token_list.add(new Token(TType.MULTIPLICATION_ASSIGNMENT, "*=", target));
 205				i++;
 206				continue;
 207			}
 208			//This has to come before division and equals
 209			if(c == '/' && c2 == '=' && !state_in_quote){
 210				if(buf.length() > 0){
 211					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 212					buf = new StringBuilder();
 213				}
 214				token_list.add(new Token(TType.DIVISION_ASSIGNMENT, "/=", target));
 215				i++;
 216				continue;
 217			}
 218			if(c == '.' && c2 == '=' && !state_in_quote){
 219				if(buf.length() > 0){
 220					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 221					buf = new StringBuilder();
 222				}
 223				token_list.add(new Token(TType.CONCAT_ASSIGNMENT, "/=", target));
 224				i++;
 225				continue;
 226			}
 227			//This has to come before subtraction and greater than
 228			if (c == '-' && c2 == '>' && !state_in_quote) {
 229				if (buf.length() > 0) {
 230					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 231					buf = new StringBuilder();
 232				}
 233				token_list.add(new Token(TType.DEREFERENCE, "->", target));
 234				i++;
 235				continue;
 236			}
 237			//Increment and decrement must come before plus and minus
 238			if (c == '+' && c2 == '+' && !state_in_quote) {
 239				if (buf.length() > 0) {
 240					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 241					buf = new StringBuilder();
 242				}
 243				token_list.add(new Token(TType.INCREMENT, "++", target));
 244				i++;
 245				continue;
 246			}
 247			if (c == '-' && c2 == '-' && !state_in_quote) {
 248				if (buf.length() > 0) {
 249					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 250					buf = new StringBuilder();
 251				}
 252				token_list.add(new Token(TType.DECREMENT, "--", target));
 253				i++;
 254				continue;
 255			}
 256
 257			if (c == '%' && !state_in_quote) {
 258				if (buf.length() > 0) {
 259					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 260					buf = new StringBuilder();
 261				}
 262				token_list.add(new Token(TType.MODULO, "%", target));
 263				continue;
 264			}
 265
 266			//Math symbols must come after comment parsing, due to /* and */ block comments
 267			//Block comments are caught above
 268			if (c == '*' && c2 == '*' && !state_in_quote) {
 269				if (buf.length() > 0) {
 270					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 271					buf = new StringBuilder();
 272				}
 273				token_list.add(new Token(TType.EXPONENTIAL, "**", target));
 274				i++;
 275				continue;
 276			}
 277			if (c == '*' && !state_in_quote) {
 278				if (buf.length() > 0) {
 279					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 280					buf = new StringBuilder();
 281				}
 282				token_list.add(new Token(TType.MULTIPLICATION, "*", target));
 283				continue;
 284			}
 285			if (c == '+' && !state_in_quote) {
 286				if (buf.length() > 0) {
 287					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 288					buf = new StringBuilder();
 289				}
 290				token_list.add(new Token(TType.PLUS, "+", target));
 291				continue;
 292			}
 293			if (c == '-' && !state_in_quote) {
 294				if (buf.length() > 0) {
 295					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 296					buf = new StringBuilder();
 297				}
 298				token_list.add(new Token(TType.MINUS, "-", target));
 299				continue;
 300			}
 301			//Protect against commands
 302			if (c == '/' && !Character.isLetter(c2) && !state_in_quote) {
 303				if (buf.length() > 0) {
 304					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 305					buf = new StringBuilder();
 306				}
 307				token_list.add(new Token(TType.DIVISION, "/", target));
 308				continue;
 309			}
 310			//Logic symbols
 311			if (c == '>' && c2 == '=' && !state_in_quote) {
 312				if (buf.length() > 0) {
 313					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 314					buf = new StringBuilder();
 315				}
 316				token_list.add(new Token(TType.GTE, ">=", target));
 317				i++;
 318				continue;
 319			}
 320			if (c == '<' && c2 == '=' && !state_in_quote) {
 321				if (buf.length() > 0) {
 322					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 323					buf = new StringBuilder();
 324				}
 325				token_list.add(new Token(TType.LTE, "<=", target));
 326				i++;
 327				continue;
 328			}
 329			//multiline has to come before gt/lt
 330			if (c == '<' && c2 == '<' && c3 == '<' && !state_in_quote) {
 331				if (buf.length() > 0) {
 332					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 333					buf = new StringBuilder();
 334				}
 335				token_list.add(new Token(TType.MULTILINE_END, "<<<", target));
 336				inMultiline = false;
 337				i++;
 338				i++;
 339				continue;
 340			}
 341			if (c == '>' && c2 == '>' && c3 == '>' && !state_in_quote) {
 342				if (buf.length() > 0) {
 343					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 344					buf = new StringBuilder();
 345				}
 346				token_list.add(new Token(TType.MULTILINE_START, ">>>", target));
 347				inMultiline = true;
 348				i++;
 349				i++;
 350				continue;
 351			}
 352			if (c == '<' && !state_in_quote) {
 353				if (buf.length() > 0) {
 354					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 355					buf = new StringBuilder();
 356				}
 357				token_list.add(new Token(TType.LT, "<", target));
 358				continue;
 359			}
 360			if (c == '>' && !state_in_quote) {
 361				if (buf.length() > 0) {
 362					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 363					buf = new StringBuilder();
 364				}
 365				token_list.add(new Token(TType.GT, ">", target));
 366				continue;
 367			}
 368			if (c == '=' && c2 == '=' && c3 == '=' && !state_in_quote) {
 369				if (buf.length() > 0) {
 370					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 371					buf = new StringBuilder();
 372				}
 373				token_list.add(new Token(TType.STRICT_EQUALS, "===", target));
 374				i++;
 375				i++;
 376				continue;
 377			}
 378			if (c == '!' && c2 == '=' && c3 == '=' && !state_in_quote) {
 379				if (buf.length() > 0) {
 380					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 381					buf = new StringBuilder();
 382				}
 383				token_list.add(new Token(TType.STRICT_NOT_EQUALS, "!==", target));
 384				i++;
 385				i++;
 386				continue;
 387			}
 388			if (c == '=' && c2 == '=' && !state_in_quote) {
 389				if (buf.length() > 0) {
 390					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 391					buf = new StringBuilder();
 392				}
 393				token_list.add(new Token(TType.EQUALS, "==", target));
 394				i++;
 395				continue;
 396			}
 397			if (c == '!' && c2 == '=' && !state_in_quote) {
 398				if (buf.length() > 0) {
 399					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 400					buf = new StringBuilder();
 401				}
 402				token_list.add(new Token(TType.NOT_EQUALS, "!=", target));
 403				i++;
 404				continue;
 405			}
 406			if (c == '&' && c2 == '&' && !state_in_quote) {
 407				if (buf.length() > 0) {
 408					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 409					buf = new StringBuilder();
 410				}
 411				token_list.add(new Token(TType.LOGICAL_AND, "&&", target));
 412				i++;
 413				continue;
 414			}
 415			if (c == '|' && c2 == '|' && !state_in_quote) {
 416				if (buf.length() > 0) {
 417					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 418					buf = new StringBuilder();
 419				}
 420				token_list.add(new Token(TType.LOGICAL_OR, "||", target));
 421				i++;
 422				continue;
 423			}
 424			if (c == '!' && !state_in_quote) {
 425				if (buf.length() > 0) {
 426					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 427					buf = new StringBuilder();
 428				}
 429				token_list.add(new Token(TType.LOGICAL_NOT, "!", target));
 430				continue;
 431			}
 432			if (c == '{' && !state_in_quote) {
 433				if (buf.length() > 0) {
 434					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 435					buf = new StringBuilder();
 436				}
 437				token_list.add(new Token(TType.LCURLY_BRACKET, "{", target));
 438				continue;
 439			}
 440			if (c == '}' && !state_in_quote) {
 441				if (buf.length() > 0) {
 442					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 443					buf = new StringBuilder();
 444				}
 445				token_list.add(new Token(TType.RCURLY_BRACKET, "}", target));
 446				continue;
 447			}
 448			//I don't want to use these symbols yet, especially since bitwise operations are rare.
 449//            if(c == '&' && !state_in_quote){
 450//                if (buf.length() > 0) {
 451//                    token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 452//                    buf = new StringBuilder();
 453//                }
 454//                token_list.add(new Token(TType.BIT_AND, "&", target));
 455//                continue;
 456//            }
 457//            if(c == '|' && !state_in_quote){
 458//                if (buf.length() > 0) {
 459//                    token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 460//                    buf = new StringBuilder();
 461//                }
 462//                token_list.add(new Token(TType.BIT_OR, "|", target));
 463//                continue;
 464//            }
 465//            if(c == '^' && !state_in_quote){
 466//                if (buf.length() > 0) {
 467//                    token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 468//                    buf = new StringBuilder();
 469//                }
 470//                token_list.add(new Token(TType.BIT_XOR, "^", target));
 471//                continue;
 472//            }
 473
 474			if (c == '.' && c2 == '.' && !state_in_quote) {
 475				//This one has to come before plain .
 476				if (buf.length() > 0) {
 477					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 478					buf = new StringBuilder();
 479				}
 480				token_list.add(new Token(TType.SLICE, "..", target));
 481				i++;
 482				continue;
 483			}
 484			if(c == '.' && !state_in_quote){
 485				if (buf.length() > 0){
 486					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 487					buf = new StringBuilder();
 488				}
 489				// Dots are resolved later, because order of operations actually matters here, depending on whether
 490				// or not the previous token is a string or a number. But actually, it isn't about the previous token, it's
 491				// about the previous construct, and we want to handle it in a more robust way, so we pass it along to
 492				// the compiler stage.
 493				token_list.add(new Token(TType.DOT, ".", target));
 494				continue;
 495			}
 496			if (c == ':' && c2 == ':' && !state_in_quote) {
 497				if (buf.length() > 0) {
 498					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 499					buf = new StringBuilder();
 500				}
 501				token_list.add(new Token(TType.DEREFERENCE, "::", target));
 502				i++;
 503				continue;
 504			}
 505			if (c == '[' && !state_in_quote) {
 506				if (buf.length() > 0) {
 507					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 508					buf = new StringBuilder();
 509				}
 510				token_list.add(new Token(TType.LSQUARE_BRACKET, "[", target));
 511				in_opt_var = true;
 512				continue;
 513			}
 514			//This has to come after == and ===
 515			if (c == '=' && !state_in_quote) {
 516				if (buf.length() > 0) {
 517					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 518					buf = new StringBuilder();
 519				}
 520				if(inCommand){
 521					if (in_opt_var) {
 522						token_list.add(new Token(TType.OPT_VAR_ASSIGN, "=", target));
 523					} else {
 524						token_list.add(new Token(TType.ALIAS_END, "=", target));
 525						inCommand = false;
 526					}
 527				} else {
 528					token_list.add(new Token(TType.ASSIGNMENT, "=", target));
 529				}
 530				continue;
 531			}
 532			if (c == ']' && !state_in_quote) {
 533				if (buf.length() > 0) {
 534					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 535					buf = new StringBuilder();
 536				}
 537				token_list.add(new Token(TType.RSQUARE_BRACKET, "]", target));
 538				in_opt_var = false;
 539				continue;
 540			}
 541			if (c == ':' && !state_in_quote) {
 542				if (buf.length() > 0) {
 543					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 544					buf = new StringBuilder();
 545				}
 546				token_list.add(new Token(TType.LABEL, ":", target));
 547				continue;
 548			}
 549			if (c == ',' && !state_in_quote) {
 550				if (buf.length() > 0) {
 551					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 552					buf = new StringBuilder();
 553				}
 554				token_list.add(new Token(TType.COMMA, ",", target));
 555				continue;
 556			}
 557			if (c == '(' && !state_in_quote) {
 558				if (buf.length() > 0) {
 559					token_list.add(new Token(TType.FUNC_NAME, buf.toString(), target));
 560					buf = new StringBuilder();
 561				} else {
 562					//The previous token, if unknown, should be changed to a FUNC_NAME. If it's not
 563					//unknown, we may be doing standalone parenthesis, so auto tack on the __autoconcat__ function
 564					try {
 565						int count = 1;
 566						while (token_list.get(token_list.size() - count).type == TType.WHITESPACE) {
 567							count++;
 568						}
 569						if (token_list.get(token_list.size() - count).type == TType.UNKNOWN) {
 570							token_list.get(token_list.size() - count).type = TType.FUNC_NAME;
 571							//Go ahead and remove the whitespace here too, it breaks things
 572							count--;
 573							for (int a = 0; a < count; a++) {
 574								token_list.remove(token_list.size() - 1);
 575							}
 576						} else {
 577							token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
 578						}
 579					} catch (IndexOutOfBoundsException e) {
 580						//This is the first element on the list, so, it's another autoconcat.
 581						token_list.add(new Token(TType.FUNC_NAME, "__autoconcat__", target));
 582					}
 583				}
 584				token_list.add(new Token(TType.FUNC_START, "(", target));
 585				continue;
 586			}
 587			if (c == ')' && !state_in_quote) {
 588				if (buf.length() > 0) {
 589					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 590					buf = new StringBuilder();
 591				}
 592				token_list.add(new Token(TType.FUNC_END, ")", target));
 593				continue;
 594			}
 595			if(c == ';' && !state_in_quote){
 596				if(buf.length() > 0){
 597					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 598					buf = new StringBuilder();
 599				}
 600				token_list.add(new Token(TType.SEMICOLON, ";", target));
 601				continue;
 602			}
 603			if (Character.isWhitespace(c) && !state_in_quote && c != '\n') {
 604				//keep the whitespace, but end the previous token, unless the last character
 605				//was also whitespace. All whitespace is added as a single space.
 606				if (buf.length() > 0) {
 607					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 608					buf = new StringBuilder();
 609				}
 610				if (token_list.size() > 0
 611						&& token_list.get(token_list.size() - 1).type != TType.WHITESPACE) {
 612					token_list.add(new Token(TType.WHITESPACE, " ", target));
 613				}
 614				continue;
 615			}
 616			if (c == '\'') {
 617				if (state_in_quote && !in_smart_quote) {
 618					token_list.add(new Token(TType.STRING, buf.toString(), target));
 619					buf = new StringBuilder();
 620					state_in_quote = false;
 621					continue;
 622				} else if (!state_in_quote) {
 623					state_in_quote = true;
 624					quoteLineNumberStart = line_num;
 625					in_smart_quote = false;
 626					if (buf.length() > 0) {
 627						token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 628						buf = new StringBuilder();
 629					}
 630					continue;
 631				} else {
 632					//we're in a smart quote
 633					buf.append("'");
 634				}
 635			} else if (c == '"') {
 636				if (state_in_quote && in_smart_quote) {
 637					token_list.add(new Token(TType.SMART_STRING, buf.toString(), target));
 638					buf = new StringBuilder();
 639					state_in_quote = false;
 640					in_smart_quote = false;
 641					continue;
 642				} else if (!state_in_quote) {
 643					state_in_quote = true;
 644					in_smart_quote = true;
 645					smartQuoteLineNumberStart = line_num;
 646					if (buf.length() > 0) {
 647						token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 648						buf = new StringBuilder();
 649					}
 650					continue;
 651				} else {
 652					//we're in normal quotes
 653					buf.append('"');
 654				}
 655			} else if (c == '\\') {
 656				//escaped characters
 657				if (state_in_quote) {
 658					if (c2 == '\\') {
 659						buf.append("\\");
 660					} else if (c2 == '\'') {
 661						buf.append("'");
 662					} else if (c2 == '"') {
 663						buf.append('"');
 664					} else if (c2 == 'n') {
 665						buf.append("\n");
 666					} else if (c2 == 'r'){
 667						buf.append("\r");
 668					} else if(c2 == 't'){
 669						buf.append("\t");
 670					} else if(c2 == '@' && in_smart_quote){
 671						buf.append("\\@");
 672					} else if (c2 == 'u') {
 673						//Grab the next 4 characters, and check to see if they are numbers
 674						StringBuilder unicode = new StringBuilder();
 675						for (int m = 0; m < 4; m++) {
 676							unicode.append(script.charAt(i + 2 + m));
 677						}
 678						try {
 679							Integer.parseInt(unicode.toString(), 16);
 680						} catch (NumberFormatException e) {
 681							throw new ConfigCompileException("Unrecognized unicode escape sequence", target);
 682						}
 683						buf.append(Character.toChars(Integer.parseInt(unicode.toString(), 16)));
 684						i += 4;
 685					} else {
 686						//Since we might expand this list later, don't let them
 687						//use unescaped backslashes
 688						throw new ConfigCompileException("The escape sequence \\" + c2 + " is not a recognized escape sequence", target);
 689					}
 690
 691					i++;
 692					continue;
 693				} else {
 694					//Control character backslash
 695					token_list.add(new Token(TType.SEPERATOR, "\\", target));
 696				}
 697			} else if (state_in_quote) {
 698				buf.append(c);
 699				continue;
 700			} else if (c == '\n' && !comment_is_block) {
 701				if (buf.length() > 0) {
 702					token_list.add(new Token(TType.UNKNOWN, buf.toString(), target));
 703					buf = new StringBuilder();
 704				}
 705				token_list.add(new Token(TType.NEWLINE, "\n", target));
 706				in_comment = false;
 707				comment_is_block = false;
 708				continue;
 709			} else { //in a literal
 710				buf.append(c);
 711				continue;
 712			}
 713		} //end lexing
 714		if (state_in_quote) {
 715			if (in_smart_quote) {
 716				throw new ConfigCompileException("Unended string literal. You started the last double quote on line " + smartQuoteLineNumberStart, target);
 717			} else {
 718				throw new ConfigCompileException("Unended string literal. You started the last single quote on line " + quoteLineNumberStart, target);
 719			}
 720		}
 721		if (in_comment || comment_is_block) {
 722			throw new ConfigCompileException("Unended block comment. You started the comment on line " + commentLineNumberStart, target);
 723		}
 724		//look at the tokens, and get meaning from them. Also, look for improper symbol locations,
 725		//and go ahead and absorb unary +- into the token
 726		for (int i = 0; i < token_list.size(); i++) {
 727			Token t = token_list.get(i);
 728			Token prev2 = i - 2 >= 0 ? token_list.get(i - 2) : new Token(TType.UNKNOWN, "", t.target);
 729			Token prev1 = i - 1 >= 0 ? token_list.get(i - 1) : new Token(TType.UNKNOWN, "", t.target);
 730			Token next = i + 1 < token_list.size() ? token_list.get(i + 1) : new Token(TType.UNKNOWN, "", t.target);
 731
 732			if (t.type == TType.UNKNOWN && prev1.type.isPlusMinus() && !prev2.type.isIdentifier()
 733					&& !prev2.type.equals(TType.FUNC_END)
 734					&& !t.val().matches("(\\@|\\$)[a-zA-Z0-9_]+")) { // Last boolean makes -@b equal to - @b, instead of a string.
 735				//It is a negative/positive number. Absorb the sign
 736				t.value = prev1.value + t.value;
 737				token_list.remove(i - 1);
 738				i--;
 739			}
 740
 741			if (t.type.equals(TType.UNKNOWN)) {
 742				if (t.val().charAt(0) == '/' && t.val().length() > 1) {
 743					t.type = TType.COMMAND;
 744				} else if (VAR_PATTERN.matcher(t.val()).matches()) {
 745					t.type = TType.VARIABLE;
 746				} else if (IVAR_PATTERN.matcher(t.val()).matches()) {
 747					t.type = TType.IVARIABLE;
 748				} else if (t.val().charAt(0) == '@') {
 749					throw new ConfigCompileException("IVariables must match the regex: @[a-zA-Z0-9_]+", target);
 750				} else if (t.val().equals("$")) {
 751					t.type = TType.FINAL_VAR;
 752				} else if(keywords.contains(t.val())){
 753					t.type = TType.KEYWORD;
 754				} else {
 755					t.type = TType.LIT;
 756				}
 757			}
 758			//Skip this check if we're not in pure mscript
 759			if(inPureMScript){
 760				if (t.type.isSymbol() && !t.type.isUnary() && !next.type.isUnary()) {
 761					if (prev1.type.equals(TType.FUNC_START) || prev1.type.equals(TType.COMMA)
 762							|| next.type.equals(TType.FUNC_END) || next.type.equals(TType.COMMA)
 763							|| prev1.type.isSymbol() || next.type.isSymbol()) {
 764						throw new ConfigCompileException("Unexpected symbol (" + t.val() + ")", t.getTarget());
 765					}
 766				}
 767			}
 768
 769		}
 770		return token_list;
 771	}
 772
 773	/**
 774	 * This function breaks the token stream into parts, separating the
 775	 * aliases/MethodScript from the command triggers
 776	 *
 777	 * @param tokenStream
 778	 * @return
 779	 * @throws ConfigCompileException
 780	 */
 781	public static List<Script> preprocess(List<Token> tokenStream) throws ConfigCompileException {
 782		if(tokenStream == null || tokenStream.isEmpty()){
 783			return new ArrayList<>();
 784		}
 785		//First, pull out the duplicate newlines
 786		ArrayList<Token> temp = new ArrayList<>();
 787		for (int i = 0; i < tokenStream.size(); i++) {
 788			try {
 789				if (tokenStream.get(i).type.equals(TType.NEWLINE)) {
 790					temp.add(new Token(TType.NEWLINE, "\n", tokenStream.get(i).target));
 791					while (tokenStream.get(++i).type.equals(TType.NEWLINE)) {
 792					}
 793				}
 794				if (tokenStream.get(i).type != TType.WHITESPACE) {
 795					temp.add(tokenStream.get(i));
 796				}
 797			} catch (IndexOutOfBoundsException e) {
 798			}
 799		}
 800
 801		if (temp.size() > 0 && temp.get(0).type.equals(TType.NEWLINE)) {
 802			temp.remove(0);
 803		}
 804
 805		tokenStream = temp;
 806
 807		//Handle multiline constructs
 808		ArrayList<Token> tokens1_1 = new ArrayList<>();
 809		boolean inside_multiline = false;
 810		Token thisToken = null;
 811		for (int i = 0; i < tokenStream.size(); i++) {
 812			Token prevToken = i - 1 >= tokenStream.size() ? tokenStream.get(i - 1) : new Token(TType.UNKNOWN, "", Target.UNKNOWN);
 813			thisToken = tokenStream.get(i);
 814			Token nextToken = i + 1 < tokenStream.size() ? tokenStream.get(i + 1) : new Token(TType.UNKNOWN, "", Target.UNKNOWN);
 815			//take out newlines between the = >>> and <<< tokens (also the tokens)
 816			if (thisToken.type.equals(TType.ALIAS_END) && nextToken.val().equals(">>>")) {
 817				inside_multiline = true;
 818				tokens1_1.add(thisToken);
 819				i++;
 820				continue;
 821			}
 822			if (thisToken.val().equals("<<<")) {
 823				if (!inside_multiline) {
 824					throw new ConfigCompileException("Found multiline end symbol, and no multiline start found",
 825							thisToken.target);
 826				}
 827				inside_multiline = false;
 828				continue;
 829			}
 830			if (thisToken.val().equals(">>>") && inside_multiline) {
 831				throw new ConfigCompileException("Did not expect a multiline start symbol here, are you missing a multiline end symbol above this line?", thisToken.target);
 832			}
 833			if (thisToken.val().equals(">>>") && !prevToken.type.equals(TType.ALIAS_END)) {
 834				throw new ConfigCompileException("Multiline symbol must follow the alias_end (=) symbol", thisToken.target);
 835			}
 836
 837			//If we're not in a multiline construct, or we are in it and it's not a newline, add
 838			//it
 839			if (!inside_multiline || !thisToken.type.equals(TType.NEWLINE)) {
 840				tokens1_1.add(thisToken);
 841			}
 842		}
 843
 844		assert thisToken != null;
 845
 846		if (inside_multiline) {
 847			throw new ConfigCompileException("Expecting a multiline end symbol, but your last multiline alias appears to be missing one.", thisToken.target);
 848		}
 849
 850		//take out newlines that are behind a \
 851		ArrayList<Token> tokens2 = new ArrayList<>();
 852		for (int i = 0; i < tokens1_1.size(); i++) {
 853			if (!tokens1_1.get(i).type.equals(TType.STRING) && tokens1_1.get(i).val().equals("\\") && tokens1_1.size() > i
 854					&& tokens1_1.get(i + 1).type.equals(TType.NEWLINE)) {
 855				tokens2.add(tokens1_1.get(i));
 856				i++;
 857				continue;
 858			}
 859			tokens2.add(tokens1_1.get(i));
 860		}
 861
 862		//Now that we have all lines minified, we should be able to split
 863		//on newlines, and easily find the left and right sides
 864
 865		List<Token> left = new ArrayList<>();
 866		List<Token> right = new ArrayList<>();
 867		List<Script> scripts = new ArrayList<>();
 868		boolean inLeft = true;
 869		for (Token t : tokens2) {
 870			if (inLeft) {
 871				if (t.type == TType.ALIAS_END) {
 872					inLeft = false;
 873				} else {
 874					left.add(t);
 875				}
 876			} else {
 877				if (t.type == TType.NEWLINE) {
 878					inLeft = true;
 879					// Check for spurious symbols, which indicate an issue with the
 880					// script, but ignore any whitespace.
 881					for(int j = left.size() - 1; j >= 0; j--){
 882						if(left.get(j).type == TType.NEWLINE){
 883							if(j > 0 && left.get(j - 1).type != TType.WHITESPACE){
 884								throw new ConfigCompileException("Unexpected token: " + left.get(j - 1).val(), left.get(j - 1).getTarget());
 885							}
 886						}
 887					}
 888					Script s = new Script(left, right);
 889					scripts.add(s);
 890					left = new ArrayList<>();
 891					right = new ArrayList<>();
 892				} else {
 893					right.add(t);
 894				}
 895			}
 896		}
 897		return scripts;
 898	}
 899
 900	/**
 901	 * Compiles the token stream into a valid ParseTree. This also includes optimization
 902	 * and reduction.
 903	 * @param stream The token stream, as generated by {@link #lex(String, File, boolean) lex}
 904	 * @return A fully compiled, optimized, and reduced parse tree. If {@code stream} is
 905	 * null or empty, null is returned.
 906	 * @throws ConfigCompileException If the script contains syntax errors. Additionally,
 907	 * during optimization, certain methods may cause compile errors. Any function that
 908	 * can optimize static occurrences and throws a {@link ConfigRuntimeException} will
 909	 * have that exception converted to a ConfigCompileException.
 910	 */
 911	@SuppressWarnings("UnnecessaryContinue")
 912	public static ParseTree compile(List<Token> stream) throws ConfigCompileException, ConfigCompileGroupException {
 913		Set<ConfigCompileException> compilerErrors = new HashSet<>();
 914		if(stream == null || stream.isEmpty()){
 915			return null;
 916		}
 917		Target unknown;
 918		try {
 919			//Instead of using Target.UNKNOWN, we can at least set the file.
 920			unknown = new Target(0, stream.get(0).target.file(), 0);
 921		} catch (Exception e) {
 922			unknown = Target.UNKNOWN;
 923		}
 924
 925		List<Token> tempStream = new ArrayList<>(stream.size());
 926		for (Token t : stream) {
 927			if(!t.type.isWhitespace()){
 928				tempStream.add(t);
 929			}
 930		}
 931		stream = tempStream;
 932
 933		ParseTree tree = new ParseTree(fileOptions);
 934		tree.setData(CNull.NULL);
 935		Stack<ParseTree> parents = new Stack<>();
 936		/**
 937		 * constructCount is used to determine if we need to use autoconcat
 938		 * when reaching a FUNC_END. The previous constructs, if the count
 939		 * is greater than 1, will be moved down into an autoconcat.
 940		 */
 941		Stack<AtomicInteger> constructCount = new Stack<>();
 942		constructCount.push(new AtomicInteger(0));
 943		parents.push(tree);
 944
 945		tree.addChild(new ParseTree(new CFunction("__autoconcat__", unknown), fileOptions));
 946		parents.push(tree.getChildAt(0));
 947		tree = tree.getChildAt(0);
 948		constructCount.push(new AtomicInteger(0));
 949
 950		/**
 951		 * The array stack is used to keep track of the number
 952		 * of square braces in use.
 953		 */
 954		Stack<AtomicInteger> arrayStack = new Stack<>();
 955		arrayStack.add(new AtomicInteger(-1));
 956		
 957		Stack<AtomicInteger> minusArrayStack = new Stack<>();
 958		Stack<AtomicInteger> minusFuncStack = new Stack<>();
 959		
 960		int parens = 0;
 961		Token t = null;
 962
 963		int bracketCount = 0;
 964
 965		for (int i = 0; i < stream.size(); i++) {
 966			t = stream.get(i);
 967			//Token prev2 = i - 2 >= 0 ? stream.get(i - 2) : new Token(TType.UNKNOWN, "", t.target);
 968			Token prev1 = i - 1 >= 0 ? stream.get(i - 1) : new Token(TType.UNKNOWN, "", t.target);
 969			Token next1 = i + 1 < stream.size() ? stream.get(i + 1) : new Token(TType.UNKNOWN, "", t.target);
 970			Token next2 = i + 2 < stream.size() ? stream.get(i + 2) : new Token(TType.UNKNOWN, "", t.target);
 971			Token next3 = i + 3 < stream.size() ? stream.get(i + 3) : new Token(TType.UNKNOWN, "", t.target);
 972
 973			// Brace handling
 974			if(t.type == TType.LCURLY_BRACKET){
 975				ParseTree b = new ParseTree(new CFunction("__cbrace__", t.getTarget()), fileOptions);
 976				tree.addChild(b);
 977				tree = b;
 978				parents.push(b);
 979				bracketCount++;
 980				constructCount.push(new AtomicInteger(0));
 981				continue;
 982			}
 983
 984			if(t.type == TType.RCURLY_BRACKET){
 985				bracketCount--;
 986				if (constructCount.peek().get() > 1) {
 987					//We need to autoconcat some stuff
 988					int stacks = constructCount.peek().get();
 989					int replaceAt = tree.getChildren().size() - stacks;
 990					ParseTree c = new ParseTree(new CFunction("__autoconcat__", tree.getTarget()), fileOptions);
 991					List<ParseTree> subChildren = new ArrayList<>();
 992					for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
 993						subChildren.add(tree.getChildAt(b));
 994					}
 995					c.setChildren(subChildren);
 996					if (replaceAt > 0) {
 997						List<ParseTree> firstChildren = new ArrayList<>();
 998						for (int d = 0; d < replaceAt; d++) {
 999							firstChildren.add(tree.getChildAt(d));
1000						}
1001						tree.setChildren(firstChildren);
1002					} else {
1003						tree.removeChildren();
1004					}
1005					tree.addChild(c);
1006				}
1007				parents.pop();
1008				tree = parents.peek();
1009				constructCount.pop();
1010				try {
1011					constructCount.peek().incrementAndGet();
1012				} catch (EmptyStackException e) {
1013					throw new ConfigCompileException("Unexpected end curly brace", t.target);
1014				}
1015				continue;
1016			}
1017
1018			//Associative array/label handling
1019			if(t.type == TType.LABEL && tree.getChildren().size() > 0){
1020				//If it's not an atomic identifier it's an error.
1021				if(!prev1.type.isAtomicLit() && prev1.type != TType.IVARIABLE && prev1.type != TType.KEYWORD){
1022					ConfigCompileException error = new ConfigCompileException("Invalid label specified", t.getTarget());
1023					if(prev1.type == TType.FUNC_END){
1024						// This is a fairly common mistake, so we have special handling for this,
1025						// because otherwise we would get a "Mismatched parenthesis" warning (which doesn't make sense),
1026						// and potentially lots of other invalid errors down the line, so we go ahead
1027						// and stop compilation at this point.
1028						throw error;
1029					}
1030					compilerErrors.add(error);
1031				}
1032				// Wrap previous construct in a CLabel
1033				ParseTree cc = tree.getChildren().get(tree.getChildren().size() - 1);
1034				tree.removeChildAt(tree.getChildren().size() - 1);
1035				tree.addChild(new ParseTree(new CLabel(cc.getData()), fileOptions));
1036				continue;
1037			}
1038
1039			//Array notation handling
1040			if (t.type.equals(TType.LSQUARE_BRACKET)) {
1041				arrayStack.push(new AtomicInteger(tree.getChildren().size() - 1));
1042				continue;
1043			} else if (t.type.equals(TType.RSQUARE_BRACKET)) {
1044				boolean emptyArray = false;
1045				if (prev1.type.equals(TType.LSQUARE_BRACKET)) {
1046					emptyArray = true;
1047				}
1048				if (arrayStack.size() == 1) {
1049					throw new ConfigCompileException("Mismatched square bracket", t.target);
1050				}
1051				//array is the location of the array
1052				int array = arrayStack.pop().get();
1053				//index is the location of the first node with the index
1054				int index = array + 1;
1055				if (!tree.hasChildren()) {
1056					throw new ConfigCompileException("Brackets are illegal here", t.target);
1057				}
1058				ParseTree myArray = tree.getChildAt(array);
1059				ParseTree myIndex;
1060				if (!emptyArray) {
1061					myIndex = new ParseTree(new CFunction("__autoconcat__", myArray.getTarget()), fileOptions);
1062
1063					for (int j = index; j < tree.numberOfChildren(); j++) {
1064						myIndex.addChild(tree.getChildAt(j));
1065					}
1066				} else {
1067					myIndex = new ParseTree(new CSlice("0..-1", t.target), fileOptions);
1068				}
1069				tree.setChildren(tree.getChildren().subList(0, array));
1070				ParseTree arrayGet = new ParseTree(new CFunction("array_get", t.target), fileOptions);
1071				arrayGet.addChild(myArray);
1072				arrayGet.addChild(myIndex);
1073				
1074				// Check if the @var[...] had a negating "-" in front. If so, add a neg().
1075				if (minusArrayStack.size() != 0 && arrayStack.size() + 1 == minusArrayStack.peek().get()) {
1076					if (!next1.type.equals(TType.LSQUARE_BRACKET)) { // Wait if there are more array_get's comming.
1077						ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
1078						negTree.addChild(arrayGet);
1079						tree.addChild(negTree);
1080						minusArrayStack.pop();
1081					} else {
1082						// Negate the next array_get instead, so just add this one to the tree.
1083						tree.addChild(arrayGet);
1084					}
1085				} else {
1086					tree.addChild(arrayGet);
1087				}
1088				constructCount.peek().set(constructCount.peek().get() - myIndex.numberOfChildren());
1089				continue;
1090			}
1091
1092			//Smart strings
1093			if (t.type == TType.SMART_STRING) {
1094				if(t.val().contains("@")) {
1095					ParseTree function = new ParseTree(fileOptions);
1096					function.setData(new CFunction(new Compiler.smart_string().getName(), t.target));
1097					ParseTree string = new ParseTree(fileOptions);
1098					string.setData(new CString(t.value, t.target));
1099					function.addChild(string);
1100					tree.addChild(function);
1101				} else {
1102					tree.addChild(new ParseTree(new CString(t.val(), t.target), fileOptions));
1103				}
1104				constructCount.peek().incrementAndGet();
1105				continue;
1106			}
1107
1108			if (t.type == TType.DEREFERENCE) {
1109				//Currently unimplemented, but going ahead and making it strict
1110				compilerErrors.add(new ConfigCompileException("The '" + t.val() + "' symbol is not currently allowed in raw strings. You must quote all"
1111						+ " symbols.", t.target));
1112			}
1113
1114			if (t.type.equals(TType.FUNC_NAME)) {
1115				CFunction func = new CFunction(t.val(), t.target);
1116				ParseTree f = new ParseTree(func, fileOptions);
1117				tree.addChild(f);
1118				constructCount.push(new AtomicInteger(0));
1119				tree = f;
1120				parents.push(f);
1121			} else if (t.type.equals(TType.FUNC_START)) {
1122				if (!prev1.type.equals(TType.FUNC_NAME)) {
1123					throw new ConfigCompileException("Unexpected parenthesis", t.target);
1124				}
1125				parens++;
1126			} else if (t.type.equals(TType.FUNC_END)) {
1127				if (parens <= 0) {
1128					throw new ConfigCompileException("Unexpected parenthesis", t.target);
1129				}
1130				parens--;
1131				ParseTree function = parents.pop();
1132				if (constructCount.peek().get() > 1) {
1133					//We need to autoconcat some stuff
1134					int stacks = constructCount.peek().get();
1135					int replaceAt = tree.getChildren().size() - stacks;
1136					ParseTree c = new ParseTree(new CFunction("__autoconcat__", tree.getTarget()), fileOptions);
1137					List<ParseTree> subChildren = new ArrayList<>();
1138					for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
1139						subChildren.add(tree.getChildAt(b));
1140					}
1141					c.setChildren(subChildren);
1142					if (replaceAt > 0) {
1143						List<ParseTree> firstChildren = new ArrayList<>();
1144						for (int d = 0; d < replaceAt; d++) {
1145							firstChildren.add(tree.getChildAt(d));
1146						}
1147						tree.setChildren(firstChildren);
1148					} else {
1149						tree.removeChildren();
1150					}
1151					tree.addChild(c);
1152				}
1153				constructCount.pop();
1154				try {
1155					constructCount.peek().incrementAndGet();
1156				} catch (EmptyStackException e) {
1157					throw new ConfigCompileException("Unexpected end parenthesis", t.target);
1158				}
1159				try {
1160					tree = parents.peek();
1161				} catch (EmptyStackException e) {
1162					throw new ConfigCompileException("Unexpected end parenthesis", t.target);
1163				}
1164				
1165				// Handle "-func(args)" and "-func(args)[index]".
1166				if (minusFuncStack.size() != 0 && minusFuncStack.peek().get() == parens + 1) {
1167					if(next1.type.equals(TType.LSQUARE_BRACKET)) {
1168						// Move the negation to the array_get which contains this function.
1169						minusArrayStack.push(new AtomicInteger(arrayStack.size() + 1)); // +1 because the bracket isn't counted yet.
1170					} else {
1171						// Negate this function.
1172						ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
1173						negTree.addChild(tree.getChildAt(tree.numberOfChildren() - 1));
1174						tree.removeChildAt(tree.numberOfChildren() - 1);
1175						tree.addChildAt(tree.numberOfChildren(), negTree);
1176					}
1177					minusFuncStack.pop();
1178				}
1179				
1180			} else if (t.type.equals(TType.COMMA)) {
1181				if (constructCount.peek().get() > 1) {
1182					int stacks = constructCount.peek().get();
1183					int replaceAt = tree.getChildren().size() - stacks;
1184					ParseTree c = new ParseTree(new CFunction("__autoconcat__", unknown), fileOptions);
1185					List<ParseTree> subChildren = new ArrayList<>();
1186					for (int b = replaceAt; b < tree.numberOfChildren(); b++) {
1187						subChildren.add(tree.getChildAt(b));
1188					}
1189					c.setChildren(subChildren);
1190					if (replaceAt > 0) {
1191						List<ParseTree> firstChildren = new ArrayList<>();
1192						for (int d = 0; d < replaceAt; d++) {
1193							firstChildren.add(tree.getChildAt(d));
1194						}
1195						tree.setChildren(firstChildren);
1196					} else {
1197						tree.removeChildren();
1198					}
1199					tree.addChild(c);
1200				}
1201				constructCount.peek().set(0);
1202				continue;
1203			}
1204			if(t.type == TType.SLICE){
1205				//We got here because the previous token isn't being ignored, because it's
1206				//actually a control character, instead of whitespace, but this is a
1207				//"empty first" slice notation. Compare this to the code below.
1208				try{
1209					CSlice slice;
1210					String value = next1.val();
1211					if(next1.type == TType.MINUS || next1.type == TType.PLUS){
1212						value = next1.val() + next2.val();
1213						i++;
1214					}
1215					slice = new CSlice(".." + value, t.getTarget());
1216					i++;
1217					tree.addChild(new ParseTree(slice, fileOptions));
1218					constructCount.peek().incrementAndGet();
1219					continue;
1220				} catch(ConfigRuntimeException ex){
1221					//CSlice can throw CREs, but at this stage, we have to
1222					//turn them into a CCE.
1223					throw new ConfigCompileException(ex);
1224				}
1225			}
1226			if (next1.type.equals(TType.SLICE)) {
1227				//Slice notation handling
1228				try {
1229					CSlice slice;
1230					if (t.type.isSeparator() || (t.type.isWhitespace() && prev1.type.isSeparator()) || t.type.isKeyword()) {
1231						//empty first
1232						String value = next2.val();
1233						i++;
1234						if(next2.type == TType.MINUS || next2.type == TType.PLUS){
1235							value = next2.val() + next3.val();
1236							i++;
1237						}
1238						slice = new CSlice(".." + value, next1.getTarget());
1239						if(t.type.isKeyword()){
1240							tree.addChild(new ParseTree(new CKeyword(t.val(), t.getTarget()), fileOptions));
1241							constructCount.peek().incrementAndGet();
1242						}
1243					} else if (next2.type.isSeparator() || next2.type.isKeyword()) {
1244						//empty last
1245						String modifier = "";
1246						if(prev1.type == TType.MINUS || prev1.type == TType.PLUS){
1247							//The negative would have already been inserted into the tree
1248							modifier = prev1.val();
1249							tree.removeChildAt(tree.getChildren().size() - 1);
1250						}
1251						slice = new CSlice(modifier + t.value + "..", t.target);
1252					} else {
1253						//both are provided
1254						String modifier1 = "";
1255						if(prev1.type == TType.MINUS || prev1.type == TType.PLUS){
1256							//It's a negative, incorporate that here, and remove the
1257							//minus from the tree
1258							modifier1 = prev1.val();
1259							tree.removeChildAt(tree.getChildren().size() - 1);
1260						}
1261						Token first = t;
1262						if(first.type.isWhitespace()){
1263							first = prev1;
1264						}
1265						Token second = next2;
1266						i++;
1267						String modifier2 = "";
1268						if(next2.type == TType.MINUS || next2.type == TType.PLUS){
1269							modifier2 = next2.val();
1270							second = next3;
1271							i++;
1272						}
1273						slice = new CSlice(modifier1 + first.value + ".." + modifier2 + second.value, t.target);
1274					}
1275					i++;
1276					tree.addChild(new ParseTree(slice, fileOptions));
1277					constructCount.peek().incrementAndGet();
1278					continue;
1279				} catch(ConfigRuntimeException ex){
1280					//CSlice can throw CREs, but at this stage, we have to
1281					//turn them into a CCE.
1282					throw new ConfigCompileException(ex);
1283				}
1284			} else if (t.type == TType.LIT) {
1285				Construct c = Static.resolveConstruct(t.val(), t.target);
1286				if(c instanceof CString && fileOptions.isStrict()){
1287					compilerErrors.add(new ConfigCompileException("Bare strings are not allowed in strict mode", t.target));
1288				} else if(c instanceof CInt && next1.type == TType.DOT && next2.type == TType.LIT) {
1289					// make CDouble here because otherwise Long.parseLong() will remove
1290					// minus zero before decimals and leading zeroes after decimals
1291					try {
1292						c = new CDouble(Double.parseDouble(t.val() + '.' + next2.val()), t.target);
1293						i += 2;
1294					} catch (NumberFormatException e) {
1295						// Not a double
1296					}
1297				}
1298				tree.addChild(new ParseTree(c, fileOptions));
1299				constructCount.peek().incrementAndGet();
1300			} else if (t.type.equals(TType.STRING) || t.type.equals(TType.COMMAND)) {
1301				tree.addChild(new ParseTree(new CString(t.val(), t.target), fileOptions));
1302				constructCount.peek().incrementAndGet();
1303			} else if (t.type.equals(TType.IDENTIFIER)) {
1304				tree.addChild(new ParseTree(new CPreIdentifier(t.val(), t.target), fileOptions));
1305				constructCount.peek().incrementAndGet();
1306			} else if(t.type.isKeyword()){
1307				tree.addChild(new ParseTree(new CKeyword(t.val(), t.getTarget()), fileOptions));
1308				constructCount.peek().incrementAndGet();
1309			} else if (t.type.equals(TType.IVARIABLE)) {
1310				tree.addChild(new ParseTree(new IVariable(t.val(), t.target), fileOptions));
1311				constructCount.peek().incrementAndGet();
1312			} else if (t.type.equals(TType.UNKNOWN)) {
1313				tree.addChild(new ParseTree(Static.resolveConstruct(t.val(), t.target), fileOptions));
1314				constructCount.peek().incrementAndGet();
1315			} else if (t.type.isSymbol()) { //Logic and math symbols
1316				
1317				// Attempt to find "-@var" and change it to "neg(@var)" if it's not @a - @b. Else just add the symbol.
1318				// Also handles "-function()" and "-@var[index]".
1319				if (t.type.equals(TType.MINUS) && !prev1.type.isAtomicLit() && !prev1.type.equals(TType.IVARIABLE)
1320						&& !prev1.type.equals(TType.VARIABLE) && !prev1.type.equals(TType.RCURLY_BRACKET)
1321						&& !prev1.type.equals(TType.RSQUARE_BRACKET) && !prev1.type.equals(TType.FUNC_END)
1322						&& (next1.type.equals(TType.IVARIABLE) || next1.type.equals(TType.VARIABLE) || next1.type.equals(TType.FUNC_NAME))) {
1323					
1324					// Check if we are negating a value from an array, function or variable.
1325					if (next2.type.equals(TType.LSQUARE_BRACKET)) {
1326						minusArrayStack.push(new AtomicInteger(arrayStack.size() + 1)); // +1 because the bracket isn't counted yet.
1327					} else if (next1.type.equals(TType.FUNC_NAME)) {
1328						minusFuncStack.push(new AtomicInteger(parens + 1)); // +1 because the function isn't counted yet.
1329					} else {
1330						ParseTree negTree = new ParseTree(new CFunction("neg", unknown), fileOptions);
1331						negTree.addChild(new ParseTree(new IVariable(next1.value, next1.target), fileOptions));
1332						tree.addChild(negTree);
1333						constructCount.peek().incrementAndGet();
1334						i++; // Skip the next variable as we've just handled it.
1335					}
1336				} else {
1337					tree.addChild(new ParseTree(new CSymbol(t.val(), t.type, t.target), fileOptions));
1338					constructCount.peek().incrementAndGet();
1339				}
1340				
1341			} else if (t.type == TType.DOT){
1342				// Check for doubles that start with a decimal, otherwise concat
1343				Construct c = null;
1344				if(next1.type == TType.LIT && prev1.type != TType.STRING && prev1.type != TType.SMART_STRING) {
1345					try {
1346						c = new CDouble(Double.parseDouble('.' + next1.val()), t.target);
1347						i++;
1348					} catch (NumberFormatException e) {
1349						// Not a double
1350					}
1351				}
1352				if(c == null) {
1353					c = new CSymbol(".", TType.CONCAT, t.target);
1354				}
1355				tree.addChild(new ParseTree(c, fileOptions));
1356				constructCount.peek().incrementAndGet();
1357			} else if (t.type.equals(TType.VARIABLE) || t.type.equals(TType.FINAL_VAR)) {
1358				tree.addChild(new ParseTree(new Variable(t.val(), null, false, t.type.equals(TType.FINAL_VAR), t.target), fileOptions));
1359				constructCount.peek().incrementAndGet();
1360				//right_vars.add(new Variable(t.val(), null, t.line_num));
1361			}
1362
1363		}
1364
1365		assert t != null;
1366
1367		if (arrayStack.size() != 1) {
1368			throw new ConfigCompileException("Mismatched square brackets", t.target);
1369		}
1370		if (parens != 0) {
1371			throw new ConfigCompileException("Mismatched parenthesis", t.target);
1372		}
1373		if (bracketCount != 0){
1374			throw new ConfigCompileException("Mismatched curly braces", t.target);
1375		}
1376

Large files files are truncated, but you can click here to view the full file