PageRenderTime 310ms CodeModel.GetById 61ms app.highlight 219ms RepoModel.GetById 12ms app.codeStats 1ms

/hphp/parser/xhpast2/parser.h

http://github.com/facebook/hiphop-php
C++ Header | 1610 lines | 1342 code | 149 blank | 119 comment | 113 complexity | 0544d1e2763078328b424a37eb8b7d54 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*
   2   +----------------------------------------------------------------------+
   3   | HipHop for PHP                                                       |
   4   +----------------------------------------------------------------------+
   5   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
   6   +----------------------------------------------------------------------+
   7   | This source file is subject to version 3.01 of the PHP license,      |
   8   | that is bundled with this package in the file LICENSE, and is        |
   9   | available through the world-wide-web at the following url:           |
  10   | http://www.php.net/license/3_01.txt                                  |
  11   | If you did not receive a copy of the PHP license and are unable to   |
  12   | obtain it through the world-wide-web, please send a note to          |
  13   | license@php.net so we can mail you a copy immediately.               |
  14   +----------------------------------------------------------------------+
  15*/
  16#ifndef incl_HPHP_PARSER_XHPAST2_H_
  17#define incl_HPHP_PARSER_XHPAST2_H_
  18
  19#include <stdexcept>
  20#include <iostream>
  21#include <cstdarg>
  22#include "folly/Format.h"
  23#include "folly/String.h"
  24#include "hphp/parser/parser.h"
  25#include "astnode.hpp"
  26
  27#define HPHP_PARSER_NS XHPAST2
  28
  29#define HPHP_PARSER_ERROR(fmt, p, args...)      \
  30  throw std::runtime_error(folly::format(       \
  31    "{}:{}:{}", (p)->file(), (p)->line1(),      \
  32      folly::stringPrintf(fmt, ##args)).str())
  33
  34namespace HPHP { namespace HPHP_PARSER_NS {
  35
  36extern bool g_verifyMode;
  37
  38//////////////////////////////////////////////////////////////////////
  39
  40struct ExtraInfo {
  41  virtual ~ExtraInfo() {}
  42};
  43
  44struct OnNameEI : ExtraInfo {
  45  HPHP::ParserBase::NameKind kind;
  46  explicit OnNameEI(HPHP::ParserBase::NameKind k) : kind(k) {};
  47};
  48
  49struct OnVariableEI : ExtraInfo {
  50  bool constant;
  51  const std::string& docComment;
  52  OnVariableEI(bool c, const std::string& dc) : constant(c), docComment(dc) {}
  53};
  54
  55struct OnDynamicVariableEI : ExtraInfo {
  56  bool encap;
  57  explicit OnDynamicVariableEI(bool e) : encap(e) {}
  58};
  59
  60struct OnCallParamEI : ExtraInfo {
  61  bool ref;
  62  explicit OnCallParamEI(bool r) : ref(r) {}
  63};
  64
  65struct OnCallEI : ExtraInfo {
  66  bool dynamic;
  67  bool fromCompiler;
  68  OnCallEI(bool d, bool f) : dynamic(d), fromCompiler(f) {}
  69};
  70
  71struct OnEncapsListEI : ExtraInfo {
  72  int type;
  73  explicit OnEncapsListEI(int t) : type(t) {}
  74};
  75
  76struct AddEncapEI : ExtraInfo {
  77  int type;
  78  explicit AddEncapEI(int t) : type(t) {}
  79};
  80
  81struct OnScalarEI : ExtraInfo {
  82  int type;
  83  explicit OnScalarEI(int t) : type(t) {}
  84};
  85
  86struct OnListAssignmentEI : ExtraInfo {
  87  bool rhsFirst;
  88  explicit OnListAssignmentEI(bool rhs) : rhsFirst(rhs) {}
  89};
  90
  91struct OnAssignEI : ExtraInfo {
  92  bool ref;
  93  bool rhsFirst;
  94  OnAssignEI(bool r, bool rhs) : ref(r), rhsFirst(rhs) {}
  95};
  96
  97struct OnUnaryOpExpEI : ExtraInfo {
  98  int op;
  99  bool front;
 100  OnUnaryOpExpEI(int o, bool f) : op(o), front(f) {}
 101};
 102
 103struct OnBinaryOpExpEI : ExtraInfo {
 104  int op;
 105  explicit OnBinaryOpExpEI(int o) : op(o) {}
 106};
 107
 108struct OnArrayEI : ExtraInfo {
 109  int op;
 110  explicit OnArrayEI(int o) : op(o) {}
 111};
 112
 113struct OnArrayPairEI : ExtraInfo {
 114  bool ref;
 115  explicit OnArrayPairEI(bool r) : ref(r) {}
 116};
 117
 118struct OnClassConstEI : ExtraInfo {
 119  bool text;
 120  explicit OnClassConstEI(bool t) : text(t) {}
 121};
 122
 123struct OnParamEI : ExtraInfo {
 124  bool ref;
 125  explicit OnParamEI(bool r) : ref(r) {}
 126};
 127
 128struct OnClassEI : ExtraInfo {
 129  int type;
 130  explicit OnClassEI(int t) : type(t) {}
 131};
 132
 133struct OnMethodEI : ExtraInfo {
 134  bool reloc;
 135  explicit OnMethodEI(bool r) : reloc(r) {}
 136};
 137
 138struct OnReturnEI : ExtraInfo {
 139  bool checkYield;
 140  explicit OnReturnEI(bool c) : checkYield(c) {}
 141};
 142
 143struct OnEchoEI : ExtraInfo {
 144  bool html;
 145  explicit OnEchoEI(bool h) : html(h) {}
 146};
 147
 148struct OnClosureEI : ExtraInfo {
 149  bool is_static;
 150  explicit OnClosureEI(bool i) : is_static(i) {}
 151};
 152
 153struct OnClosureParamEI : ExtraInfo {
 154  bool ref;
 155  explicit OnClosureParamEI(bool r) : ref(r) {}
 156};
 157
 158struct OnGotoEI : ExtraInfo {
 159  bool limited;
 160  explicit OnGotoEI(bool l) : limited(l) {}
 161};
 162
 163struct OnTypeSpecializationEI : ExtraInfo {
 164  char specialization;
 165  explicit OnTypeSpecializationEI(char s) : specialization(s) {}
 166};
 167
 168enum NodeType {
 169  RAW = 0,
 170  ONNAME = 1,
 171  ONVARIABLE = 2,
 172  ONSTATICVARIABLE = 3,
 173  ONCLASSVARIABLESTART = 4,
 174  ONCLASSVARIABLE = 5,
 175  ONCLASSCONSTANT = 6,
 176  ONSIMPLEVARIABLE = 7,
 177  ONSYNTHESIZEDVARIABLE = 8,
 178  ONDYNAMICVARIABLE = 9,
 179  ONINDIRECTREF = 10,
 180  ONSTATICMEMBER = 11,
 181  ONREFDIM = 12,
 182  ONCALLPARAM = 13,
 183  ONCALL = 14,
 184  ONENCAPSLIST = 15,
 185  ADDENCAP = 16,
 186  ENCAPREFDIM = 17,
 187  ENCAPOBJPROP = 18,
 188  ENCAPARRAY = 19,
 189  ONCONSTANTVALUE = 20,
 190  ONSCALAR = 21,
 191  ONEXPRLISTELEM = 22,
 192  ONOBJECTPROPERTY = 23,
 193  ONOBJECTMETHODCALL = 24,
 194  ONLISTASSIGNMENT = 25,
 195  ONALISTVAR = 26,
 196  ONALISTSUB = 27,
 197  ONASSIGN = 28,
 198  ONASSIGNNEW = 29,
 199  ONNEWOBJECT = 30,
 200  ONUNARYOPEXP = 31,
 201  ONBINARYOPEXP = 32,
 202  ONQOP = 33,
 203  ONARRAY = 34,
 204  ONARRAYPAIR = 35,
 205  ONCOLLECTIONPAIR = 36,
 206  ONUSERATTRIBUTE = 37,
 207  ONCLASSCONST = 38,
 208  ONFUNCTION = 39,
 209  ONPARAM = 40,
 210  ONCLASS = 41,
 211  ONINTERFACE = 42,
 212  ONINTERFACENAME = 43,
 213  ONTRAITUSE = 44,
 214  ONTRAITNAME = 45,
 215  ONTRAITRULE = 46,
 216  ONTRAITPRECRULE = 47,
 217  ONTRAITALIASRULESTART = 48,
 218  ONTRAITALIASRULEMODIFY = 49,
 219  ONMETHOD = 50,
 220  ONMEMBERMODIFIER = 51,
 221  ONSTATEMENTLISTSTART = 52,
 222  ADDSTATEMENT = 53,
 223  ONCLASSSTATEMENT = 54,
 224  FINISHSTATEMENT = 55,
 225  ONBLOCK = 56,
 226  ONIF = 57,
 227  ONELSEIF = 58,
 228  ONWHILE = 59,
 229  ONDO = 60,
 230  ONFOR = 61,
 231  ONSWITCH = 62,
 232  ONCASE = 63,
 233  ONBREAK = 64,
 234  ONCONTINUE = 65,
 235  ONRETURN = 66,
 236  ONYIELD = 67,
 237  ONYIELDBREAK = 68,
 238  ONGLOBAL = 69,
 239  ONGLOBALVAR = 70,
 240  ONSTATIC = 71,
 241  ONECHO = 72,
 242  ONUNSET = 73,
 243  ONEXPSTATEMENT = 74,
 244  ONFOREACH = 75,
 245  ONTRYCATCHFINALLY = 76,
 246  ONTRYFINALLY = 77,
 247  ONCATCH = 78,
 248  ONFINALLY = 79,
 249  ONTHROW = 80,
 250  ONCLOSURE = 81,
 251  ONCLOSUREPARAM = 82,
 252  ONLABEL = 83,
 253  ONGOTO = 84,
 254  ONTYPEDEF = 85,
 255  ONTYPEANNOTATION = 86,
 256  ONTYPELIST = 87,
 257  ONTYPESPECIALIZATION = 88,
 258  ONYIELDPAIR = 89,
 259  ONAWAIT = 90,
 260};
 261
 262struct Token : ScannerToken {
 263  // this parser implementation could leak memory and is unsuitable for long
 264  // running processes
 265  NodeType nodeType;
 266  ExtraInfo *extra;
 267  std::vector<Token *> children;
 268  Token() : nodeType(RAW),extra(nullptr) {}
 269  Token& operator=(int num) {
 270    ScannerToken::m_num = num;
 271    return *this;
 272  }
 273  Token& operator=(Token& other) {
 274    ScannerToken::operator=(other);
 275    nodeType = other.nodeType;
 276    extra = other.extra;
 277    children.clear();
 278    appendChildren(&other);
 279    return *this;
 280  }
 281  Token* operator->() {
 282    return this;
 283  }
 284  Token& operator+(const char* str) {
 285    ScannerToken::m_text += str;
 286    return *this;
 287  }
 288  Token& operator+(const Token& token) {
 289    ScannerToken::m_num += token.m_num;
 290    ScannerToken::m_text += token.m_text;
 291    return *this;
 292  }
 293  Token& appendChild(Token *token) {
 294    if (token) {
 295      Token *c = new Token();
 296      *c = *token;
 297      children.push_back(c);
 298    } else {
 299      children.push_back(token);
 300    }
 301    return *this;
 302  }
 303  Token& appendChildren(Token *token) {
 304    for (std::vector<Token *>::iterator i = token->children.begin();
 305         i != token->children.end(); ++i) {
 306      children.push_back(*i);
 307    }
 308    return *this;
 309  }
 310  Token& setNodeType(NodeType t) {
 311    nodeType = t;
 312    children.clear();
 313    extra = nullptr;
 314    return *this;
 315  }
 316  Token& setExtra(ExtraInfo *ei) {
 317    extra = ei;
 318    return *this;
 319  }
 320  friend std::ostream& operator<<(std::ostream& out, Token& t) {
 321    if (t.nodeType == RAW) {
 322      out << "RAW:" << t.m_text;
 323    } else {
 324      out << "[" << t.nodeType << ":" << t.m_num << ":" << t.m_id;
 325      if (t.children.size() > 0) {
 326        std::cout << " [";
 327        for (int i = 0; i < t.children.size(); i++) {
 328          Token *c = t.children[i];
 329          if (c) {
 330            out << *c;
 331          } else {
 332            out << "NULL";
 333          }
 334          if (i < t.children.size() - 1) {
 335            out << ",";
 336          }
 337        }
 338        std::cout << "]";
 339      }
 340      out << "]";
 341    }
 342    return out;
 343//    return out << t.m_text << ":" << t.m_num << ":" << t.m_id;
 344  }
 345};
 346
 347struct XHPASTTokenListener : HPHP::TokenListener {
 348  std::vector<xhpast::Token*> tokens;
 349  virtual int publish(const char *rawText, int rawLeng, int type) {
 350    // single chars like ':' with no explicit token type
 351    if ((type == -1) && (rawLeng == 1)) {
 352      type = (unsigned int)rawText[0];
 353    }
 354    tokens.push_back(
 355      new xhpast::Token((unsigned int)type,
 356                        const_cast<char*>(rawText),
 357                        (unsigned int)rawLeng)
 358    );
 359    return tokens.size() - 1;
 360  }
 361  friend std::ostream& operator<<(std::ostream& out, XHPASTTokenListener& l) {
 362    for (std::vector<xhpast::Token*>::iterator it = l.tokens.begin();
 363         it != l.tokens.end(); ++it) {
 364      out << "[" << (*it)->type << "," << (*it)->value.size() << "]";
 365    }
 366    return out;
 367  }
 368  ~XHPASTTokenListener() {
 369    for (std::vector<xhpast::Token*>::iterator it = tokens.begin();
 370         it != tokens.end(); ++it) {
 371//      std::cout << "[" << (*it)->value << "," << (*it)->type << "]";
 372      delete (*it);
 373    }
 374  }
 375};
 376
 377/*
 378 * Parser that creates an AST
 379 */
 380struct Parser : ParserBase {
 381  XHPASTTokenListener m_listener;
 382
 383  explicit Parser(Scanner& scanner,
 384                  const char* filename)
 385      : ParserBase(scanner, filename) {
 386    scanner.setListener(dynamic_cast<HPHP::TokenListener*>(&m_listener));
 387  }
 388
 389#define X(...)
 390//#define X(...) traceCb(__FUNCTION__,## __VA_ARGS__)
 391
 392  void fatal(const Location* loc, const char* msg) {
 393    throw std::runtime_error(folly::format(
 394      "{}:{}: {}", m_fileName, m_loc.line0, msg).str());
 395  }
 396
 397  void parseFatal(const Location* loc, const char* msg) {
 398    throw std::runtime_error(folly::format(
 399      "{}:{}: {}", m_fileName, m_loc.line0, msg).str());
 400  }
 401
 402  void parse() {
 403
 404    if (!parseImpl()) {
 405      error("parse failure: %s\n", getMessage().c_str());
 406    }
 407  }
 408
 409 public:
 410  bool parseImpl();
 411
 412  void error(const char* fmt, ...) {
 413    va_list ap;
 414    va_start(ap, fmt);
 415    std::string msg;
 416    Util::string_vsnprintf(msg, fmt, ap);
 417    va_end(ap);
 418
 419    throw std::runtime_error(folly::format(
 420      "{}:{}: {}", m_fileName, m_loc.line0, msg).str());
 421  }
 422
 423  void invalidateGoto(TStatementPtr, ParserBase::GotoError) { }
 424  void invalidateLabel(TStatementPtr) { }
 425  void* extractStatement(ScannerToken*) { return nullptr; }
 426
 427  bool enableFinallyStatement() { return true; }
 428
 429  IMPLEMENT_XHP_ATTRIBUTES;
 430
 431  Token tree;
 432
 433  void initParseTree() {}
 434  void finiParseTree() {}
 435
 436  void onName(Token& out, Token& name, NameKind kind) {
 437    out.setNodeType(ONNAME).setExtra(new OnNameEI(kind)).appendChild(&name);
 438  }
 439
 440  void onVariable(Token& out, Token* exprs, Token& var, Token* value,
 441                  bool constant = false,
 442                  const std::string& docComment = "") {
 443    out.setNodeType(ONVARIABLE).setExtra(new OnVariableEI(constant, docComment))
 444      .appendChild(exprs).appendChild(&var).appendChild(value);
 445  }
 446
 447  void onStaticVariable(Token &out, Token *exprs, Token &var, Token *value) {
 448    out.setNodeType(ONSTATICVARIABLE).appendChild(exprs).appendChild(&var)
 449      .appendChild(value);
 450  }
 451
 452  void onClassVariableModifer(Token &mod) { /* TODO */
 453    X(mod);
 454  }
 455
 456  void onClassVariableStart(Token &out, Token *modifiers, Token &decl,
 457                            Token *type) {
 458    out.setNodeType(ONCLASSVARIABLESTART).appendChild(modifiers)
 459      .appendChild(&decl).appendChild(type);
 460  }
 461
 462  void onClassVariable(Token &out, Token *exprs, Token &var, Token *value) {
 463    out.setNodeType(ONCLASSVARIABLE).appendChild(exprs).appendChild(&var)
 464      .appendChild(value);
 465  }
 466
 467  void onClassConstant(Token &out, Token *exprs, Token &var, Token &value) {
 468    out.setNodeType(ONCLASSCONSTANT).appendChild(exprs).appendChild(&var)
 469      .appendChild(&value);
 470  }
 471
 472  void onSimpleVariable(Token &out, Token &var) {
 473    out.setNodeType(ONSIMPLEVARIABLE).appendChild(&var);
 474  }
 475
 476  void onSynthesizedVariable(Token& out, Token &var) {
 477    out.setNodeType(ONSYNTHESIZEDVARIABLE).appendChild(&var);
 478  }
 479
 480  void onDynamicVariable(Token &out, Token &expr, bool encap) {
 481    out.setNodeType(ONDYNAMICVARIABLE).appendChild(&expr)
 482      .setExtra(new OnDynamicVariableEI(encap));
 483  }
 484
 485  void onIndirectRef(Token &out, Token &refCount, Token &var) {
 486    out.setNodeType(ONINDIRECTREF).appendChild(&refCount).appendChild(&var);
 487  }
 488
 489  void onStaticMember(Token &out, Token &cls, Token &name) {
 490    out.setNodeType(ONSTATICMEMBER).appendChild(&cls).appendChild(&name);
 491  }
 492
 493  void onRefDim(Token &out, Token &var, Token &offset) {
 494    out.setNodeType(ONREFDIM).appendChild(&var).appendChild(&offset);
 495  }
 496
 497  void onCallParam(Token &out, Token *params, Token &expr, bool ref) {
 498    out.setNodeType(ONCALLPARAM).appendChild(params).appendChild(&expr)
 499      .setExtra(new OnCallParamEI(ref));
 500  }
 501
 502  void onCall(Token &out, bool dynamic, Token &name, Token &params,
 503              Token *cls, bool fromCompiler = false) {
 504    out.setNodeType(ONCALL).appendChild(&name).appendChild(&params)
 505      .appendChild(cls).setExtra(new OnCallEI(dynamic, fromCompiler));
 506  }
 507
 508  void onEncapsList(Token &out, int type, Token &list) {
 509    out.setNodeType(ONENCAPSLIST).appendChild(&list)
 510      .setExtra(new OnEncapsListEI(type));
 511  }
 512
 513  void addEncap(Token &out, Token *list, Token &expr, int type) {
 514    out.setNodeType(ADDENCAP).appendChild(list).appendChild(&expr)
 515      .setExtra(new AddEncapEI(type));
 516  }
 517
 518  void encapRefDim(Token &out, Token &var, Token &offset) {
 519    out.setNodeType(ENCAPREFDIM).appendChild(&var).appendChild(&offset);
 520  }
 521
 522  void encapObjProp(Token &out, Token &var, Token &name) {
 523    out.setNodeType(ENCAPOBJPROP).appendChild(&var).appendChild(&name);
 524  }
 525
 526  void encapArray(Token &out, Token &var, Token &expr) {
 527    out.setNodeType(ENCAPARRAY).appendChild(&var).appendChild(&expr);
 528  }
 529
 530  void onConstantValue(Token &out, Token &constant) {
 531    out.setNodeType(ONCONSTANTVALUE).appendChild(&constant);
 532  }
 533
 534  void onScalar(Token &out, int type, Token &scalar) {
 535    out.setNodeType(ONSCALAR).appendChild(&scalar)
 536      .setExtra(new OnScalarEI(type));
 537  }
 538
 539  void onExprListElem(Token &out, Token *exprs, Token &expr) {
 540    out.setNodeType(ONEXPRLISTELEM).appendChild(exprs).appendChild(&expr);
 541  }
 542
 543  void onObjectProperty(Token &out, Token &base, Token &prop) {
 544    out.setNodeType(ONOBJECTPROPERTY).appendChild(&base).appendChild(&prop);
 545  }
 546
 547  void onObjectMethodCall(Token &out, Token &base, Token &prop,
 548                          Token &params) {
 549    out.setNodeType(ONOBJECTMETHODCALL).appendChild(&base).appendChild(&prop)
 550      .appendChild(&params);
 551  }
 552
 553  void onListAssignment(Token &out, Token &vars, Token *expr,
 554                        bool rhsFirst = false) {
 555    out.setNodeType(ONLISTASSIGNMENT).appendChild(&vars).appendChild(expr)
 556      .setExtra(new OnListAssignmentEI(rhsFirst));
 557  }
 558
 559  void onAListVar(Token &out, Token *list, Token *var) {
 560    out.setNodeType(ONALISTVAR).appendChild(list).appendChild(var);
 561  }
 562
 563  void onAListSub(Token &out, Token *list, Token &sublist) {
 564    out.setNodeType(ONALISTSUB).appendChild(list).appendChild(&sublist);
 565  }
 566
 567  void onAssign(Token& out, Token& var, Token& expr, bool ref,
 568                bool rhsFirst = false) {
 569    out.setNodeType(ONASSIGN).appendChild(&var).appendChild(&expr)
 570      .setExtra(new OnAssignEI(ref, rhsFirst));
 571  }
 572
 573  void onAssignNew(Token &out, Token &var, Token &name, Token &args) {
 574    out.setNodeType(ONASSIGNNEW).appendChild(&var).appendChild(&name)
 575      .appendChild(&args);
 576  }
 577
 578  void onNewObject(Token &out, Token &name, Token &args) {
 579    out.setNodeType(ONNEWOBJECT).appendChild(&name).appendChild(&args);
 580  }
 581
 582  void onUnaryOpExp(Token &out, Token &operand, int op, bool front) {
 583    out.setNodeType(ONUNARYOPEXP).appendChild(&operand)
 584      .setExtra(new OnUnaryOpExpEI(op, front));
 585  }
 586
 587  void onBinaryOpExp(Token &out, Token &operand1, Token &operand2, int op) {
 588    out.setNodeType(ONBINARYOPEXP).appendChild(&operand1).appendChild(&operand2)
 589      .setExtra(new OnBinaryOpExpEI(op));
 590  }
 591
 592  void onQOp(Token &out, Token &exprCond, Token *expYes, Token &expNo) {
 593    out.setNodeType(ONQOP).appendChild(&exprCond).appendChild(expYes)
 594      .appendChild(&expNo);
 595  }
 596
 597  void onArray(Token &out, Token &pairs, int op = T_ARRAY) {
 598    out.setNodeType(ONARRAY).appendChild(&pairs)
 599      .setExtra(new OnArrayEI(op));
 600  }
 601
 602  void onArrayPair(Token &out, Token *pairs, Token *name, Token &value,
 603                   bool ref) {
 604    out.setNodeType(ONARRAYPAIR).appendChild(pairs).appendChild(name)
 605      .appendChild(&value).setExtra(new OnArrayPairEI(ref));
 606  }
 607
 608  void onEmptyCollection(Token &out) { X(); /* TODO */}
 609
 610  void onCollectionPair(Token &out, Token *pairs, Token *name, Token &value) {
 611    out.setNodeType(ONCOLLECTIONPAIR).appendChild(pairs).appendChild(name)
 612      .appendChild(&value);
 613  }
 614
 615  void onUserAttribute(Token &out, Token *attrList, Token &name,
 616                       Token &value) {
 617    out.setNodeType(ONUSERATTRIBUTE).appendChild(attrList).appendChild(&name)
 618      .appendChild(&value);
 619  }
 620
 621  void onClassConst(Token &out, Token &cls, Token &name, bool text) {
 622    out.setNodeType(ONCLASSCONST).appendChild(&cls).appendChild(&name)
 623      .setExtra(new OnClassConstEI(text));
 624  }
 625
 626  void fixStaticVars() { /* TODO */}
 627
 628  void onFunctionStart(Token& name, bool doPushComment = true) {
 629    /* TODO */
 630  }
 631
 632  void onClosureStart(Token& name) {
 633    /* TODO */
 634  }
 635
 636  void onFunction(Token& out, Token *modifiers, Token& ret, Token& ref,
 637                  Token& name, Token& params, Token& stmt, Token* attr) {
 638    out.setNodeType(ONFUNCTION).appendChild(modifiers).appendChild(&ret)
 639      .appendChild(&ref).appendChild(&name).appendChild(&params)
 640      .appendChild(&stmt).appendChild(attr);
 641  }
 642
 643  void onParam(Token &out, Token *params, Token &type, Token &var,
 644               bool ref, Token *defValue, Token *attr, Token *mods) {
 645    out.setNodeType(ONPARAM).appendChild(params).appendChild(&type)
 646      .appendChild(&var).appendChild(defValue).appendChild(attr)
 647      .appendChild(mods).setExtra(new OnParamEI(ref));
 648  }
 649
 650  void onClassStart(int type, Token &name) {
 651    /* TODO */
 652  }
 653
 654  void onClass(Token &out, int type, Token &name, Token &base,
 655               Token &baseInterface, Token &stmt, Token *attr) {
 656    out.setNodeType(ONCLASS).appendChild(&name).appendChild(&base)
 657      .appendChild(&baseInterface).appendChild(&stmt).appendChild(attr)
 658      .setExtra(new OnClassEI(type));
 659  }
 660
 661  void onInterface(Token &out, Token &name, Token &base, Token &stmt,
 662                   Token *attr) {
 663    out.setNodeType(ONINTERFACE).appendChild(&name).appendChild(&base)
 664      .appendChild(&stmt).appendChild(attr);
 665  }
 666
 667  void onInterfaceName(Token &out, Token *names, Token &name) {
 668    out.setNodeType(ONINTERFACENAME).appendChild(names).appendChild(&name);
 669  }
 670
 671  void onTraitUse(Token &out, Token &traits, Token &rules) {
 672    out.setNodeType(ONTRAITUSE).appendChild(&traits).appendChild(&rules);
 673  }
 674
 675  void onTraitName(Token &out, Token *names, Token &name) {
 676    out.setNodeType(ONTRAITNAME).appendChild(names).appendChild(&name);
 677  }
 678
 679  void onTraitRule(Token &out, Token &stmtList, Token &newStmt) {
 680    out.setNodeType(ONTRAITRULE).appendChild(&stmtList).appendChild(&newStmt);
 681  }
 682
 683  void onTraitPrecRule(Token &out, Token &className, Token &methodName,
 684                       Token &otherClasses) {
 685    out.setNodeType(ONTRAITPRECRULE).appendChild(&className)
 686      .appendChild(&methodName).appendChild(&otherClasses);
 687  }
 688
 689  void onTraitAliasRuleStart(Token &out, Token &className, Token &methodName) {
 690    out.setNodeType(ONTRAITALIASRULESTART).appendChild(&className)
 691      .appendChild(&methodName);
 692  }
 693
 694  void onTraitAliasRuleModify(Token &out, Token &rule, Token &accessModifiers,
 695                              Token &newMethodName) {
 696    out.setNodeType(ONTRAITALIASRULEMODIFY).appendChild(&rule)
 697      .appendChild(&accessModifiers).appendChild(&newMethodName);
 698  }
 699
 700  void onMethodStart(Token &name, Token &mods, bool doPushComment = true) {
 701    /* TODO */
 702  }
 703
 704  void onMethod(Token &out, Token &modifiers, Token &ret, Token &ref,
 705                Token &name, Token &params, Token &stmt, Token *attr,
 706                bool reloc = true) {
 707    // modifiers could be garbage Tokens if no modifier specified.
 708    out.setNodeType(ONMETHOD).appendChild(&modifiers).appendChild(&ret)
 709      .appendChild(&ref).appendChild(&name).appendChild(&params)
 710      .appendChild(&stmt).appendChild(attr).setExtra(new OnMethodEI(reloc));
 711  }
 712
 713  void onMemberModifier(Token &out, Token *modifiers, Token &modifier) {
 714    out.setNodeType(ONMEMBERMODIFIER).appendChild(modifiers)
 715      .appendChild(&modifier);
 716  }
 717
 718  void onStatementListStart(Token &out) {
 719    out.setNodeType(ONSTATEMENTLISTSTART);
 720  }
 721
 722  void addTopStatement(Token &new_stmt) { this->tree = new_stmt; }
 723
 724  // TODO
 725  void onHaltCompiler() {}
 726
 727  void addStatement(Token& out, Token& stmts, Token& new_stmt) {
 728    out.setNodeType(ADDSTATEMENT).appendChild(&stmts).appendChild(&new_stmt);
 729  }
 730
 731  void onClassStatement(Token &out, Token &stmts, Token &new_stmt) {
 732    out.setNodeType(ONCLASSSTATEMENT).appendChild(&stmts)
 733      .appendChild(&new_stmt);
 734  }
 735
 736  void finishStatement(Token& out, Token& stmts) {
 737    out.setNodeType(FINISHSTATEMENT).appendChild(&stmts);
 738  }
 739
 740  void onBlock(Token& out, Token& stmts) {
 741    out.setNodeType(ONBLOCK).appendChild(&stmts);
 742  }
 743
 744  void onIf(Token &out, Token &cond, Token &stmt, Token &elseifs,
 745            Token &elseStmt) {
 746    out.setNodeType(ONIF).appendChild(&cond).appendChild(&stmt)
 747      .appendChild(&elseifs).appendChild(&elseStmt);
 748  }
 749
 750  void onElseIf(Token& out, Token& elseifs, Token& cond, Token& stmt) {
 751    out.setNodeType(ONELSEIF).appendChild(&elseifs).appendChild(&cond)
 752      .appendChild(&stmt);
 753  }
 754
 755  void onWhile(Token &out, Token &cond, Token &stmt) {
 756    out.setNodeType(ONWHILE).appendChild(&cond).appendChild(&stmt);
 757  }
 758
 759  void onDo(Token &out, Token &stmt, Token &cond) {
 760    out.setNodeType(ONDO).appendChild(&stmt).appendChild(&cond);
 761  }
 762
 763  void onFor(Token &out, Token &expr1, Token &expr2, Token &expr3,
 764             Token &stmt) {
 765    out.setNodeType(ONFOR).appendChild(&expr1).appendChild(&expr2)
 766      .appendChild(&expr3).appendChild(&stmt);
 767  }
 768
 769  void onSwitch(Token &out, Token &expr, Token &cases) {
 770    out.setNodeType(ONSWITCH).appendChild(&expr).appendChild(&cases);
 771  }
 772
 773  void onCase(Token &out, Token &cases, Token *cond, Token &stmt) {
 774    out.setNodeType(ONCASE).appendChild(&cases).appendChild(cond)
 775      .appendChild(&stmt);
 776  }
 777
 778  void onBreakContinue(Token &out, bool isBreak, Token *expr) {
 779    if (isBreak) {
 780      out.setNodeType(ONBREAK).appendChild(expr);
 781    } else {
 782      out.setNodeType(ONCONTINUE).appendChild(expr);
 783    }
 784  }
 785
 786  void onReturn(Token &out, Token *expr, bool checkYield = true) {
 787    out.setNodeType(ONRETURN).appendChild(expr)
 788      .setExtra(new OnReturnEI(checkYield));
 789  }
 790
 791  void onYield(Token &out, Token &expr) {
 792    out.setNodeType(ONYIELD).appendChild(&expr);
 793  }
 794
 795  void onYieldPair(Token &out, Token &key, Token &val) {
 796    out.setNodeType(ONYIELDPAIR).appendChild(&key).appendChild(&val);
 797  }
 798
 799  void onYieldBreak(Token &out) {
 800    out.setNodeType(ONYIELDBREAK);
 801  }
 802
 803  void onAwait(Token &out, Token &expr) {
 804    out.setNodeType(ONAWAIT).appendChild(&expr);
 805  }
 806
 807  void onGlobal(Token &out, Token &expr) {
 808    out.setNodeType(ONGLOBAL).appendChild(&expr);
 809  }
 810  void onGlobalVar(Token &out, Token *exprs, Token &expr) {
 811    out.setNodeType(ONGLOBALVAR).appendChild(exprs).appendChild(&expr);
 812  }
 813
 814  void onStatic(Token &out, Token &expr) {
 815    out.setNodeType(ONSTATIC).appendChild(&expr);
 816  }
 817  void onEcho(Token &out, Token &expr, bool html) {
 818    out.setNodeType(ONECHO).appendChild(&expr).setExtra(new OnEchoEI(html));
 819  }
 820  void onUnset(Token &out, Token &expr) {
 821    out.setNodeType(ONUNSET).appendChild(&expr);
 822  }
 823
 824  void onExpStatement(Token& out, Token& expr) {
 825    out.setNodeType(ONEXPSTATEMENT).appendChild(&expr);
 826  }
 827
 828  void onForEachStart() {/* TODO */}
 829  void onForEach(Token &out, Token &arr, Token &name, Token &value,
 830                 Token &stmt) {
 831    out.setNodeType(ONFOREACH).appendChild(&arr).appendChild(&name)
 832      .appendChild(&value).appendChild(&stmt);
 833  }
 834
 835  void onTry(Token &out, Token &tryStmt, Token &className, Token &var,
 836             Token &catchStmt, Token &catches, Token &finallyStmt) {
 837    out.setNodeType(ONTRYCATCHFINALLY).appendChild(&tryStmt)
 838      .appendChild(&className).appendChild(&var).appendChild(&catchStmt)
 839      .appendChild(&catches).appendChild(&finallyStmt);
 840  }
 841
 842  void onTry(Token &out, Token &tryStmt, Token &finallyStmt) {
 843    out.setNodeType(ONTRYFINALLY).appendChild(&tryStmt)
 844      .appendChild(&finallyStmt);
 845  }
 846
 847  void onCatch(Token &out, Token &catches, Token &className, Token &var,
 848               Token &stmt) {
 849    out.setNodeType(ONCATCH).appendChild(&catches).appendChild(&className)
 850      .appendChild(&var).appendChild(&stmt);
 851  }
 852
 853  void onFinally(Token &out, Token &stmt) {
 854    out.setNodeType(ONFINALLY).appendChild(&stmt);
 855  }
 856
 857  void onThrow(Token &out, Token &expr) {
 858    out.setNodeType(ONTHROW).appendChild(&expr);
 859  }
 860
 861  void onClosure(Token &out, Token *modifiers, Token &ret, Token &ref,
 862                 Token &params, Token &cparams, Token &stmts) {
 863    out.setNodeType(ONCLOSURE).appendChild(modifiers).appendChild(&ret)
 864      .appendChild(&ref).appendChild(&params).appendChild(&cparams)
 865      .appendChild(&stmts);
 866  }
 867
 868  void onClosureParam(Token &out, Token *params, Token &param, bool ref) {
 869    out.setNodeType(ONCLOSUREPARAM).appendChild(params).appendChild(&param)
 870      .setExtra(new OnClosureParamEI(ref));
 871  }
 872
 873  void onLabel(Token &out, Token &label) {
 874    out.setNodeType(ONLABEL).appendChild(&label);
 875  }
 876  void onGoto(Token &out, Token &label, bool limited) {
 877    out.setNodeType(ONGOTO).appendChild(&label).setExtra(new OnGotoEI(limited));
 878  }
 879  void onTypedef(Token& out, Token& name, Token& value) {
 880    out.setNodeType(ONTYPEDEF).appendChild(&name).appendChild(&value);
 881  }
 882  void onTypeAnnotation(Token& out, const Token& name, const Token& typeArgs) {
 883    out.setNodeType(ONTYPEANNOTATION).appendChild(const_cast<Token *>(&name))
 884      .appendChild(const_cast<Token *>(&typeArgs));
 885  }
 886  void onTypeList(Token& type1, const Token& type2) {
 887    // TODO: no out?
 888//    out.setNodeType(ONTYPELIST).appendChild(const_cast<Token *>(&type2));
 889  }
 890  void onTypeSpecialization(Token& type, char specialization) {
 891    // TODO: no out?
 892//    out.setNodeType(ONTYPESPECIALIZATION)
 893//      .setExtra(new OnTypeSpecializationEI(specialization));
 894  }
 895
 896  // for namespace support
 897  void onNamespaceStart(const std::string &ns, bool file_scope = false) {
 898    // TODO
 899  }
 900  void onNamespaceEnd() {}
 901  void onUse(const std::string &ns, const std::string &as) {
 902    // TODO
 903  }
 904  void nns(bool declare = false) {
 905    // TODO
 906  }
 907  std::string nsDecl(const std::string &name) {
 908    // TODO
 909    return name;
 910  }
 911  std::string resolve(const std::string &ns, bool cls) {
 912    // TODO
 913    return ns;
 914  }
 915
 916
 917
 918  /////////////////////////////////////////////////////////////////////////////
 919  // Functions below are used to output XHPAST
 920  /////////////////////////////////////////////////////////////////////////////
 921
 922  void coalesceTree() {
 923    coalesceTreeImpl(&tree);
 924  }
 925  // Collapse/elide certain nodes to avoid deep trees
 926  void coalesceTreeImpl(Token *node) {
 927    if (node != nullptr) {
 928      // copy children
 929      std::vector<Token *> children1(node->children);
 930      node->children.clear();
 931      for (std::vector<Token *>::iterator i = children1.begin();
 932           i < children1.end();
 933           ++i) {
 934        Token *child = *i;
 935        if (child && child->nodeType == ONSTATEMENTLISTSTART) {
 936          delete child; // no longer in the tree
 937          continue;
 938        }
 939        coalesceTreeImpl(child);
 940        if (child && (child->nodeType == node->nodeType) &&
 941            ((child->nodeType == ADDSTATEMENT))) {
 942          node->children.insert(node->children.end(),
 943                                child->children.begin(),
 944                                child->children.end());
 945          delete child; // no longer in the tree
 946        } else {
 947          node->children.push_back(child);
 948        }
 949      }
 950    }
 951  }
 952  xhpast::Node* outputXHPAST() {
 953    xhpast::Node* xhpast_tree = outputXHPASTImpl(&tree);
 954    xhpast::Node* root = new xhpast::Node(n_PROGRAM);
 955    root->appendChild(xhpast_tree);
 956    root->l_tok = xhpast_tree->l_tok;
 957    return root;
 958  }
 959  xhpast::Node* extend_right(xhpast::Node *n, int type) {
 960    // Extend r_tok to the first token of type
 961    n->r_tok = scan_forward(n->r_tok + 1, type);
 962    return n;
 963  }
 964  xhpast::Node* extend_left(xhpast::Node *n, int type) {
 965    // Extend l_tok to the first token of type
 966    for (int i = n->l_tok - 1; i >= 0; i--) {
 967      xhpast::Token *t = m_listener.tokens[i];
 968      if (t->type == type) {
 969        n->l_tok = i;
 970        return n;
 971      }
 972    }
 973    // missing token!
 974    always_assert(false);
 975  }
 976  int scan_forward(int start, int type) {
 977    for (int i = start; i < m_listener.tokens.size(); i++) {
 978      if (m_listener.tokens[i]->type == type) {
 979        return i;
 980      }
 981    }
 982    // missing token!
 983    always_assert(false);
 984  }
 985  int scan_backward(int start, int type) {
 986    for (int i = start; i >= 0; i--) {
 987      if (m_listener.tokens[i]->type == type) {
 988        return i;
 989      }
 990    }
 991    // missing token!
 992    always_assert(false);
 993  }
 994  xhpast::Node* extend_to_delimiters(xhpast::Node *n, int left, int right) {
 995    extend_left(n, left);
 996    return extend_right(n, right);
 997  }
 998  // returns l_tok of first real child
 999  int transform_children(Token *node, xhpast::Node *n) {
1000    int l_tok = -2;
1001    for (std::vector<Token *>::iterator i = node->children.begin();
1002         i < node->children.end();
1003         ++i) {
1004      // TODO: It is possible that in all cases with NULL children, we just
1005      // want to append n_EMPTY, but just continue quietly for now.
1006      if (!(*i)) continue;
1007      xhpast::Node *newChild = outputXHPASTImpl(*i);
1008      n->appendChild(newChild);
1009      if (l_tok == -2) {
1010        l_tok = newChild->l_tok;
1011      }
1012      // optionally insert an n_OPEN_TAG if it follows an ONECHO
1013      // TODO: only for HTML?
1014      if ((*i)->nodeType == ONECHO) {
1015        int id = (*i)->ID() + 1;
1016        if (id < m_listener.tokens.size()) {
1017          xhpast::Token *t = m_listener.tokens[id];
1018          if (t->type == T_OPEN_TAG) {
1019            n->appendChild(new xhpast::Node(n_OPEN_TAG, id, id));
1020          }
1021        }
1022      }
1023    }
1024    return l_tok;
1025  }
1026  int insert_binary_operator(xhpast::Node *n) {
1027    // Locate operator and insert between children
1028    xhpast::Node *c1 = n->children.front();
1029    xhpast::Node *c2 = n->children.back();
1030    int id1 = c1->r_tok; // must be greater than this
1031    int id2 = c2->l_tok;  // and less than this
1032    for (int i = id1 + 1; i < id2; i++) {
1033      xhpast::Token *t = m_listener.tokens[i];
1034      switch (t->type) {
1035        case T_COMMENT:
1036        case T_DOC_COMMENT: // needed?
1037        case T_WHITESPACE: {
1038          continue;
1039        }
1040        default: {
1041          n->children.pop_back();
1042          n->children.push_back(new xhpast::Node(n_OPERATOR, i, i));
1043          n->children.push_back(c2);
1044          return t->type;
1045        }
1046      }
1047    }
1048    always_assert(false); // couldn't find the operator!
1049  }
1050  xhpast::Node* outputCondition(Token *cond) {
1051    xhpast::Node *control_cond = new xhpast::Node(n_CONTROL_CONDITION);
1052    control_cond->appendChild(outputXHPASTImpl(cond));
1053    extend_to_delimiters(control_cond, '(', ')');
1054    return control_cond;
1055  }
1056  xhpast::Node *childOf(xhpast::Node *child, int type) {
1057    xhpast::Node *parent = new xhpast::Node(type);
1058    parent->appendChild(child);
1059    return parent;
1060  }
1061  // parse ;, stmt;, {} or { stmts; }
1062  xhpast::Node *possibleStatements(int start, Token *node) {
1063    if (node->nodeType == RAW) {
1064      // We need to find the ";"
1065      int semicolon = scan_forward(start, ';');
1066      xhpast::Node *stmt = new xhpast::Node(n_STATEMENT, semicolon);
1067      stmt->appendChild(new xhpast::Node(n_EMPTY));
1068      return stmt;
1069    } else {
1070      return outputXHPASTImpl(node);
1071    }
1072  }
1073  xhpast::Node *declareParamList(Token *params, int name_pos) {
1074    xhpast::Node *param_list =
1075      new xhpast::Node(n_DECLARATION_PARAMETER_LIST);
1076    while (params && (params->nodeType == ONPARAM)) {
1077      std::vector<Token *>::iterator i = params->children.begin();
1078      params = *i++;
1079      UNUSED Token *type = *i++; // TODO
1080      Token *var = *i++;
1081      UNUSED Token *defValue = *i++; // TODO
1082      UNUSED Token *attr = *i++; // TODO
1083      UNUSED Token *mods = *i; // TODO
1084      // TODO: bool ref
1085      xhpast::Node *p = new xhpast::Node(n_DECLARATION_PARAMETER);
1086      // TODO: class type
1087      p->appendChild(new xhpast::Node(n_EMPTY));
1088      p->appendChild(new xhpast::Node(n_VARIABLE, var->ID()));
1089      // TODO: default value
1090      p->appendChild(new xhpast::Node(n_EMPTY));
1091      // Must prepend to get argument order right
1092      param_list->prependChild(p);
1093    }
1094    if (param_list->children.size() == 0) {
1095      // no params
1096      param_list->l_tok = scan_forward(name_pos, '(');
1097      param_list->r_tok = scan_forward(param_list->l_tok, ')');
1098    } else {
1099      extend_to_delimiters(param_list, '(', ')');
1100    }
1101    return param_list;
1102  }
1103  xhpast::Node *statementList(Token *stmt, int last_loc) {
1104    // statement list: need to extend this to brackets
1105    if (stmt) {
1106      xhpast::Node *stmts = outputXHPASTImpl(stmt);
1107      if (stmt->nodeType != FINISHSTATEMENT) {
1108        extend_to_delimiters(stmts, '{', '}');
1109      }
1110      return stmts;
1111    } else {
1112      // empty body; start looking for braces after the params
1113      int open_brace = scan_forward(last_loc, '{');
1114      xhpast::Node *braces = new xhpast::Node(n_STATEMENT_LIST,
1115                                              open_brace);
1116      extend_right(braces, '}');
1117      return braces;
1118    }
1119  }
1120  xhpast::Node *callParamList(Token *params, int name_pos) {
1121    xhpast::Node *param_list = new xhpast::Node(n_CALL_PARAMETER_LIST);
1122    while (params && (params->nodeType == ONCALLPARAM)) {
1123      std::vector<Token *>::iterator i = params->children.begin();
1124      params = *i++;
1125      // TODO: ref
1126      param_list->prependChild(outputXHPASTImpl(*i));
1127    }
1128    if (param_list->children.size() == 0) {
1129      // no params
1130      param_list->l_tok = scan_forward(name_pos, '(');
1131      param_list->r_tok = scan_forward(param_list->l_tok, ')');
1132    } else {
1133      extend_to_delimiters(param_list, '(', ')');
1134    }
1135    return param_list;
1136  }
1137  xhpast::Node *objectProperty(Token *base, Token *prop) {
1138    xhpast::Node *arrow = new xhpast::Node(n_OBJECT_PROPERTY_ACCESS);
1139    arrow->appendChild(outputXHPASTImpl(base));
1140    arrow->appendChild(new xhpast::Node(n_STRING, prop->ID()));
1141    return arrow;
1142  }
1143  xhpast::Node *outputXHPASTImpl(Token *node) {
1144   xhpast::Node* n = new xhpast::Node();
1145    n->l_tok = node->ID();
1146    n->r_tok = node->ID();
1147
1148    switch (node->nodeType) {
1149      case ONSIMPLEVARIABLE: {
1150        // no children
1151        n->type = n_VARIABLE;
1152        break;
1153      }
1154      case ONECHO: {
1155        std::vector<Token *>::iterator i = node->children.begin();
1156        if ((*i)->nodeType == RAW) {
1157          // no children
1158          n->type = n_INLINE_HTML;
1159        } else {
1160          n->type = n_ECHO_LIST;
1161          Token *current = *i;
1162          while (current) {
1163            std::vector<Token *>::iterator i = current->children.begin();
1164            current = *i++;
1165            xhpast::Node *child = outputXHPASTImpl(*i);
1166            if (n->r_tok < child->r_tok) {
1167              n->r_tok = child->r_tok;
1168            }
1169            n->prependChild(child);
1170          }
1171          xhpast::Node *stmt = childOf(n, n_STATEMENT);
1172          return extend_right(stmt, ';');
1173        }
1174        break;
1175      }
1176      case ONSTATEMENTLISTSTART: {
1177        // Should never reach here because parent should have elided this
1178        always_assert(false);
1179        break;
1180      }
1181      case ONEXPSTATEMENT: {
1182        n->type = n_STATEMENT;
1183        transform_children(node, n);
1184        extend_right(n, ';');
1185        break;
1186      }
1187      case ADDSTATEMENT: {
1188        n->type = n_STATEMENT_LIST;
1189        // ADDSTATEMENT simply contains its children, so set l_tok to be l_tok
1190        // of first child; r_tok will be set correctly by transform_children
1191        n->l_tok = transform_children(node, n);
1192        break;
1193      }
1194      case ONRETURN: {
1195        n->type = n_RETURN;
1196        std::vector<Token *>::iterator i = node->children.begin();
1197        Token *expr = *i;
1198        if (expr) {
1199          transform_children(node, n);
1200        } else {
1201          n->appendChild(new xhpast::Node(n_EMPTY));
1202        }
1203        xhpast::Node *stmt = childOf(n, n_STATEMENT);
1204        return extend_right(stmt, ';');
1205      }
1206      case ONMETHOD: {
1207        std::vector<Token *>::iterator i = node->children.begin();
1208        Token *modifiers = *i++;
1209        UNUSED Token *ret = *i++; // TODO
1210        UNUSED Token *ref = *i++; // TODO
1211        Token *name = *i++;
1212        Token *params = *i++;
1213        Token *stmt = *i++;
1214        UNUSED Token *attr = *i; // TODO
1215        // TODO: reloc
1216        n->type = n_METHOD_DECLARATION;
1217        // modifiers private public protected, etc.
1218        xhpast::Node *mods = new xhpast::Node(n_METHOD_MODIFIER_LIST);
1219        while (modifiers && (modifiers->nodeType == ONMEMBERMODIFIER)) {
1220          i = modifiers->children.begin();
1221          modifiers = *i++;
1222          mods->appendChild(new xhpast::Node(n_STRING, (*i)->ID()));
1223        }
1224        n->appendChild(mods);
1225        // TODO
1226        n->appendChild(new xhpast::Node(n_EMPTY));
1227        n->appendChild(new xhpast::Node(n_STRING, name->ID()));
1228        xhpast::Node *param_list = declareParamList(params, name->ID());
1229        n->appendChild(param_list);
1230        // TODO
1231        n->appendChild(new xhpast::Node(n_EMPTY));
1232        n->appendChild(statementList(stmt, param_list->r_tok));
1233        // Hang method off n_statement
1234        return childOf(n, n_STATEMENT);
1235      }
1236      case ONFUNCTION: {
1237        // children are modifiers (maybe null), ret, ref, name, params, stmt,
1238        // attr (maybe null)
1239        std::vector<Token *>::iterator i = node->children.begin();
1240        UNUSED Token *modifiers = *i++; // TODO
1241        UNUSED Token *ret = *i++; // TODO
1242        UNUSED Token *ref = *i++; // TODO
1243        Token *name = *i++;
1244        Token *params = *i++;
1245        Token *stmt = *i++;
1246        UNUSED Token *attr = *i; // TODO
1247        n->type = n_FUNCTION_DECLARATION;
1248        // TODO: T_STATIC
1249        n->appendChild(new xhpast::Node(n_EMPTY));
1250        // TODO: is_reference
1251        n->appendChild(new xhpast::Node(n_EMPTY));
1252        // function name
1253        n->appendChild(new xhpast::Node(n_STRING, name->ID()));
1254        // params
1255        xhpast::Node *param_list = declareParamList(params, name->ID());
1256        n->appendChild(param_list);
1257        // lexical vars
1258        n->appendChild(new xhpast::Node(n_EMPTY));
1259        // statement list: need to extend this to brackets
1260        n->appendChild(statementList(stmt, param_list->r_tok));
1261        // Hang function off n_statement
1262        return childOf(n, n_STATEMENT);
1263      }
1264      case ONPARAM: {
1265        // should be handled in onfunction / onmethod
1266        always_assert(false);
1267        break;
1268      }
1269      case ONASSIGN:
1270      case ONBINARYOPEXP: {
1271        n->type = n_BINARY_EXPRESSION;
1272        transform_children(node, n);
1273        int t = insert_binary_operator(n);
1274        // See if n_CONCATENATION_LIST
1275        if (t == '.') {
1276          n->type = n_CONCATENATION_LIST;
1277          xhpast::Node *second = n->children.back();
1278          n->children.pop_back();
1279          xhpast::Node *op = n->children.back();
1280          n->children.pop_back();
1281          xhpast::Node *first = n->children.back();
1282          n->children.pop_back();
1283          if (first->type == n_CONCATENATION_LIST) {
1284            n->appendChildren(first);
1285          } else {
1286            n->appendChild(first);
1287          }
1288          n->appendChild(op);
1289          // note: the second child should never be an n_CONCATENATION_LIST due
1290          // to the structure of the parse tree
1291          n->appendChild(second);
1292        }
1293        break;
1294      }
1295      case ONIF: {
1296        // children are cond stmt elsifs elsestmt
1297        n->type = n_IF;
1298        std::vector<Token *>::iterator i = node->children.begin();
1299        Token *cond = *i++;
1300        Token *stmt = *i++;
1301        Token *elseifs = *i++;
1302        Token *elsestmt = *i;
1303        n->type = n_IF;
1304        // condition
1305        n->appendChild(outputCondition(cond));
1306        // statements
1307        n->appendChild(outputXHPASTImpl(stmt));
1308        // elseifs
1309        while (elseifs->nodeType == ONELSEIF) {
1310          i = elseifs->children.begin();
1311          Token *next_elseifs = *i++;
1312          xhpast::Node *elseif_out = new xhpast::Node(n_ELSEIF);
1313          elseif_out->appendChild(outputCondition(*i++));
1314          elseif_out->appendChild(outputXHPASTImpl(*i));
1315          extend_left(elseif_out, T_ELSEIF);
1316          n->appendChild(elseif_out);
1317          elseifs = next_elseifs;
1318        }
1319        xhpast::Node *else_out = new xhpast::Node(n_ELSE);
1320        else_out->appendChild(outputXHPASTImpl(elsestmt));
1321        extend_left(else_out, T_ELSE);
1322        n->appendChild(else_out);
1323
1324        xhpast::Node *cond_list = new xhpast::Node(n_CONDITION_LIST);
1325        cond_list->appendChild(n);
1326        return childOf(cond_list, n_STATEMENT);
1327      }
1328      case ONWHILE: {
1329        n->type = n_WHILE;
1330        std::vector<Token *>::iterator i = node->children.begin();
1331        n->appendChild(outputCondition(*i++));
1332        n->appendChild(possibleStatements(n->r_tok, *i));
1333        return childOf(n, n_STATEMENT);
1334      }
1335      case ONFOR: {
1336        n->type = n_FOR;
1337        std::vector<Token *>::iterator i = node->children.begin();
1338        // initialization
1339        xhpast::Node *loop_exprs = childOf(outputXHPASTImpl(*i++),
1340                                           n_FOR_EXPRESSION);
1341        // termination condition
1342        loop_exprs->appendChild(outputXHPASTImpl(*i++));
1343        // loop update
1344        loop_exprs->appendChild(outputXHPASTImpl(*i++));
1345        extend_to_delimiters(loop_exprs, '(', ')');
1346        n->appendChild(loop_exprs);
1347        // loop body
1348        n->appendChild(possibleStatements(n->r_tok, *i));
1349        return childOf(n, n_STATEMENT);
1350      }
1351      case ONEXPRLISTELEM: {
1352        transform_children(node, n);
1353        n->type = n_EXPRESSION_LIST;
1354        break;
1355      }
1356      case ONBLOCK: {
1357        transform_children(node, n);
1358        if (node->children.size() > 0) {
1359          xhpast::Node *stmts = n->firstChild();
1360          extend_to_delimiters(stmts, '{', '}');
1361          return stmts;
1362        } else {
1363          n->type = n_STATEMENT_LIST;
1364          extend_right(n, '}');
1365        }
1366        break;
1367      }
1368      case ONSCALAR: {
1369        OnScalarEI *ei = dynamic_cast<OnScalarEI*>(node->extra);
1370        switch (ei->type) {
1371          case T_DNUMBER:
1372          case T_LNUMBER: {
1373            n->type = n_NUMERIC_SCALAR;
1374            break;
1375          }
1376          case T_CONSTANT_ENCAPSED_STRING: {
1377            n->type = n_STRING_SCALAR;
1378            break;
1379          }
1380          case T_LINE:
1381          case T_FILE:
1382          case T_DIR:
1383          case T_CLASS_C:
1384          case T_TRAIT_C:
1385          case T_METHOD_C:
1386          case T_FUNC_C:
1387          case T_NS_C: {
1388            n->type = n_MAGIC_SCALAR;
1389            break;
1390          }
1391          default: {
1392            // where do we output n_HEREDOC?
1393            always_assert(false); // unexpected
1394          }
1395        }
1396        break;
1397      }
1398      case ONUNARYOPEXP: {
1399        transform_children(node, n);
1400        OnUnaryOpExpEI *ei = dynamic_cast<OnUnaryOpExpEI*>(node->extra);
1401        if (ei->front) {
1402          n->type = n_UNARY_PREFIX_EXPRESSION;
1403          int loc = scan_backward(node->ID(), ei->op);
1404          n->prependChild(new xhpast::Node(n_OPERATOR, loc));
1405        } else { // back
1406          n->type = n_UNARY_POSTFIX_EXPRESSION;
1407          int loc = scan_forward(node->ID(), ei->op);
1408          n->appendChild(new xhpast::Node(n_OPERATOR, loc));
1409        }
1410        break;
1411      }
1412      case ONBREAK: {
1413        n->type = n_BREAK;
1414        // TODO: break expr
1415        n->appendChild(new xhpast::Node(n_EMPTY));
1416        xhpast::Node* top_stmt = childOf(n, n_STATEMENT);
1417        return extend_right(top_stmt, ';');
1418      }
1419      case ONCONTINUE: {
1420        n->type = n_CONTINUE;
1421        // TODO: continue expr
1422        n->appendChild(new xhpast::Node(n_EMPTY));
1423        xhpast::Node* top_stmt = childOf(n, n_STATEMENT);
1424        return extend_right(top_stmt, ';');
1425      }
1426      case ONSWITCH: {
1427        n->type = n_SWITCH;
1428        std::vector<Token *>::iterator i = node->children.begin();
1429        // first child is control condition
1430        n->appendChild(outputCondition(*i++));
1431        // next child is list of cases
1432        xhpast::Node *case_list = new xhpast::Node(n_STATEMENT_LIST);
1433        int right_most = n->r_tok;
1434        Token* cases = *i;
1435        while (cases->nodeType == ONCASE) {
1436          i = cases->children.begin();
1437          std::vector<Token *>::iterator end = cases->children.end();
1438          cases = *i++;
1439          // cse because case is a reserved word
1440          xhpast::Node *cse;
1441          if (*i) {
1442            cse = new xhpast::Node(n_CASE);
1443            // condition
1444            cse->appendChild(outputXHPASTImpl(*i));
1445            extend_left(cse, T_CASE);
1446          } else {
1447            std::cout << right_most << std::endl;
1448            int loc = scan_forward(right_most, T_DEFAULT);
1449            cse = new xhpast::Node(n_DEFAULT, loc);
1450          }
1451          if (i != end) {
1452            i++;
1453          }
1454          // body
1455          if (i != end) {
1456            // TODO: replace with possibleStatements?
1457            cse->appendChild(outputXHPASTImpl(*i));
1458          } else {
1459            cse->appendChild(new xhpast::Node(n_STATEMENT_LIST));
1460          }
1461          // must prepend to get the order right
1462          case_list->prependChild(cse);
1463          right_most = case_list->r_tok;
1464        }
1465        n->appendChild(extend_to_delimiters(case_list, '{', '}'));
1466        return childOf(n, n_STATEMENT);
1467      }
1468      case ONARRAY: {
1469        // TODO: depends on type
1470        n->type = n_ARRAY_LITERAL;
1471        xhpast::Node *value_list = new xhpast::Node(n_ARRAY_VALUE_LIST);
1472        std::vector<Token *>::iterator i = node->children.begin();
1473        Token *pair = *i;
1474        while (pair && pair->nodeType == ONARRAYPAIR) {
1475          // TODO: refs
1476          i = pair->children.begin();
1477          Token *next_pair = *i++;
1478          Token *name = *i++;
1479          Token *value = *i;
1480          xhpast::Node *entry = new xhpast::Node(n_ARRAY_VALUE);
1481          if (name) {
1482            entry->appendChild(outputXHPASTImpl(name));
1483          } else {
1484            entry->appendChild(new xhpast::Node(n_EMPTY));
1485          }
1486          entry->appendChild(outputXHPASTImpl(value));
1487          // must prepend to get the order right
1488          value_list->prependChild(entry);
1489          pair = next_pair;
1490        }
1491        n->appendChild(value_list);
1492        extend_right(n, ')');
1493        break;
1494      }
1495      case ONOBJECTMETHODCALL: {
1496        std::vector<Token *>::iterator i = node->children.begin();
1497        Token *base = *i++;
1498        Token *prop = *i++;
1499        Token *params = *i;
1500        n->type = n_METHOD_CALL;
1501        n->appendChild(objectProperty(base, prop));
1502        n->appendChild(callParamList(params, prop->ID()));
1503        break;
1504      }
1505      case ONCALL: {
1506        std::vector<Token *>::iterator i = node->children.begin();
1507        Token *name = *i++;
1508        Token *params = *i++;
1509        UNUSED Token *cls = *i; // TODO
1510        // TODO: dynamic, fromCompiler, cls
1511        n->type = n_FUNCTION_CALL;
1512        n->appendChild(new xhpast::Node(n_SYMBOL_NAME, name->ID()));
1513        n->appendChild(callParamList(params, name->ID()));
1514        break;
1515      }
1516      case ONCLASS: {
1517        std::vector<Token *>::iterator i = node->children.begin();
1518        Token *name = *i++;
1519        UNUSED Token *base = *i++; // TODO
1520        UNUSED Token *baseInterface = *i++; // TODO
1521        Token *stmt = *i++;
1522        UNUSED Token *attr = *i; // TODO
1523        n->type = n_CLASS_DECLARATION;
1524        // TODO attributes
1525        xhpast::Node *attr_out = new xhpast::Node(n_CLASS_ATTRIBUTES, n->l_tok);
1526        attr_out->appendChild(new xhpast::Node(n_EMPTY));
1527        n->appendChild(attr_out);
1528        n->appendChild(new xhpast::Node(n_CLASS_NAME, name->ID()));
1529        // TODO
1530        n->appendChild(new xhpast::Node(n_EMPTY));
1531        // TODO
1532        n->appendChild(new xhpast::Node(n_EMPTY));
1533        n->appendChild(outputXHPASTImpl(stmt));
1534        return childOf(n, n_STATEMENT);
1535      }
1536      case ONCLASSSTATEMENT: {
1537        n->type = n_STATEMENT_LIST;
1538        Token *stmts = node;
1539        do {
1540          std::vector<Token *>::iterator i = stmts->children.begin();
1541          stmts = *i++;
1542          Token *stmt = *i;
1543          n->prependChild(outputXHPASTImpl(stmt));
1544        } while(stmts->nodeType == ONCLASSSTATEMENT);
1545        extend_to_delimiters(n, '{', '}');
1546        break;
1547      }
1548      case FINISHSTATEMENT: {
1549        if (node->children.size() > 0) {
1550          std::vector<Token *>::iterator i = node->children.begin();
1551          return extend_to_delimiters(outputXHPASTImpl(*i), '{', '}');
1552        } else {
1553          n->type = n_STATEMENT_LIST;
1554          return extend_right(n, '}');
1555        }
1556      }
1557      case ONDO: {
1558        n->type = n_DO_WHILE;
1559        std::vector<Token *>::iterator i = node->children.begin();
1560        n->appendChild(possibleStatements(n->l_tok, *i++));
1561        n->appendChild(outputCondition(*i));
1562        xhpast::Node *ret = childOf(n

Large files files are truncated, but you can click here to view the full file