PageRenderTime 46ms CodeModel.GetById 14ms RepoModel.GetById 0ms app.codeStats 1ms

/modules/asset-merger/vendor/coffeescript/classes/lexer.php

https://github.com/data-quest/histat-web
PHP | 1108 lines | 950 code | 142 blank | 16 comment | 105 complexity | 7017c4092169ade497d638d0ffef2575 MD5 | raw file
Possible License(s): BSD-3-Clause, LGPL-3.0, LGPL-2.1
  1. <?php
  2. namespace CoffeeScript;
  3. require_once 'errors.php';
  4. require_once 'helpers.php';
  5. require_once 'nodes.php';
  6. require_once 'rewriter.php';
  7. /**
  8. * CoffeeScript lexer. For the most part it's directly from the original
  9. * source code, though there are some relatively minor differences in how it
  10. * works with the parser (since we're using Lemon).
  11. */
  12. class Lexer
  13. {
  14. static $COFFEE_ALIASES = array(
  15. 'and' => '&&',
  16. 'or' => '||',
  17. 'is' => '==',
  18. 'isnt' => '!=',
  19. 'not' => '!',
  20. 'yes' => 'true',
  21. 'no' => 'false',
  22. 'on' => 'true',
  23. 'off' => 'false'
  24. );
  25. static $COFFEE_KEYWORDS = array(
  26. 'by',
  27. 'loop',
  28. 'of',
  29. 'then',
  30. 'undefined',
  31. 'unless',
  32. 'until',
  33. 'when'
  34. );
  35. // exports.RESERVED.
  36. static $COFFEE_RESERVED = array();
  37. static $JS_KEYWORDS = array(
  38. 'break',
  39. 'catch',
  40. 'class',
  41. 'continue',
  42. 'debugger',
  43. 'delete',
  44. 'do',
  45. 'else',
  46. 'extends',
  47. 'false',
  48. 'finally',
  49. 'for',
  50. 'if',
  51. 'in',
  52. 'instanceof',
  53. 'new',
  54. 'null',
  55. 'this',
  56. 'throw',
  57. 'typeof',
  58. 'return',
  59. 'switch',
  60. 'super',
  61. 'true',
  62. 'try',
  63. 'while',
  64. );
  65. // RESERVED.
  66. static $JS_RESERVED = array(
  67. '__bind',
  68. '__extends',
  69. '__hasProp',
  70. '__indexOf',
  71. '__slice',
  72. 'case',
  73. 'const',
  74. 'default',
  75. 'enum',
  76. 'export',
  77. 'function',
  78. 'import',
  79. 'let',
  80. 'native',
  81. 'var',
  82. 'void',
  83. 'with',
  84. );
  85. static $JS_FORBIDDEN = array();
  86. static $ASSIGNED = '/^\s*@?([$A-Za-z_][$\w\x7f-\x{ffff}]*|[\'"].*[\'"])[^\n\S]*?[:=][^:=>]/u';
  87. static $CODE = '/^[-=]>/';
  88. static $COMMENT = '/^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/';
  89. static $HEREDOC = '/^("""|\'\'\')([\s\S]*?)(?:\n[^\n\S]*)?\1/';
  90. static $HEREDOC_INDENT = '/\n+([^\n\S]*)/';
  91. static $HEREDOC_ILLEGAL = '%\*/%';
  92. static $HEREGEX = '%^/{3}([\s\S]+?)/{3}([imgy]{0,4})(?!\w)%';
  93. static $HEREGEX_OMIT = '/\s+(?:#.*)?/';
  94. static $IDENTIFIER = '/^([$A-Za-z_\x7f-\x{ffff}][$\w\x7f-\x{ffff}]*)([^\n\S]*:(?!:))?/u';
  95. static $JSTOKEN = '/^`[^\\\\`]*(?:\\\\.[^\\\\`]*)*`/';
  96. static $LINE_CONTINUER = '/^\s*(?:,|\??\.(?![.\d])|::)/';
  97. static $MULTI_DENT = '/^(?:\n[^\n\S]*)+/';
  98. static $MULTILINER = '/\n/';
  99. static $NO_NEWLINE = '#^(?:[-+*&|/%=<>!.\\\\][<>=&|]*|and|or|is(?:nt)?|n(?:ot|ew)|delete|typeof|instanceof)$#';
  100. static $NUMBER = '/^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i';
  101. static $OPERATOR = '#^(?:[-=]>|[-+*/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>])\2=?|\?\.|\.{2,3})#';
  102. static $REGEX = '%^/(?![\s=])[^[/\n\\\\]*(?:(?:\\\\[\s\S]|\[[^\]\n\\\\]*(?:\\\\[\s\S][^\]\n\\\\]*)*\])[^[/\n\\\\]*)*/[imgy]{0,4}(?!\w)%';
  103. static $SIMPLESTR = '/^\'[^\\\\\']*(?:\\\\.[^\\\\\']*)*\'/';
  104. static $TRAILING_SPACES = '/\s+$/';
  105. static $WHITESPACE = '/^[^\n\S]+/';
  106. static $BOOL = array('TRUE', 'FALSE', 'NULL', 'UNDEFINED');
  107. static $CALLABLE = array('IDENTIFIER', 'STRING', 'REGEX', ')', ']', '}', '?', '::', '@', 'THIS', 'SUPER');
  108. static $COMPARE = array('==', '!=', '<', '>', '<=', '>=');
  109. static $COMPOUND_ASSIGN = array('-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=' );
  110. static $INDEXABLE = array('NUMBER', 'BOOL');
  111. static $LINE_BREAK = array('INDENT', 'OUTDENT', 'TERMINATOR');
  112. static $LOGIC = array('&&', '||', '&', '|', '^');
  113. static $MATH = array('*', '/', '%');
  114. static $NOT_REGEX = array('NUMBER', 'REGEX', 'BOOL', '++', '--', ']');
  115. static $NOT_SPACED_REGEX = array(')', '}', 'THIS', 'IDENTIFIER', 'STRING');
  116. static $RELATION = array('IN', 'OF', 'INSTANCEOF');
  117. static $SHIFT = array('<<', '>>', '>>>');
  118. static $UNARY = array('!', '~', 'NEW', 'TYPEOF', 'DELETE', 'DO');
  119. function __construct($code, $options)
  120. {
  121. if (preg_match(self::$WHITESPACE, $code))
  122. {
  123. $code = "\n{$code}";
  124. }
  125. $code = preg_replace(self::$TRAILING_SPACES, '', str_replace("\r", '', $code));
  126. $options = array_merge(array(
  127. 'indent' => 0,
  128. 'index' => 0,
  129. 'line' => 0,
  130. 'rewrite' => TRUE
  131. ),
  132. $options);
  133. $this->code = $code;
  134. $this->chunk = $code;
  135. $this->indent = 0;
  136. $this->indents = array();
  137. $this->indebt = 0;
  138. $this->index = $options['index'];
  139. $this->length = strlen($this->code);
  140. $this->line = $options['line'];
  141. $this->outdebt = 0;
  142. $this->options = $options;
  143. $this->tokens = array();
  144. }
  145. function assignment_error()
  146. {
  147. throw new SyntaxError('Reserved word "'.$this->value().'" on line '.($this->line + 1).' can\'t be assigned');
  148. }
  149. function balanced_string($str, $end)
  150. {
  151. $stack = array($end);
  152. $prev = NULL;
  153. for ($i = 1; $i < strlen($str); $i++)
  154. {
  155. switch ($letter = $str{$i})
  156. {
  157. case '\\':
  158. $i++;
  159. continue 2;
  160. case $end:
  161. array_pop($stack);
  162. if ( ! count($stack))
  163. {
  164. return substr($str, 0, $i + 1);
  165. }
  166. $end = $stack[count($stack) - 1];
  167. continue 2;
  168. }
  169. if ($end === '}' && ($letter === '"' || $letter === '\''))
  170. {
  171. $stack[] = $end = $letter;
  172. }
  173. else if ($end === '}' && $letter === '{')
  174. {
  175. $stack[] = $end = '}';
  176. }
  177. else if ($end === '"' && $prev === '#' && $letter === '{')
  178. {
  179. $stack[] = $end = '}';
  180. }
  181. $prev = $letter;
  182. }
  183. throw new Error('missing '.array_pop($stack).' starting on line '.($this->line + 1));
  184. }
  185. function close_indentation()
  186. {
  187. $this->outdent_token($this->indent);
  188. }
  189. function comment_token()
  190. {
  191. if ( ! preg_match(self::$COMMENT, $this->chunk, $match))
  192. {
  193. return 0;
  194. }
  195. $comment = $match[0];
  196. if (isset($match[1]) && ($here = $match[1]))
  197. {
  198. $this->token('HERECOMMENT', $this->sanitize_heredoc($here, array(
  199. 'herecomment' => TRUE,
  200. 'indent' => str_pad('', $this->indent)
  201. )));
  202. $this->token('TERMINATOR', "\n");
  203. }
  204. $this->line += substr_count($comment, "\n");
  205. return strlen($comment);
  206. }
  207. function escape_lines($str, $heredoc = NULL)
  208. {
  209. return preg_replace(self::$MULTILINER, $heredoc ? '\\n' : '', $str);
  210. }
  211. function heredoc_token()
  212. {
  213. if ( ! preg_match(self::$HEREDOC, $this->chunk, $match))
  214. {
  215. return 0;
  216. }
  217. $heredoc = $match[0];
  218. $quote = $heredoc{0};
  219. $doc = $this->sanitize_heredoc($match[2], array('quote' => $quote, 'indent' => NULL));
  220. if ($quote === '"' && strpos($doc, '#{') !== FALSE)
  221. {
  222. $this->interpolate_string($doc, array('heredoc' => TRUE));
  223. }
  224. else
  225. {
  226. $this->token('STRING', $this->make_string($doc, $quote, TRUE));
  227. }
  228. $this->line += substr_count($heredoc, "\n");
  229. return strlen($heredoc);
  230. }
  231. function heregex_token($match)
  232. {
  233. list($heregex, $body, $flags) = $match;
  234. if (strpos($body, '#{') === FALSE)
  235. {
  236. $re = preg_replace(self::$HEREGEX_OMIT, '', $body);
  237. $re = preg_replace('/\//', '\\/', $re);
  238. $this->token('REGEX', '/'.($re ? $re : '(?:)').'/'.$flags);
  239. return strlen($heregex);
  240. }
  241. $this->token('IDENTIFIER', 'RegExp');
  242. $this->tokens[] = array(t('CALL_START'), '(');
  243. $tokens = array();
  244. foreach ($this->interpolate_string($body, array('regex' => TRUE)) as $token)
  245. {
  246. list($tag, $value) = $token;
  247. if ($tag === 'TOKENS')
  248. {
  249. $tokens = array_merge($tokens, (array) $value);
  250. }
  251. else
  252. {
  253. if ( ! ($value = preg_replace(self::$HEREGEX_OMIT, '', $value)))
  254. {
  255. continue;
  256. }
  257. $value = preg_replace('/\\\\/', '\\\\\\\\', $value);
  258. $tokens[] = array(t('STRING'), $this->make_string($value, '"', TRUE));
  259. }
  260. $tokens[] = array(t('+'), '+');
  261. }
  262. array_pop($tokens);
  263. if ( ! (isset($tokens[0]) && $tokens[0][0] === 'STRING'))
  264. {
  265. array_push($this->tokens, array(t('STRING'), '""'), array(t('+'), '+'));
  266. }
  267. $this->tokens = array_merge($this->tokens, $tokens);
  268. if ($flags)
  269. {
  270. array_push($this->tokens, array(t(','), ','), array(t('STRING'), "\"{$flags}\""));
  271. }
  272. $this->token(')', ')');
  273. return strlen($heregex);
  274. }
  275. function identifier_error($word)
  276. {
  277. throw new SyntaxError('Reserved word "'.$word.'" on line '.($this->line + 1));
  278. }
  279. function identifier_token()
  280. {
  281. if ( ! preg_match(self::$IDENTIFIER, $this->chunk, $match))
  282. {
  283. return 0;
  284. }
  285. list($input, $id) = $match;
  286. $colon = isset($match[2]) ? $match[2] : NULL;
  287. if ($id === 'own' && $this->tag() === t('FOR'))
  288. {
  289. $this->token('OWN', $id);
  290. return strlen($id);
  291. }
  292. $forced_identifier = $colon || ($prev = last($this->tokens)) &&
  293. (in_array($prev[0], t('.', '?.', '::')) ||
  294. ( ! (isset($prev['spaced']) && $prev['spaced']) && $prev[0] === t('@')));
  295. $tag = 'IDENTIFIER';
  296. if (in_array($id, self::$JS_KEYWORDS) || ! $forced_identifier && in_array($id, self::$COFFEE_KEYWORDS))
  297. {
  298. $tag = strtoupper($id);
  299. if ($tag === 'WHEN' && in_array($this->tag(), t(self::$LINE_BREAK)))
  300. {
  301. $tag = 'LEADING_WHEN';
  302. }
  303. else if ($tag === 'FOR')
  304. {
  305. $this->seen_for = TRUE;
  306. }
  307. else if ($tag === 'UNLESS')
  308. {
  309. $tag = 'IF';
  310. }
  311. else if (in_array($tag, self::$UNARY))
  312. {
  313. $tag = 'UNARY';
  314. }
  315. else if (in_array($tag, self::$RELATION))
  316. {
  317. if ($tag !== 'INSTANCEOF' && (isset($this->seen_for) && $this->seen_for))
  318. {
  319. $tag = 'FOR'.$tag;
  320. $this->seen_for = FALSE;
  321. }
  322. else
  323. {
  324. $tag = 'RELATION';
  325. if ($this->value() === '!')
  326. {
  327. array_pop($this->tokens);
  328. $id = '!'. $id;
  329. }
  330. }
  331. }
  332. }
  333. // Since data types in PHP are primitive, we can't easily attach parameters
  334. // to token values, and must resort to doing so on the token array.
  335. $reserved = FALSE;
  336. if (in_array($id, self::$JS_FORBIDDEN))
  337. {
  338. if ($forced_identifier)
  339. {
  340. // $id = (object) $id;
  341. // $id->reserved = TRUE;
  342. $tag = 'IDENTIFIER';
  343. $reserved = TRUE;
  344. }
  345. else if (in_array($id, self::$JS_RESERVED, TRUE))
  346. {
  347. $this->identifier_error($id);
  348. }
  349. }
  350. if ( ! $forced_identifier)
  351. {
  352. if (isset(self::$COFFEE_ALIASES[$id]))
  353. {
  354. $id = self::$COFFEE_ALIASES[$id];
  355. }
  356. $map = array(
  357. 'UNARY' => array('!'),
  358. 'COMPARE' => array('==', '!='),
  359. 'LOGIC' => array('&&', '||'),
  360. 'BOOL' => array('true', 'false', 'null', 'undefined'),
  361. 'STATEMENT' => array('break', 'continue', 'debugger')
  362. );
  363. foreach ($map as $k => $v)
  364. {
  365. if (in_array($id, $v))
  366. {
  367. $tag = $k;
  368. break;
  369. }
  370. }
  371. }
  372. $this->token($tag, $id, array('reserved' => $reserved));
  373. if ($colon)
  374. {
  375. $this->token(':', ':');
  376. }
  377. return strlen($input);
  378. }
  379. /**
  380. * Initialize some static variables (called at the end of this file).
  381. */
  382. static function init()
  383. {
  384. self::$COFFEE_KEYWORDS = array_merge(self::$COFFEE_KEYWORDS, array_keys(self::$COFFEE_ALIASES));
  385. self::$COFFEE_RESERVED = array_merge(array_merge(self::$JS_RESERVED, self::$JS_KEYWORDS), self::$COFFEE_KEYWORDS);
  386. self::$JS_FORBIDDEN = array_merge(self::$JS_KEYWORDS, self::$JS_RESERVED);
  387. self::$INDEXABLE = array_merge(self::$CALLABLE, self::$INDEXABLE);
  388. self::$NOT_SPACED_REGEX = array_merge(self::$NOT_REGEX, self::$NOT_SPACED_REGEX);
  389. }
  390. function interpolate_string($str, array $options = array()) // #{0}
  391. {
  392. $options = array_merge(array(
  393. 'heredoc' => '',
  394. 'regex' => NULL
  395. ),
  396. $options);
  397. $tokens = array();
  398. $pi = 0;
  399. $i = -1;
  400. while ( isset($str{++$i}) )
  401. {
  402. $letter = $str{$i};
  403. if ($letter === '\\')
  404. {
  405. $i++;
  406. continue;
  407. }
  408. if ( ! ($letter === '#' && $str{$i + 1} === '{' &&
  409. ($expr = $this->balanced_string(substr($str, $i + 1), '}'))) )
  410. {
  411. continue;
  412. }
  413. if ($pi < $i)
  414. {
  415. $tokens[] = array('NEOSTRING', substr($str, $pi, $i - $pi));
  416. }
  417. $inner = substr($expr, 1, -1);
  418. if (strlen($inner))
  419. {
  420. $lexer = new Lexer($inner, array(
  421. 'line' => $this->line,
  422. 'rewrite' => FALSE,
  423. ));
  424. $nested = $lexer->tokenize();
  425. array_pop($nested);
  426. if (isset($nested[0]) && $nested[0][0] === t('TERMINATOR'))
  427. {
  428. array_shift($nested);
  429. }
  430. if ( ($length = count($nested)) )
  431. {
  432. if ($length > 1)
  433. {
  434. array_unshift($nested, array(t('('), '('));
  435. $nested[] = array(t(')'), ')');
  436. }
  437. $tokens[] = array('TOKENS', $nested);
  438. }
  439. }
  440. $i += strlen($expr);
  441. $pi = $i + 1;
  442. }
  443. if ($i > $pi && $pi < strlen($str))
  444. {
  445. $tokens[] = array('NEOSTRING', substr($str, $pi));
  446. }
  447. if ($options['regex'])
  448. {
  449. return $tokens;
  450. }
  451. if ( ! count($tokens))
  452. {
  453. return $this->token('STRING', '""');
  454. }
  455. if ( ! ($tokens[0][0] === 'NEOSTRING'))
  456. {
  457. array_unshift($tokens, array('', ''));
  458. }
  459. if ( ($interpolated = count($tokens) > 1) )
  460. {
  461. $this->token('(', '(');
  462. }
  463. for ($i = 0; $i < count($tokens); $i++)
  464. {
  465. list($tag, $value) = $tokens[$i];
  466. if ($i)
  467. {
  468. $this->token('+', '+');
  469. }
  470. if ($tag === 'TOKENS')
  471. {
  472. $this->tokens = array_merge($this->tokens, $value);
  473. }
  474. else
  475. {
  476. $this->token('STRING', $this->make_string($value, '"', $options['heredoc']));
  477. }
  478. }
  479. if ($interpolated)
  480. {
  481. $this->token(')', ')');
  482. }
  483. return $tokens;
  484. }
  485. function js_token()
  486. {
  487. if ( ! ($this->chunk{0} === '`' && preg_match(self::$JSTOKEN, $this->chunk, $match)))
  488. {
  489. return 0;
  490. }
  491. $this->token('JS', substr($script = $match[0], 1, -1));
  492. return strlen($script);
  493. }
  494. function line_token()
  495. {
  496. if ( ! preg_match(self::$MULTI_DENT, $this->chunk, $match))
  497. {
  498. return 0;
  499. }
  500. $indent = $match[0];
  501. $this->line += substr_count($indent, "\n");
  502. // $prev = & last($this->tokens, 1);
  503. $size = strlen($indent) - 1 - strrpos($indent, "\n");
  504. $no_newlines = $this->unfinished();
  505. if (($size - $this->indebt) === $this->indent)
  506. {
  507. if ($no_newlines)
  508. {
  509. $this->suppress_newlines();
  510. }
  511. else
  512. {
  513. $this->newline_token();
  514. }
  515. return strlen($indent);
  516. }
  517. if ($size > $this->indent)
  518. {
  519. if ($no_newlines)
  520. {
  521. $this->indebt = $size - $this->indent;
  522. $this->suppress_newlines();
  523. return strlen($indent);
  524. }
  525. $diff = $size - $this->indent + $this->outdebt;
  526. $this->token('INDENT', $diff);
  527. $this->indents[] = $diff;
  528. $this->outdebt = $this->indebt = 0;
  529. }
  530. else
  531. {
  532. $this->indebt = 0;
  533. $this->outdent_token($this->indent - $size, $no_newlines);
  534. }
  535. $this->indent = $size;
  536. return strlen($indent);
  537. }
  538. function literal_token()
  539. {
  540. if (preg_match(self::$OPERATOR, $this->chunk, $match))
  541. {
  542. list($value) = $match;
  543. if (preg_match(self::$CODE, $value))
  544. {
  545. $this->tag_parameters();
  546. }
  547. }
  548. else
  549. {
  550. $value = $this->chunk{0};
  551. }
  552. $tag = $value;
  553. $prev = & last($this->tokens);
  554. if ($value === '=' && $prev)
  555. {
  556. if (isset($prev['reserved']) && $prev['reserved'] && in_array($prev[1], t(self::$JS_FORBIDDEN)))
  557. {
  558. $this->assignment_error();
  559. }
  560. if (in_array($prev[1], array('||', '&&')))
  561. {
  562. $prev[0] = t('COMPOUND_ASSIGN');
  563. $prev[1] .= '=';
  564. return 1; // strlen($value);
  565. }
  566. }
  567. $map = array(
  568. 'TERMINATOR' => array(';'),
  569. 'MATH' => self::$MATH,
  570. 'COMPARE' => self::$COMPARE,
  571. 'COMPOUND_ASSIGN' => self::$COMPOUND_ASSIGN,
  572. 'UNARY' => self::$UNARY,
  573. 'SHIFT' => self::$SHIFT
  574. );
  575. $mapped = FALSE;
  576. foreach ($map as $k => $v)
  577. {
  578. if (in_array($value, $v))
  579. {
  580. $tag = $k;
  581. $mapped = TRUE;
  582. break;
  583. }
  584. }
  585. if ( ! $mapped)
  586. {
  587. if (in_array($value, self::$LOGIC) || $value === '?' && ($prev && isset($prev['spaced']) && $prev['spaced']))
  588. {
  589. $tag = 'LOGIC';
  590. }
  591. else if ($prev && ! (isset($prev['spaced']) && $prev['spaced']))
  592. {
  593. if ($value === '(' && in_array($prev[0], t(self::$CALLABLE)))
  594. {
  595. if ($prev[0] === t('?'))
  596. {
  597. $prev[0] = t('FUNC_EXIST');
  598. }
  599. $tag = 'CALL_START';
  600. }
  601. else if ($value === '[' && in_array($prev[0], t(self::$INDEXABLE)))
  602. {
  603. $tag = 'INDEX_START';
  604. if ($prev[0] === t('?'))
  605. {
  606. $prev[0] = t('INDEX_SOAK');
  607. }
  608. else if ($prev[0] === t('::'))
  609. {
  610. $prev[0] = t('INDEX_PROTO');
  611. }
  612. }
  613. }
  614. }
  615. $this->token($tag, $value);
  616. return strlen($value);
  617. }
  618. function make_string($body, $quote, $heredoc = NULL)
  619. {
  620. if ( ! $body)
  621. {
  622. return $quote.$quote;
  623. }
  624. $body = preg_replace_callback('/\\\\([\s\S])/', function($match) use ($quote)
  625. {
  626. $contents = $match[1];
  627. if (in_array($contents, array("\n", $quote)))
  628. {
  629. return $contents;
  630. }
  631. return $match[0];
  632. },
  633. $body);
  634. $body = preg_replace('/'.$quote.'/', '\\\\$0', $body);
  635. return $quote.$this->escape_lines($body, $heredoc).$quote;
  636. }
  637. function newline_token()
  638. {
  639. if ($this->tag() !== t('TERMINATOR'))
  640. {
  641. $this->token('TERMINATOR', "\n");
  642. }
  643. }
  644. function number_token()
  645. {
  646. if ( ! preg_match(self::$NUMBER, $this->chunk, $match))
  647. {
  648. return 0;
  649. }
  650. $this->token('NUMBER', $number = $match[0]);
  651. return strlen($number);
  652. }
  653. function outdent_token($move_out, $no_newlines = FALSE, $close = NULL)
  654. {
  655. while ($move_out > 0)
  656. {
  657. $len = count($this->indents) - 1;
  658. if ( ! isset($this->indents[$len]))
  659. {
  660. $move_out = 0;
  661. }
  662. else if ($this->indents[$len] === $this->outdebt)
  663. {
  664. $move_out -= $this->outdebt;
  665. $this->outdebt = 0;
  666. }
  667. else if ($this->indents[$len] < $this->outdebt)
  668. {
  669. $this->outdebt -= $this->indents[$len];
  670. $move_out -= $this->indents[$len];
  671. }
  672. else
  673. {
  674. $dent = array_pop($this->indents) - $this->outdebt;
  675. $move_out -= $dent;
  676. $this->outdebt = 0;
  677. $this->token('OUTDENT', $dent);
  678. }
  679. }
  680. if (isset($dent) && $dent)
  681. {
  682. $this->outdebt -= $move_out;
  683. }
  684. if ( ! ($this->tag() === t('TERMINATOR') || $no_newlines))
  685. {
  686. $this->token('TERMINATOR', "\n");
  687. }
  688. return $this;
  689. }
  690. function regex_token()
  691. {
  692. if ($this->chunk{0} !== '/')
  693. {
  694. return 0;
  695. }
  696. if (preg_match(self::$HEREGEX, $this->chunk, $match))
  697. {
  698. $length = $this->heregex_token($match);
  699. // This seems to be broken in the JavaScript compiler...
  700. // $this->line += substr_count($match[0], "\n");
  701. return $length;
  702. }
  703. $prev = last($this->tokens);
  704. if ($prev)
  705. {
  706. if (in_array($prev[0], t((isset($prev['spaced']) && $prev['spaced']) ?
  707. self::$NOT_REGEX : self::$NOT_SPACED_REGEX)))
  708. {
  709. return 0;
  710. }
  711. }
  712. if ( ! preg_match(self::$REGEX, $this->chunk, $match))
  713. {
  714. return 0;
  715. }
  716. $regex = $match[0];
  717. $this->token('REGEX', $regex === '//' ? '/(?:)/' : $regex);
  718. return strlen($regex);
  719. }
  720. function sanitize_heredoc($doc, array $options)
  721. {
  722. $herecomment = isset($options['herecomment']) ? $options['herecomment'] : NULL;
  723. $indent = isset($options['indent']) ? $options['indent'] : NULL;
  724. if ($herecomment)
  725. {
  726. if (preg_match(self::$HEREDOC_ILLEGAL, $doc))
  727. {
  728. throw new Error('block comment cannot contain \"*/\", starting on line '.($line + 1));
  729. }
  730. if (strpos($doc, "\n") == 0) // No match or 0
  731. {
  732. return $doc;
  733. }
  734. }
  735. else
  736. {
  737. $offset = 0;
  738. while (preg_match(self::$HEREDOC_INDENT, $doc, $match, PREG_OFFSET_CAPTURE, $offset))
  739. {
  740. $attempt = $match[1][0];
  741. $offset = strlen($match[0][0]) + $match[0][1];
  742. if ( is_null($indent) || (strlen($indent) > strlen($attempt) && strlen($attempt) > 0))
  743. {
  744. $indent = $attempt;
  745. }
  746. }
  747. }
  748. if ($indent)
  749. {
  750. $doc = preg_replace('/\n'.$indent.'/', "\n", $doc);
  751. }
  752. if ( ! $herecomment)
  753. {
  754. $doc = preg_replace('/^\n/', '', $doc);
  755. }
  756. return $doc;
  757. }
  758. function string_token()
  759. {
  760. switch ($this->chunk{0})
  761. {
  762. case "'":
  763. if ( ! preg_match(self::$SIMPLESTR, $this->chunk, $match))
  764. {
  765. return 0;
  766. }
  767. $this->token('STRING', preg_replace(self::$MULTILINER, "\\\n", $string = $match[0]));
  768. break;
  769. case '"':
  770. if ( ! ($string = $this->balanced_string($this->chunk, '"')))
  771. {
  772. return 0;
  773. }
  774. if (strpos($string, '#{', 1) > 0)
  775. {
  776. $this->interpolate_string(substr($string, 1, -1));
  777. }
  778. else
  779. {
  780. $this->token('STRING', $this->escape_lines($string));
  781. }
  782. break;
  783. default:
  784. return 0;
  785. }
  786. $this->line += substr_count($string, "\n");
  787. return strlen($string);
  788. }
  789. function suppress_newlines()
  790. {
  791. if ($this->value() === '\\')
  792. {
  793. array_pop($this->tokens);
  794. }
  795. }
  796. function tag($index = 0, $tag = NULL)
  797. {
  798. $token = & last($this->tokens, $index);
  799. if ( ! is_null($tag))
  800. {
  801. $token[0] = $tag;
  802. }
  803. return $token[0];
  804. }
  805. function tag_parameters()
  806. {
  807. if ($this->tag() !== t(')'))
  808. {
  809. return $this;
  810. }
  811. $stack = array();
  812. $tokens = &$this->tokens;
  813. $i = count($tokens);
  814. $tokens[--$i][0] = t('PARAM_END');
  815. while ( ($tok = &$tokens[--$i]) )
  816. {
  817. if ($tok[0] === t(')'))
  818. {
  819. $stack[] = $tok;
  820. }
  821. else if (in_array($tok[0], t('(', 'CALL_START')))
  822. {
  823. if (count($stack))
  824. {
  825. array_pop($stack);
  826. }
  827. else if ($tok[0] === t('('))
  828. {
  829. $tok[0] = t('PARAM_START');
  830. return $this;
  831. }
  832. }
  833. }
  834. return $this;
  835. }
  836. function token($tag, $value = NULL, $props = array())
  837. {
  838. if ( ! is_numeric($tag))
  839. {
  840. $tag = t($tag);
  841. }
  842. $token = array($tag, $value, $this->line);
  843. if ($props)
  844. {
  845. foreach ($props as $k => $v)
  846. {
  847. $token[$k] = $v;
  848. }
  849. }
  850. return ($this->tokens[] = $token);
  851. }
  852. function tokenize()
  853. {
  854. while ( ($this->chunk = substr($this->code, $this->index)) !== FALSE )
  855. {
  856. $types = array('identifier', 'comment', 'whitespace', 'line', 'heredoc',
  857. 'string', 'number', 'regex', 'js', 'literal');
  858. foreach ($types as $type)
  859. {
  860. if ( ($d = $this->{$type.'_token'}()) )
  861. {
  862. $this->index += $d;
  863. break;
  864. }
  865. }
  866. }
  867. $this->close_indentation();
  868. if ($this->options['rewrite'])
  869. {
  870. $rewriter = new Rewriter($this->tokens);
  871. $this->tokens = $rewriter->rewrite();
  872. }
  873. return $this->tokens;
  874. }
  875. function value($index = 0, $value = NULL)
  876. {
  877. $token = & last($this->tokens, $index);
  878. if ( ! is_null($value))
  879. {
  880. $token[1] = $value;
  881. }
  882. return $token[1];
  883. }
  884. function unfinished()
  885. {
  886. return
  887. preg_match(self::$LINE_CONTINUER, $this->chunk) ||
  888. ($prev = last($this->tokens, 1)) &&
  889. ($prev[0] !== t('.')) &&
  890. ($value = $this->value()) &&
  891. // ( ! (isset($value->reserved) && $value->reserved)) &&
  892. ( ! (isset($prev['reserved']) && $prev['reserved'])) &&
  893. preg_match(self::$NO_NEWLINE, $value) &&
  894. ( ! preg_match(self::$CODE, $value)) &&
  895. ( ! preg_match(self::$ASSIGNED, $this->chunk));
  896. }
  897. function whitespace_token()
  898. {
  899. if ( ! (preg_match(self::$WHITESPACE, $this->chunk, $match) || ($nline = ($this->chunk{0} === "\n"))))
  900. {
  901. return 0;
  902. }
  903. $prev = & last($this->tokens);
  904. if ($prev)
  905. {
  906. $prev[$match ? 'spaced' : 'newLine'] = TRUE;
  907. }
  908. return $match ? strlen($match[0]) : 0;
  909. }
  910. }
  911. Lexer::init();
  912. ?>