/php/PHP_CodeSniffer/src/Tokenizers/Tokenizer.php

Large files files are truncated, but you can click here to view the full file

<?php
/**
 * The base tokenizer class.
 *
 * @author    Greg Sherwood <gsherwood@squiz.net>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Tokenizers;

use PHP_CodeSniffer\Exceptions\RuntimeException;
use PHP_CodeSniffer\Util;

abstract class Tokenizer
{

    /**
     * The config data for the run.
     *
     * @var \PHP_CodeSniffer\Config
     */
    protected $config = null;

    /**
     * The EOL char used in the content.
     *
     * @var string
     */
    protected $eolChar = [];

    /**
     * A token-based representation of the content.
     *
     * @var array
     */
    protected $tokens = [];

    /**
     * The number of tokens in the tokens array.
     *
     * @var integer
     */
    protected $numTokens = 0;

    /**
     * A list of tokens that are allowed to open a scope.
     *
     * @var array
     */
    public $scopeOpeners = [];

    /**
     * A list of tokens that end the scope.
     *
     * @var array
     */
    public $endScopeTokens = [];

    /**
     * Known lengths of tokens.
     *
     * @var array<int, int>
     */
    public $knownLengths = [];

    /**
     * A list of lines being ignored due to error suppression comments.
     *
     * @var array
     */
    public $ignoredLines = [];


    /**
     * Initialise and run the tokenizer.
     *
     * @param string                         $content The content to tokenize,
     * @param \PHP_CodeSniffer\Config | null $config  The config data for the run.
     * @param string                         $eolChar The EOL char used in the content.
     *
     * @return void
     * @throws \PHP_CodeSniffer\Exceptions\TokenizerException If the file appears to be minified.
     */
    public function __construct($content, $config, $eolChar='\n')
    {
        $this->eolChar = $eolChar;

        $this->config = $config;
        $this->tokens = $this->tokenize($content);

        if ($config === null) {
            return;
        }

        $this->createPositionMap();
        $this->createTokenMap();
        $this->createParenthesisNestingMap();
        $this->createScopeMap();
        $this->createLevelMap();

        // Allow the tokenizer to do additional processing if required.
        $this->processAdditional();

    }//end __construct()


    /**
     * Checks the content to see if it looks minified.
     *
     * @param string $content The content to tokenize.
     * @param string $eolChar The EOL char used in the content.
     *
     * @return boolean
     */
    protected function isMinifiedContent($content, $eolChar='\n')
    {
        // Minified files often have a very large number of characters per line
        // and cause issues when tokenizing.
        $numChars = strlen($content);
        $numLines = (substr_count($content, $eolChar) + 1);
        $average  = ($numChars / $numLines);
        if ($average > 100) {
            return true;
        }

        return false;

    }//end isMinifiedContent()


    /**
     * Gets the array of tokens.
     *
     * @return array
     */
    public function getTokens()
    {
        return $this->tokens;

    }//end getTokens()


    /**
     * Creates an array of tokens when given some content.
     *
     * @param string $string The string to tokenize.
     *
     * @return array
     */
    abstract protected function tokenize($string);


    /**
     * Performs additional processing after main tokenizing.
     *
     * @return void
     */
    abstract protected function processAdditional();


    /**
     * Sets token position information.
     *
     * Can also convert tabs into spaces. Each tab can represent between
     * 1 and $width spaces, so this cannot be a straight string replace.
     *
     * @return void
     */
    private function createPositionMap()
    {
        $currColumn = 1;
        $lineNumber = 1;
        $eolLen     = strlen($this->eolChar);
        $ignoring   = null;
        $inTests    = defined('PHP_CODESNIFFER_IN_TESTS');

        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $checkAnnotations = $this->config->annotations;
        $encoding         = $this->config->encoding;
        $tabWidth         = $this->config->tabWidth;

        $tokensWithTabs = [
            T_WHITESPACE               => true,
            T_COMMENT                  => true,
            T_DOC_COMMENT              => true,
            T_DOC_COMMENT_WHITESPACE   => true,
            T_DOC_COMMENT_STRING       => true,
            T_CONSTANT_ENCAPSED_STRING => true,
            T_DOUBLE_QUOTED_STRING     => true,
            T_HEREDOC                  => true,
            T_NOWDOC                   => true,
            T_INLINE_HTML              => true,
        ];

        $this->numTokens = count($this->tokens);
        for ($i = 0; $i < $this->numTokens; $i++) {
            $this->tokens[$i]['line']   = $lineNumber;
            $this->tokens[$i]['column'] = $currColumn;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === true) {
                // There are no tabs in the tokens we know the length of.
                $length      = $this->knownLengths[$this->tokens[$i]['code']];
                $currColumn += $length;
            } else if ($tabWidth === 0
                || isset($tokensWithTabs[$this->tokens[$i]['code']]) === false
                || strpos($this->tokens[$i]['content'], "\t") === false
            ) {
                // There are no tabs in this content, or we aren't replacing them.
                if ($checkEncoding === true) {
                    // Not using the default encoding, so take a bit more care.
                    $oldLevel = error_reporting();
                    error_reporting(0);
                    $length = iconv_strlen($this->tokens[$i]['content'], $encoding);
                    error_reporting($oldLevel);

                    if ($length === false) {
                        // String contained invalid characters, so revert to default.
                        $length = strlen($this->tokens[$i]['content']);
                    }
                } else {
                    $length = strlen($this->tokens[$i]['content']);
                }

                $currColumn += $length;
            } else {
                $this->replaceTabsInToken($this->tokens[$i]);
                $length      = $this->tokens[$i]['length'];
                $currColumn += $length;
            }//end if

            $this->tokens[$i]['length'] = $length;

            if (isset($this->knownLengths[$this->tokens[$i]['code']]) === false
                && strpos($this->tokens[$i]['content'], $this->eolChar) !== false
            ) {
                $lineNumber++;
                $currColumn = 1;

                // Newline chars are not counted in the token length.
                $this->tokens[$i]['length'] -= $eolLen;
            }

            if ($this->tokens[$i]['code'] === T_COMMENT
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_STRING
                || $this->tokens[$i]['code'] === T_DOC_COMMENT_TAG
                || ($inTests === true && $this->tokens[$i]['code'] === T_INLINE_HTML)
            ) {
                $commentText      = ltrim($this->tokens[$i]['content'], " \t/*");
                $commentText      = rtrim($commentText, " */\t\r\n");
                $commentTextLower = strtolower($commentText);
                if (strpos($commentText, '@codingStandards') !== false) {
                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $ownLine = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev >= 0; $prev--) {
                            if ($this->tokens[$prev]['code'] === T_WHITESPACE) {
                                continue;
                            }

                            break;
                        }

                        if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                            $ownLine = true;
                        }
                    }

                    if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreStart') !== false
                    ) {
                        $ignoring = ['.all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }
                    } else if ($ignoring !== null
                        && strpos($commentText, '@codingStandardsIgnoreEnd') !== false
                    ) {
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['.all' => true];
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    } else if ($ignoring === null
                        && strpos($commentText, '@codingStandardsIgnoreLine') !== false
                    ) {
                        $ignoring = ['.all' => true];
                        if ($ownLine === true) {
                            $this->ignoredLines[$this->tokens[$i]['line']]       = $ignoring;
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoring;
                        } else {
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                        }

                        $ignoring = null;
                    }//end if
                } else if (substr($commentTextLower, 0, 6) === 'phpcs:'
                    || substr($commentTextLower, 0, 7) === '@phpcs:'
                ) {
                    // If the @phpcs: syntax is being used, strip the @ to make
                    // comparisons easier.
                    if ($commentText[0] === '@') {
                        $commentText      = substr($commentText, 1);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If there is a comment on the end, strip it off.
                    $commentStart = strpos($commentTextLower, ' --');
                    if ($commentStart !== false) {
                        $commentText      = substr($commentText, 0, $commentStart);
                        $commentTextLower = strtolower($commentText);
                    }

                    // If this comment is the only thing on the line, it tells us
                    // to ignore the following line. If the line contains other content
                    // then we are just ignoring this one single line.
                    $lineHasOtherContent = false;
                    $lineHasOtherTokens  = false;
                    if ($i > 0) {
                        for ($prev = ($i - 1); $prev > 0; $prev--) {
                            if ($this->tokens[$prev]['line'] !== $this->tokens[$i]['line']) {
                                // Changed lines.
                                break;
                            }

                            if ($this->tokens[$prev]['code'] === T_WHITESPACE
                                || ($this->tokens[$prev]['code'] === T_INLINE_HTML
                                && trim($this->tokens[$prev]['content']) === '')
                            ) {
                                continue;
                            }

                            $lineHasOtherTokens = true;

                            if ($this->tokens[$prev]['code'] === T_OPEN_TAG) {
                                continue;
                            }

                            $lineHasOtherContent = true;
                            break;
                        }//end for

                        $changedLines = false;
                        for ($next = $i; $next < $this->numTokens; $next++) {
                            if ($changedLines === true) {
                                // Changed lines.
                                break;
                            }

                            if (isset($this->knownLengths[$this->tokens[$next]['code']]) === false
                                && strpos($this->tokens[$next]['content'], $this->eolChar) !== false
                            ) {
                                // Last token on the current line.
                                $changedLines = true;
                            }

                            if ($next === $i) {
                                continue;
                            }

                            if ($this->tokens[$next]['code'] === T_WHITESPACE
                                || ($this->tokens[$next]['code'] === T_INLINE_HTML
                                && trim($this->tokens[$next]['content']) === '')
                            ) {
                                continue;
                            }

                            $lineHasOtherTokens = true;

                            if ($this->tokens[$next]['code'] === T_CLOSE_TAG) {
                                continue;
                            }

                            $lineHasOtherContent = true;
                            break;
                        }//end for
                    }//end if

                    if (substr($commentTextLower, 0, 9) === 'phpcs:set') {
                        // Ignore standards for complete lines that change sniff settings.
                        if ($lineHasOtherTokens === false) {
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['.all' => true];
                        }

                        // Need to maintain case here, to get the correct sniff code.
                        $parts = explode(' ', substr($commentText, 10));
                        if (count($parts) >= 2) {
                            $sniffParts = explode('.', $parts[0]);
                            if (count($sniffParts) >= 3) {
                                $this->tokens[$i]['sniffCode']          = array_shift($parts);
                                $this->tokens[$i]['sniffProperty']      = array_shift($parts);
                                $this->tokens[$i]['sniffPropertyValue'] = rtrim(implode(' ', $parts), " */\r\n");
                            }
                        }

                        $this->tokens[$i]['code'] = T_PHPCS_SET;
                        $this->tokens[$i]['type'] = 'T_PHPCS_SET';
                    } else if (substr($commentTextLower, 0, 16) === 'phpcs:ignorefile') {
                        // The whole file will be ignored, but at least set the correct token.
                        $this->tokens[$i]['code'] = T_PHPCS_IGNORE_FILE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_IGNORE_FILE';
                    } else if (substr($commentTextLower, 0, 13) === 'phpcs:disable') {
                        if ($lineHasOtherContent === false) {
                            // Completely ignore the comment line.
                            $this->ignoredLines[$this->tokens[$i]['line']] = ['.all' => true];
                        }

                        if ($ignoring === null) {
                            $ignoring = [];
                        }

                        $disabledSniffs = [];

                        $additionalText = substr($commentText, 14);
                        if ($additionalText === false) {
                            $ignoring = ['.all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $sniffCode = trim($sniffCode);
                                $disabledSniffs[$sniffCode] = true;
                                $ignoring[$sniffCode]       = true;

                                // This newly disabled sniff might be disabling an existing
                                // enabled exception that we are tracking.
                                if (isset($ignoring['.except']) === true) {
                                    foreach (array_keys($ignoring['.except']) as $ignoredSniffCode) {
                                        if ($ignoredSniffCode === $sniffCode
                                            || strpos($ignoredSniffCode, $sniffCode.'.') === 0
                                        ) {
                                            unset($ignoring['.except'][$ignoredSniffCode]);
                                        }
                                    }

                                    if (empty($ignoring['.except']) === true) {
                                        unset($ignoring['.except']);
                                    }
                                }
                            }//end foreach
                        }//end if

                        $this->tokens[$i]['code']       = T_PHPCS_DISABLE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_DISABLE';
                        $this->tokens[$i]['sniffCodes'] = $disabledSniffs;
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:enable') {
                        if ($ignoring !== null) {
                            $enabledSniffs = [];

                            $additionalText = substr($commentText, 13);
                            if ($additionalText === false) {
                                $ignoring = null;
                            } else {
                                $parts = explode(',', substr($commentText, 13));
                                foreach ($parts as $sniffCode) {
                                    $sniffCode = trim($sniffCode);
                                    $enabledSniffs[$sniffCode] = true;

                                    // This new enabled sniff might remove previously disabled
                                    // sniffs if it is actually a standard or category of sniffs.
                                    foreach (array_keys($ignoring) as $ignoredSniffCode) {
                                        if ($ignoredSniffCode === $sniffCode
                                            || strpos($ignoredSniffCode, $sniffCode.'.') === 0
                                        ) {
                                            unset($ignoring[$ignoredSniffCode]);
                                        }
                                    }

                                    // This new enabled sniff might be able to clear up
                                    // previously enabled sniffs if it is actually a standard or
                                    // category of sniffs.
                                    if (isset($ignoring['.except']) === true) {
                                        foreach (array_keys($ignoring['.except']) as $ignoredSniffCode) {
                                            if ($ignoredSniffCode === $sniffCode
                                                || strpos($ignoredSniffCode, $sniffCode.'.') === 0
                                            ) {
                                                unset($ignoring['.except'][$ignoredSniffCode]);
                                            }
                                        }
                                    }
                                }//end foreach

                                if (empty($ignoring) === true) {
                                    $ignoring = null;
                                } else {
                                    if (isset($ignoring['.except']) === true) {
                                        $ignoring['.except'] += $enabledSniffs;
                                    } else {
                                        $ignoring['.except'] = $enabledSniffs;
                                    }
                                }
                            }//end if

                            if ($lineHasOtherContent === false) {
                                // Completely ignore the comment line.
                                $this->ignoredLines[$this->tokens[$i]['line']] = ['.all' => true];
                            } else {
                                // The comment is on the same line as the code it is ignoring,
                                // so respect the new ignore rules.
                                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
                            }

                            $this->tokens[$i]['sniffCodes'] = $enabledSniffs;
                        }//end if

                        $this->tokens[$i]['code'] = T_PHPCS_ENABLE;
                        $this->tokens[$i]['type'] = 'T_PHPCS_ENABLE';
                    } else if (substr($commentTextLower, 0, 12) === 'phpcs:ignore') {
                        $ignoreRules = [];

                        $additionalText = substr($commentText, 13);
                        if ($additionalText === false) {
                            $ignoreRules = ['.all' => true];
                        } else {
                            $parts = explode(',', substr($commentText, 13));
                            foreach ($parts as $sniffCode) {
                                $ignoreRules[trim($sniffCode)] = true;
                            }
                        }

                        $this->tokens[$i]['code']       = T_PHPCS_IGNORE;
                        $this->tokens[$i]['type']       = 'T_PHPCS_IGNORE';
                        $this->tokens[$i]['sniffCodes'] = $ignoreRules;

                        if ($ignoring !== null) {
                            $ignoreRules += $ignoring;
                        }

                        if ($lineHasOtherContent === false) {
                            // Completely ignore the comment line, and set the following
                            // line to include the ignore rules we've set.
                            $this->ignoredLines[$this->tokens[$i]['line']]       = ['.all' => true];
                            $this->ignoredLines[($this->tokens[$i]['line'] + 1)] = $ignoreRules;
                        } else {
                            // The comment is on the same line as the code it is ignoring,
                            // so respect the ignore rules it set.
                            $this->ignoredLines[$this->tokens[$i]['line']] = $ignoreRules;
                        }
                    }//end if
                }//end if
            }//end if

            if ($ignoring !== null && isset($this->ignoredLines[$this->tokens[$i]['line']]) === false) {
                $this->ignoredLines[$this->tokens[$i]['line']] = $ignoring;
            }
        }//end for

        // If annotations are being ignored, we clear out all the ignore rules
        // but leave the annotations tokenized as normal.
        if ($checkAnnotations === false) {
            $this->ignoredLines = [];
        }

    }//end createPositionMap()


    /**
     * Replaces tabs in original token content with spaces.
     *
     * Each tab can represent between 1 and $config->tabWidth spaces,
     * so this cannot be a straight string replace. The original content
     * is placed into an orig_content index and the new token length is also
     * set in the length index.
     *
     * @param array  $token    The token to replace tabs inside.
     * @param string $prefix   The character to use to represent the start of a tab.
     * @param string $padding  The character to use to represent the end of a tab.
     * @param int    $tabWidth The number of spaces each tab represents.
     *
     * @return void
     */
    public function replaceTabsInToken(&$token, $prefix=' ', $padding=' ', $tabWidth=null)
    {
        $checkEncoding = false;
        if (function_exists('iconv_strlen') === true) {
            $checkEncoding = true;
        }

        $currColumn = $token['column'];
        if ($tabWidth === null) {
            $tabWidth = $this->config->tabWidth;
            if ($tabWidth === 0) {
                $tabWidth = 1;
            }
        }

        if (rtrim($token['content'], "\t") === '') {
            // String only contains tabs, so we can shortcut the process.
            $numTabs = strlen($token['content']);

            $firstTabSize = ($tabWidth - (($currColumn - 1) % $tabWidth));
            $length       = ($firstTabSize + ($tabWidth * ($numTabs - 1)));
            $newContent   = $prefix.str_repeat($padding, ($length - 1));
        } else {
            // We need to determine the length of each tab.
            $tabs = explode("\t", $token['content']);

            $numTabs    = (count($tabs) - 1);
            $tabNum     = 0;
            $newContent = '';
            $length     = 0;

            foreach ($tabs as $content) {
                if ($content !== '') {
                    $newContent .= $content;
                    if ($checkEncoding === true) {
                        // Not using the default encoding, so take a bit more care.
                        $oldLevel = error_reporting();
                        error_reporting(0);
                        $contentLength = iconv_strlen($content, $this->config->encoding);
                        error_reporting($oldLevel);
                        if ($contentLength === false) {
                            // String contained invalid characters, so revert to default.
                            $contentLength = strlen($content);
                        }
                    } else {
                        $contentLength = strlen($content);
                    }

                    $currColumn += $contentLength;
                    $length     += $contentLength;
                }

                // The last piece of content does not have a tab after it.
                if ($tabNum === $numTabs) {
                    break;
                }

                // Process the tab that comes after the content.
                $lastCurrColumn = $currColumn;
                $tabNum++;

                // Move the pointer to the next tab stop.
                if (($currColumn % $tabWidth) === 0) {
                    // This is the first tab, and we are already at a
                    // tab stop, so this tab counts as a single space.
                    $currColumn++;
                } else {
                    $currColumn++;
                    while (($currColumn % $tabWidth) !== 0) {
                        $currColumn++;
                    }

                    $currColumn++;
                }

                $length     += ($currColumn - $lastCurrColumn);
                $newContent .= $prefix.str_repeat($padding, ($currColumn - $lastCurrColumn - 1));
            }//end foreach
        }//end if

        $token['orig_content'] = $token['content'];
        $token['content']      = $newContent;
        $token['length']       = $length;

    }//end replaceTabsInToken()


    /**
     * Creates a map of brackets positions.
     *
     * @return void
     */
    private function createTokenMap()
    {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START TOKEN MAP ***".PHP_EOL;
        }

        $squareOpeners   = [];
        $curlyOpeners    = [];
        $this->numTokens = count($this->tokens);

        $openers   = [];
        $openOwner = null;

        for ($i = 0; $i < $this->numTokens; $i++) {
            /*
                Parenthesis mapping.
            */

            if (isset(Util\Tokens::$parenthesisOpeners[$this->tokens[$i]['code']]) === true) {
                $this->tokens[$i]['parenthesis_opener'] = null;
                $this->tokens[$i]['parenthesis_closer'] = null;
                $this->tokens[$i]['parenthesis_owner']  = $i;
                $openOwner = $i;
            } else if ($this->tokens[$i]['code'] === T_OPEN_PARENTHESIS) {
                $openers[] = $i;
                $this->tokens[$i]['parenthesis_opener'] = $i;
                if ($openOwner !== null) {
                    $this->tokens[$openOwner]['parenthesis_opener'] = $i;
                    $this->tokens[$i]['parenthesis_owner']          = $openOwner;
                    $openOwner = null;
                }
            } else if ($this->tokens[$i]['code'] === T_CLOSE_PARENTHESIS) {
                // Did we set an owner for this set of parenthesis?
                $numOpeners = count($openers);
                if ($numOpeners !== 0) {
                    $opener = array_pop($openers);
                    if (isset($this->tokens[$opener]['parenthesis_owner']) === true) {
                        $owner = $this->tokens[$opener]['parenthesis_owner'];

                        $this->tokens[$owner]['parenthesis_closer'] = $i;
                        $this->tokens[$i]['parenthesis_owner']      = $owner;
                    }

                    $this->tokens[$i]['parenthesis_opener']      = $opener;
                    $this->tokens[$i]['parenthesis_closer']      = $i;
                    $this->tokens[$opener]['parenthesis_closer'] = $i;
                }
            }//end if

            /*
                Bracket mapping.
            */

            switch ($this->tokens[$i]['code']) {
            case T_OPEN_SQUARE_BRACKET:
                $squareOpeners[] = $i;

                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    echo str_repeat("\t", count($squareOpeners));
                    echo str_repeat("\t", count($curlyOpeners));
                    echo "=> Found square bracket opener at $i".PHP_EOL;
                }
                break;
            case T_OPEN_CURLY_BRACKET:
                if (isset($this->tokens[$i]['scope_closer']) === false) {
                    $curlyOpeners[] = $i;

                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($squareOpeners));
                        echo str_repeat("\t", count($curlyOpeners));
                        echo "=> Found curly bracket opener at $i".PHP_EOL;
                    }
                }
                break;
            case T_CLOSE_SQUARE_BRACKET:
                if (empty($squareOpeners) === false) {
                    $opener = array_pop($squareOpeners);
                    $this->tokens[$i]['bracket_opener']      = $opener;
                    $this->tokens[$i]['bracket_closer']      = $i;
                    $this->tokens[$opener]['bracket_opener'] = $opener;
                    $this->tokens[$opener]['bracket_closer'] = $i;

                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($squareOpeners));
                        echo str_repeat("\t", count($curlyOpeners));
                        echo "\t=> Found square bracket closer at $i for $opener".PHP_EOL;
                    }
                }
                break;
            case T_CLOSE_CURLY_BRACKET:
                if (empty($curlyOpeners) === false
                    && isset($this->tokens[$i]['scope_opener']) === false
                ) {
                    $opener = array_pop($curlyOpeners);
                    $this->tokens[$i]['bracket_opener']      = $opener;
                    $this->tokens[$i]['bracket_closer']      = $i;
                    $this->tokens[$opener]['bracket_opener'] = $opener;
                    $this->tokens[$opener]['bracket_closer'] = $i;

                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", count($squareOpeners));
                        echo str_repeat("\t", count($curlyOpeners));
                        echo "\t=> Found curly bracket closer at $i for $opener".PHP_EOL;
                    }
                }
                break;
            default:
                continue 2;
            }//end switch
        }//end for

        // Cleanup for any openers that we didn't find closers for.
        // This typically means there was a syntax error breaking things.
        foreach ($openers as $opener) {
            unset($this->tokens[$opener]['parenthesis_opener']);
            unset($this->tokens[$opener]['parenthesis_owner']);
        }

        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END TOKEN MAP ***".PHP_EOL;
        }

    }//end createTokenMap()


    /**
     * Creates a map for the parenthesis tokens that surround other tokens.
     *
     * @return void
     */
    private function createParenthesisNestingMap()
    {
        $map = [];
        for ($i = 0; $i < $this->numTokens; $i++) {
            if (isset($this->tokens[$i]['parenthesis_opener']) === true
                && $i === $this->tokens[$i]['parenthesis_opener']
            ) {
                if (empty($map) === false) {
                    $this->tokens[$i]['nested_parenthesis'] = $map;
                }

                if (isset($this->tokens[$i]['parenthesis_closer']) === true) {
                    $map[$this->tokens[$i]['parenthesis_opener']]
                        = $this->tokens[$i]['parenthesis_closer'];
                }
            } else if (isset($this->tokens[$i]['parenthesis_closer']) === true
                && $i === $this->tokens[$i]['parenthesis_closer']
            ) {
                array_pop($map);
                if (empty($map) === false) {
                    $this->tokens[$i]['nested_parenthesis'] = $map;
                }
            } else {
                if (empty($map) === false) {
                    $this->tokens[$i]['nested_parenthesis'] = $map;
                }
            }//end if
        }//end for

    }//end createParenthesisNestingMap()


    /**
     * Creates a scope map of tokens that open scopes.
     *
     * @return void
     * @see    recurseScopeMap()
     */
    private function createScopeMap()
    {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** START SCOPE MAP ***".PHP_EOL;
        }

        for ($i = 0; $i < $this->numTokens; $i++) {
            // Check to see if the current token starts a new scope.
            if (isset($this->scopeOpeners[$this->tokens[$i]['code']]) === true) {
                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $type    = $this->tokens[$i]['type'];
                    $content = Util\Common::prepareForOutput($this->tokens[$i]['content']);
                    echo "\tStart scope map at $i:$type => $content".PHP_EOL;
                }

                if (isset($this->tokens[$i]['scope_condition']) === true) {
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo "\t* already processed, skipping *".PHP_EOL;
                    }

                    continue;
                }

                $i = $this->recurseScopeMap($i);
            }//end if
        }//end for

        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo "\t*** END SCOPE MAP ***".PHP_EOL;
        }

    }//end createScopeMap()


    /**
     * Recurses though the scope openers to build a scope map.
     *
     * @param int $stackPtr The position in the stack of the token that
     *                      opened the scope (eg. an IF token or FOR token).
     * @param int $depth    How many scope levels down we are.
     * @param int $ignore   How many curly braces we are ignoring.
     *
     * @return int The position in the stack that closed the scope.
     */
    private function recurseScopeMap($stackPtr, $depth=1, &$ignore=0)
    {
        if (PHP_CODESNIFFER_VERBOSITY > 1) {
            echo str_repeat("\t", $depth);
            echo "=> Begin scope map recursion at token $stackPtr with depth $depth".PHP_EOL;
        }

        $opener    = null;
        $currType  = $this->tokens[$stackPtr]['code'];
        $startLine = $this->tokens[$stackPtr]['line'];

        // We will need this to restore the value if we end up
        // returning a token ID that causes our calling function to go back
        // over already ignored braces.
        $originalIgnore = $ignore;

        // If the start token for this scope opener is the same as
        // the scope token, we have already found our opener.
        if (isset($this->scopeOpeners[$currType]['start'][$currType]) === true) {
            $opener = $stackPtr;
        }

        for ($i = ($stackPtr + 1); $i < $this->numTokens; $i++) {
            $tokenType = $this->tokens[$i]['code'];

            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                $type    = $this->tokens[$i]['type'];
                $line    = $this->tokens[$i]['line'];
                $content = Util\Common::prepareForOutput($this->tokens[$i]['content']);

                echo str_repeat("\t", $depth);
                echo "Process token $i on line $line [";
                if ($opener !== null) {
                    echo "opener:$opener;";
                }

                if ($ignore > 0) {
                    echo "ignore=$ignore;";
                }

                echo "]: $type => $content".PHP_EOL;
            }//end if

            // Very special case for IF statements in PHP that can be defined without
            // scope tokens. E.g., if (1) 1; 1 ? (1 ? 1 : 1) : 1;
            // If an IF statement below this one has an opener but no
            // keyword, the opener will be incorrectly assigned to this IF statement.
            // The same case also applies to USE statements, which don't have to have
            // openers, so a following USE statement can cause an incorrect brace match.
            if (($currType === T_IF || $currType === T_ELSE || $currType === T_USE)
                && $opener === null
                && ($this->tokens[$i]['code'] === T_SEMICOLON
                || $this->tokens[$i]['code'] === T_CLOSE_TAG)
            ) {
                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $type = $this->tokens[$stackPtr]['type'];
                    echo str_repeat("\t", $depth);
                    if ($this->tokens[$i]['code'] === T_SEMICOLON) {
                        $closerType = 'semicolon';
                    } else {
                        $closerType = 'close tag';
                    }

                    echo "=> Found $closerType before scope opener for $stackPtr:$type, bailing".PHP_EOL;
                }

                return $i;
            }

            // Special case for PHP control structures that have no braces.
            // If we find a curly brace closer before we find the opener,
            // we're not going to find an opener. That closer probably belongs to
            // a control structure higher up.
            if ($opener === null
                && $ignore === 0
                && $tokenType === T_CLOSE_CURLY_BRACKET
                && isset($this->scopeOpeners[$currType]['end'][$tokenType]) === true
            ) {
                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    $type = $this->tokens[$stackPtr]['type'];
                    echo str_repeat("\t", $depth);
                    echo "=> Found curly brace closer before scope opener for $stackPtr:$type, bailing".PHP_EOL;
                }

                return ($i - 1);
            }

            if ($opener !== null
                && (isset($this->tokens[$i]['scope_opener']) === false
                || $this->scopeOpeners[$this->tokens[$stackPtr]['code']]['shared'] === true)
                && isset($this->scopeOpeners[$currType]['end'][$tokenType]) === true
            ) {
                if ($ignore > 0 && $tokenType === T_CLOSE_CURLY_BRACKET) {
                    // The last opening bracket must have been for a string
                    // offset or alike, so let's ignore it.
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        echo str_repeat("\t", $depth);
                        echo '* finished ignoring curly brace *'.PHP_EOL;
                    }

                    $ignore--;
                    continue;
                } else if ($this->tokens[$opener]['code'] === T_OPEN_CURLY_BRACKET
                    && $tokenType !== T_CLOSE_CURLY_BRACKET
                ) {
                    // The opener is a curly bracket so the closer must be a curly bracket as well.
                    // We ignore this closer to handle cases such as T_ELSE or T_ELSEIF being considered
                    // a closer of T_IF when it should not.
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $type = $this->tokens[$stackPtr]['type'];
                        echo str_repeat("\t", $depth);
                        echo "=> Ignoring non-curly scope closer for $stackPtr:$type".PHP_EOL;
                    }
                } else {
                    $scopeCloser = $i;
                    $todo        = [
                        $stackPtr,
                        $opener,
                    ];

                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $type       = $this->tokens[$stackPtr]['type'];
                        $closerType = $this->tokens[$scopeCloser]['type'];
                        echo str_repeat("\t", $depth);
                        echo "=> Found scope closer ($scopeCloser:$closerType) for $stackPtr:$type".PHP_EOL;
                    }

                    $validCloser = true;
                    if (($this->tokens[$stackPtr]['code'] === T_IF || $this->tokens[$stackPtr]['code'] === T_ELSEIF)
                        && ($tokenType === T_ELSE || $tokenType === T_ELSEIF)
                    ) {
                        // To be a closer, this token must have an opener.
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo str_repeat("\t", $depth);
                            echo "* closer needs to be tested *".PHP_EOL;
                        }

                        $i = self::recurseScopeMap($i, ($depth + 1), $ignore);

                        if (isset($this->tokens[$scopeCloser]['scope_opener']) === false) {
                            $validCloser = false;
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                echo str_repeat("\t", $depth);
                                echo "* closer is not valid (no opener found) *".PHP_EOL;
                            }
                        } else if ($this->tokens[$this->tokens[$scopeCloser]['scope_opener']]['code'] !== $this->tokens[$opener]['code']) {
                            $validCloser = false;
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                echo str_repeat("\t", $depth);
                                $type       = $this->tokens[$this->tokens[$scopeCloser]['scope_opener']]['type'];
                                $openerType = $this->tokens[$opener]['type'];
                                echo "* closer is not valid (mismatched opener type; $type != $openerType) *".PHP_EOL;
                            }
                        } else if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo str_repeat("\t", $depth);
                            echo "* closer was valid *".PHP_EOL;
                        }
                    } else {
                        // The closer was not processed, so we need to
                        // complete that token as well.
                        $todo[] = $scopeCloser;
                    }//end if

                    if ($validCloser === true) {
                        foreach ($todo as $token) {
                            $this->tokens[$token]['scope_condition'] = $stackPtr;
                            $this->tokens[$token]['scope_opener']    = $opener;
                            $this->tokens[$token]['scope_closer']    = $scopeCloser;
                        }

                        if ($this->scopeOpeners[$this->tokens[$stackPtr]['code']]['shared'] === true) {
                            // As we are going back to where we started originally, restore
                            // the ignore value back to its original value.
                            $ignore = $originalIgnore;
                            return $opener;
                        } else if ($scopeCloser === $i
                            && isset($this->scopeOpeners[$tokenType]) === true
                        ) {
                            // Unset scope_condition here or else the token will appear to have
                            // already been processed, and it will be skipped. Normally we want that,
                            // but in this case, the token is both a closer and an opener, so
                            // it needs to act like an opener. This is also why we return the
                            // token before this one; so the closer has a chance to be processed
                            // a second time, but as an opener.
                            unset($this->tokens[$scopeCloser]['scope_condition']);
                            return ($i - 1);
                        } else {
                            return $i;
                        }
                    } else {
                        continue;
                    }//end if
                }//end if
            }//end if

            // Is this an opening condition ?
            if (isset($this->scopeOpeners[$tokenType]) === true) {
                if ($opener === null) {
                    if ($tokenType === T_USE) {
                        // PHP use keywords are special because they can be
                        // used as blocks but also inline in function definitions.
                        // So if we find them nested inside another opener, just skip them.
                        continue;
                    }

                    if ($tokenType === T_FUNCTION
                        && $this->tokens[$stackPtr]['code'] !== T_FUNCTION
                    ) {
                        // Probably a closure, so process it manually.
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokens[$stackPtr]['type'];
                            echo str_repeat("\t", $depth);
                            echo "=> Found function before scope opener for $stackPtr:$type, processing manually".PHP_EOL;
                        }

                        if (isset($this->tokens[$i]['scope_closer']) === true) {
                            // We've already processed this closure.
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                echo str_repeat("\t", $depth);
                                echo '* already processed, skipping *'.PHP_EOL;
                            }

                            $i = $this->tokens[$i]['scope_closer'];
                            continue;
                        }

                        $i = self::recurseScopeMap($i, ($depth + 1), $ignore);
                        continue;
                    }//end if

                    if ($tokenType === T_CLASS) {
                        // Probably an anonymous class inside another anonymous class,
                        // so process it manually.
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            $type = $this->tokens[$stackPtr]['type'];
                            echo str_repeat("\t", $depth);
                            echo "=> Found class before scope opener for $stackPtr:$type, processing manually".PHP_EOL;
                        }

                        if (isset($this->tokens[$i]['scope_closer']) === true) {
                            // We've already processed this anon class.
                            if (PHP_CODESNIFFER_VERBOSITY > 1) {
                                echo str_repeat("\t", $depth);
                                echo '* already processed, skipping *'.PHP_EOL;
                            }

                            $i = $this->tokens[$i]['scope_closer'];
                            continue;
                        }

                        $i = self::recurseScopeMap($i, ($depth + 1), $ignore);
                        continue;
                    }//end if

                    // Found another opening condition but still haven't
                    // found our opener, so we are never going to find one.
                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
                        $type = $this->tokens[$stackPtr]['type'];
                        echo str_repeat("\t", $depth);
                        echo "=> Found new opening condition before scope opener for $stackPtr:$type, ";
                    }

                    if (($this->tokens[$stackPtr]['code'] === T_IF
                        || $this->tokens[$stackPtr]['code'] === T_ELSEIF
                        || $this->tokens[$stackPtr]['code'] === T_ELSE)
                        && ($this->tokens[$i]['code'] === T_ELSE
                        || $this->tokens[$i]['code'] === T_ELSEIF)
                    ) {
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "continuing".PHP_EOL;
                        }

                        return ($i - 1);
                    } else {
                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
                            echo "backtracking".PHP_EOL;
                        }

                        return $stackPtr;
                    }
                }//end if

                if (PHP_CODESNIFFER_VERBOSITY > 1) {
                    echo str_repeat("\t", $depth);
                    echo '* token is an opening condition *'.PHP…