Lexes an expression that is delimited with tags

@param string $closeDelimiter The close delimiter
321497
private function lexDelimitedExpression(string $closeDelimiter)
    {
        $expressionBuffer = '';
        $newLinesAfterExpression = 0;

        while (!$this->matches($closeDelimiter, false) && !$this->atEof()) {
            $currentChar = $this->getCurrentChar();

            if ($currentChar === "\n") {
                if (trim($expressionBuffer) === '') {
                    $this->line++;
                } else {
                    $newLinesAfterExpression++;
                }
            }

            $expressionBuffer .= $currentChar;
            $this->cursor++;
        }

        $expressionBuffer = trim($expressionBuffer);

        if ($expressionBuffer !== '') {
            $this->tokens[] = new Token(
                TokenTypes::T_EXPRESSION,
                $this->replaceViewFunctionCalls($expressionBuffer),
                $this->line
            );
            $this->line += $newLinesAfterExpression;
        }
    }
167775
public function setClosed($id, $closed = true)
    {
        return $this->put($this->getPath().'/'.rawurlencode($id).'/closed', array('value' => $closed));
    }
264948
protected function getKeywordsForRegex(array $keywords, $delimiter = '/')
    {
        // escape keywords for use in regex
        $delimiter = array_fill(0, count($keywords), $delimiter);
        $escaped = array_map('preg_quote', $keywords, $delimiter);

        // add word boundaries
        array_walk($keywords, function ($value) {
            return '\b'.$value.'\b';
        });

        $keywords = array_combine($keywords, $escaped);

        return $keywords;
    }
245314
public function finish(Lexer $lexer) {
        if (is_array($this->exitPattern)) {
            foreach ($this->exitPattern as $exitPattern) {
                $lexer->addExitPattern($exitPattern, $this->name);
            }
        } else {
            $lexer->addExitPattern($this->exitPattern, $this->name);
        }

        return $this;
    }
491259
private function setDelimiters($delimiters)
    {
        list($otag, $ctag) = explode(' ', $delimiters);
        $this->otag = $otag;
        $this->ctag = $ctag;
        $this->otagLen = strlen($otag);
        $this->ctagLen = strlen($ctag);
    }
1095635
@Override
  void visitEnclosingElement(Tokenizer.Token value) throws IOException {
    printIndents();
    write("<" + value.getEditsElement().toString() + ">\n");
    tagQ.push(value.getEditsElement());
    depthCounter.incLevel();
  }
60478
protected function _setTokenStart($delimiter)
    {
        if (!is_null($delimiter)) {
            $delimiter = $this->_normalizeTokenDelimiter($delimiter);
        }

        $this->tokenStart = $delimiter;
    }
985468
public function _setTokenStart($delimiter)
    {
        if (
            !($delimiter instanceof Stringable) &&
            !is_string($delimiter) &&
            !is_null($delimiter)
        ) {
            throw $this->_createInvalidArgumentException($this->__('Invalid token start delimiter'), null, null, $delimiter);
        }

        $this->tokenStart = $delimiter;
    }
696332
public function embedInto($parentRuleName, Lexer $lexer) {
        if (is_array($this->pattern)) {
            foreach ($this->pattern as $pattern) {
                $lexer->addSinglePattern($pattern, $parentRuleName, $this->name);
            }
        } else {
            $lexer->addSinglePattern($this->pattern, $parentRuleName, $this->name);
        }

        return $this;
    }
538500
public function lex()
    {
        // TODO: Sometimes, static::parse* functions make unnecessary calls to
        // is* functions. For a better performance, some rules can be deduced
        // from context.
        // For example, in `parseBool` there is no need to compare the token
        // every time with `true` and `false`. The first step would be to
        // compare with 'true' only and just after that add another letter from
        // context and compare again with `false`.
        // Another example is `parseComment`.

        $list = new TokensList();

        /**
         * Last processed token.
         *
         * @var Token
         */
        $lastToken = null;

        for ($this->last = 0, $lastIdx = 0; $this->last < $this->len; $lastIdx = ++$this->last) {
            /**
             * The new token.
             *
             * @var Token
             */
            $token = null;

            foreach (static::$PARSER_METHODS as $method) {
                if ($token = $this->$method()) {
                    break;
                }
            }

            if ($token === null) {
                // @assert($this->last === $lastIdx);
                $token = new Token($this->str[$this->last]);
                $this->error(
                    'Unexpected character.',
                    $this->str[$this->last],
                    $this->last
                );
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_SYMBOL
                && $token->flags & Token::FLAG_SYMBOL_VARIABLE
                && (
                    $lastToken->type === Token::TYPE_STRING
                    || (
                        $lastToken->type === Token::TYPE_SYMBOL
                        && $lastToken->flags & Token::FLAG_SYMBOL_BACKTICK
                    )
                )
            ) {
                // Handles ```... FROM 'user'@'%' ...```.
                $lastToken->token .= $token->token;
                $lastToken->type = Token::TYPE_SYMBOL;
                $lastToken->flags = Token::FLAG_SYMBOL_USER;
                $lastToken->value .= '@' . $token->value;
                continue;
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_KEYWORD
                && $lastToken->type === Token::TYPE_OPERATOR
                && $lastToken->value === '.'
            ) {
                // Handles ```... tbl.FROM ...```. In this case, FROM is not
                // a reserved word.
                $token->type = Token::TYPE_NONE;
                $token->flags = 0;
                $token->value = $token->token;
            }

            $token->position = $lastIdx;

            $list->tokens[$list->count++] = $token;

            // Handling delimiters.
            if ($token->type === Token::TYPE_NONE && $token->value === 'DELIMITER') {
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected whitespace(s) before delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }

                // Skipping last R (from `delimiteR`) and whitespaces between
                // the keyword `DELIMITER` and the actual delimiter.
                $pos = ++$this->last;
                if (($token = $this->parseWhitespace()) !== null) {
                    $token->position = $pos;
                    $list->tokens[$list->count++] = $token;
                }

                // Preparing the token that holds the new delimiter.
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }
                $pos = $this->last + 1;

                // Parsing the delimiter.
                $this->delimiter = null;
                $delimiterLen = 0;
                while (++$this->last < $this->len && ! Context::isWhitespace($this->str[$this->last]) && $delimiterLen < 15) {
                    $this->delimiter .= $this->str[$this->last];
                    ++$delimiterLen;
                }

                if (empty($this->delimiter)) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last
                    );
                    $this->delimiter = ';';
                }

                --$this->last;

                // Saving the delimiter and its token.
                $this->delimiterLen = strlen($this->delimiter);
                $token = new Token($this->delimiter, Token::TYPE_DELIMITER);
                $token->position = $pos;
                $list->tokens[$list->count++] = $token;
            }

            $lastToken = $token;
        }

        // Adding a final delimiter to mark the ending.
        $list->tokens[$list->count++] = new Token(null, Token::TYPE_DELIMITER);

        // Saving the tokens list.
        $this->list = $list;
    }
1334092
public final void mDELIMITER() throws RecognitionException {
		try {
			int _type = DELIMITER;
			int _channel = DEFAULT_TOKEN_CHANNEL;
			// druidG.g:590:17: ( ( 'DELIMITER' | 'delimiter' ) )
			// druidG.g:590:18: ( 'DELIMITER' | 'delimiter' )
			{
			// druidG.g:590:18: ( 'DELIMITER' | 'delimiter' )
			int alt7=2;
			int LA7_0 = input.LA(1);
			if ( (LA7_0=='D') ) {
				alt7=1;
			}
			else if ( (LA7_0=='d') ) {
				alt7=2;
			}

			else {
				NoViableAltException nvae =
					new NoViableAltException("", 7, 0, input);
				throw nvae;
			}

			switch (alt7) {
				case 1 :
					// druidG.g:590:19: 'DELIMITER'
					{
					match("DELIMITER"); 

					}
					break;
				case 2 :
					// druidG.g:590:31: 'delimiter'
					{
					match("delimiter"); 

					}
					break;

			}

			}

			state.type = _type;
			state.channel = _channel;
		}
		finally {
			// do for sure before leaving
		}
	}
412746
public function tokenize(Cursor $cursor)
    {
        while ($cursor->hasNext()) {
            foreach ($this->skips as $skip) {
                if ($token = $skip->match($cursor->src())) {
                    $cursor->consume($token);

                    continue 2;
                }
            }

            foreach ($this->rules as $rule) {
                if ($token = $rule->match($cursor->src())) {
                    $cursor->consume($token);
                    $cursor->add($rule->getType(), $rule->format($token));

                    continue 2;
                }
            }

            throw new LexerException(sprintf('Can\'t tokenize the following expression: "%s".', $cursor->src()));
        }

        $cursor->add(Token::TYPE_EOF);

        return $cursor->getTokens();
    }
1195862
public function closeDictionaryFiles()
    {
        $this->_tisFile->seek(4);
        $this->_tisFile->writeLong($this->_termCount);

        $this->_tiiFile->seek(4);
        // + 1 is used to count an additional special index entry (empty term at the start of the list)
        $this->_tiiFile->writeLong(($this->_termCount - $this->_termCount % self::$indexInterval)/self::$indexInterval + 1);
    }
59812
public static function multiExplode($string, array $delimiters, $keepDelimiters = true)
    {
        if ($keepDelimiters === true) {
            // Ensure special characters are escaped
            foreach ($delimiters as &$delimiter) {
                $delimiter = preg_quote($delimiter);
            }

            // RegEx fragment with delimiters
            $fragment = implode('|', $delimiters);
            $regEx = sprintf('@(?<=%s)@', $fragment);
            return preg_split($regEx, $string, -1, PREG_SPLIT_DELIM_CAPTURE);

        } else {

            $string = str_replace($delimiters, $delimiters[0], $string);
            return explode($delimiters[0], $string);
        }
    }
1340631
public function tokens(&$source)
    {
        // literal from source to array
        $source = \preg_replace_callback(
            "~{$this->openLiteralRegExp}(\X*?){$this->closeLiteralRegExp}~u",
            [$this, 'literal'],
            $source
        );

        // if check literal open then throw
        if (\preg_match("~{$this->openLiteralRegExp}~u", $source))
        {
            throw new Exceptions\Logic('Literal isn\'t closed');
        }

        // if check literal close then throw
        if (\preg_match("~{$this->closeLiteralRegExp}~u", $source))
        {
            throw new Exceptions\Logic('Literal isn\'t open');
        }

        // remove comments
        $source  = \preg_replace('~\{(?<q>\*)\X*?(\k<q>)\}~u', '', $source);
        $source  = \strtr($source, $this->phpTags); // remove php tags
        $lexCode = \preg_replace('~("|\'|#|\/{2}|\/\*)~u', '?>$1<?php ', $source);

        // analysis tokens
        return $this->analysis(
        // source progress with helped tokenizer
            \token_get_all('<?php' . PHP_EOL . $lexCode)
        );
    }
381945
public function _setTokenEnd($delimiter)
    {
        if (
            !($delimiter instanceof Stringable) &&
            !is_string($delimiter) &&
            !is_null($delimiter)
        ) {
            throw $this->_createInvalidArgumentException($this->__('Invalid token start delimiter'), null, null, $delimiter);
        }

        $this->tokenEnd = $delimiter;
    }
160753
public function explode($delimiter, $limit = null)
    {
        // Fix HHVM default explode limit issue
        // @see  https://github.com/facebook/hhvm/issues/7696
        // @see  https://3v4l.org/fllad
        if ($limit === null) {
            $limit = defined('HHVM_VERSION') ? 0x7FFFFFFF : PHP_INT_MAX;
        }

        return explode($delimiter, $this->string, $limit);
    }