Add token with data:
* match     - whole matched content
* alt       - alt content for image
* reference - reference name or object

@param string                  $content
@param int                     $position
@param \ViKon\Parser\TokenList $tokenList
328769
protected function handleSingleState($content, $position, TokenList $tokenList) {
        preg_match('/!\\[((?:\\\\.|[^]\\\\])+)\\][\\t ]*\\[((?:\\\\.|[^]\\\\])*)\\]/', $content, $matches);
        $reference = strtolower(empty($matches[2]) ? $matches[1] : $matches[2]);

        $referenceTokens = $tokenList->getTokensByCallback(function (Token $token) use ($reference) {
            return $token->getName() === ReferenceRule::NAME && $token->get('reference', null) === $reference;
        });

        if (($referenceToken = reset($referenceTokens)) !== false) {
            $referenceToken->set('used', true);
            $reference = $referenceToken;
        }

        $tokenList->addToken($this->name, $position)
            ->set('match', $content)
            ->set('alt', $matches[1])
            ->set('reference', $reference);
    }
895634
public function findNextTokenPosition($tokenId, $apos = null) {
		if ($apos === null) {
			$pos = $this->getCurrentPosition();
		} else {
			$pos = $apos;
		}
		$pos += 1; // Start from the token following the current position

		$nbTokens = count($this->tokens);
		while ($pos < $nbTokens) {
			$token = $this->tokens[$pos];

			if ($tokenId === $token->id) {
				return $pos;
			}

			$pos ++;
		}

		return null;
	}
266234
final public void addPosition(int position) {
    if (tokenPosition == null) {
      tokenPosition = new MtasPosition(position);
    } else {
      tokenPosition.add(position);
    }
  }
1152373
private function postInfo(
        string $context,
        string $position,
        Post $post,
        int $count
    ): string {
        return $post->info($this->posts->args[$context][$position])->list();
    }
1311030
public function enterToken(array $token, $content)
    {
        switch ($token[Lexer::TYPE]) {
            case Lexer::T_MARKUP_OPEN_ESCAPED:
                $context              = new MarkupContext($token, $this);
                $context->start       = $token[Lexer::AT];
                $context->end         = $token[Lexer::AT] + $token[Lexer::LENGTH];
                $context->text        = $token[Lexer::VALUE];
                $context->replacement = '{{';

                $this->markups[] = $context;

                break;

            case Lexer::T_MARKUP_OPEN:
                $context        = new MarkupContext($token, $this);
                $context->start = $token[Lexer::AT];

                $this->markups[] = $context;

                return $context;
        }

        return $this;
    }
501561
public function addContentItem($label, $content, $url = null, $status = null, $active = false, array $cssClasses = [], array $attrs = [])
    {
        Checkers::checkUrl($url);
        $status === null || Checkers::checkStatus($status, null, true);
        $url    === null || $this->ulList = false;
        $this->list[] = [(string) $label, $url, (string) $content, $status, (bool) $active, $cssClasses, $attrs];
        $this->listCount++;
        return $this;
    }
1003558
final private void addPositionFromToken(MtasToken token, Long ref)
      throws IOException {
    int prefixId = storePrefixAndTermRef ? token.getPrefixId() : 0;
    if (!closed && (token != null)) {
      if (token.checkPositionType(MtasPosition.POSITION_SINGLE)) {
        addSinglePoint(token.getPositionStart(), prefixId, token.getTermRef(),
            token.getId(), ref);
      } else if (token.checkPositionType(MtasPosition.POSITION_RANGE)) {
        addRange(token.getPositionStart(), token.getPositionEnd(), prefixId,
            token.getTermRef(), token.getId(), ref);
      } else if (token.checkPositionType(MtasPosition.POSITION_SET)) {
        // split set into minimum number of single points and ranges
        SortedMap<Integer, Integer> list = new TreeMap<>();
        int[] positions = token.getPositions();
        Integer lastPoint = null;
        Integer startPoint = null;
        for (int position : positions) {
          if (lastPoint == null) {
            startPoint = position;
            lastPoint = position;
          } else if ((position - lastPoint) != 1) {
            list.put(startPoint, lastPoint);
            startPoint = position;
          }
          lastPoint = position;
        }
        if (lastPoint != null) {
          list.put(startPoint, lastPoint);
        }
        for (Entry<Integer, Integer> entry : list.entrySet()) {
          if (entry.getKey().equals(entry.getValue())) {
            addSinglePoint(entry.getKey(), prefixId, token.getTermRef(),
                token.getId(), ref);
          } else {
            addRange(entry.getKey(), entry.getValue(), prefixId,
                token.getTermRef(), token.getId(), ref);
          }
        }
      }
    }
  }
394942
public function isTokenEqualToOrPrecededBy($tokenPosition, $stringValue)
    {
        return $this->isTokenEqualTo($tokenPosition, $stringValue)
            || $this->isTokenPrecededBy($tokenPosition, $stringValue);
    }
984026
private function validateNewLineAfterDocComment(int $position, array $token, bool $asSingleLine = true): void
    {
        if (!$this->isNextLineEmpty($position)) {
            $nextRelevantPos = $this->loadNextDocBlockContent($position);
            $nextToken = $this->tokens[$nextRelevantPos];

            // Register an error if we force a single line or this is no long description with more then one line.
            if ($asSingleLine || ($nextToken['code'] !== T_DOC_COMMENT_STRING)) {
                $isFixing = $this->file->addFixableWarning(
                    self::MESSAGE_NO_LINE_AFTER_DOC_COMMENT,
                    $position,
                    static::CODE_NO_LINE_AFTER_DOC_COMMENT
                );

                if ($isFixing) {
                    $this->fixNoLineAfterDocComment($position, $token);
                }
            }
        }
    }
370907
private function _parse_actions($content, &$position, $end = '}')
    {
        $result = null;
        $length = strlen($content);

        while ($position < $length) {
            $tokens    = self::tokenize($content, true, $position);
            $separator = array_pop($tokens);
            $token     = !empty($tokens) ? array_shift($tokens) : $separator;

            switch ($token) {
            case 'if':
                // nested 'if' conditions, ignore the whole rule (#5540)
                $this->_parse_actions($content, $position);
                continue 2;

            case 'discard':
            case 'keep':
            case 'stop':
                $result[] = array('type' => $token);
                break;

            case 'fileinto':
            case 'redirect':
                $action  = array('type' => $token, 'target' => array_pop($tokens));
                $args    = array('copy');
                $action += $this->action_arguments($tokens, $args);

                $result[] = $action;
                break;

            case 'vacation':
                $action  = array('type' => 'vacation', 'reason' => array_pop($tokens));
                $args    = array('mime');
                $vargs   = array('seconds', 'days', 'addresses', 'subject', 'handle', 'from');
                $action += $this->action_arguments($tokens, $args, $vargs);

                $result[] = $action;
                break;

            case 'reject':
            case 'ereject':
            case 'setflag':
            case 'addflag':
            case 'removeflag':
                $result[] = array('type' => $token, 'target' => array_pop($tokens));
                break;

            case 'include':
                $action  = array('type' => 'include', 'target' => array_pop($tokens));
                $args    = array('once', 'optional', 'global', 'personal');
                $action += $this->action_arguments($tokens, $args);

                $result[] = $action;
                break;

            case 'set':
                $action  = array('type' => 'set', 'value' => array_pop($tokens), 'name' => array_pop($tokens));
                $args    = array('lower', 'upper', 'lowerfirst', 'upperfirst', 'quotewildcard', 'length');
                $action += $this->action_arguments($tokens, $args);

                $result[] = $action;
                break;

            case 'require':
                // skip, will be build according to used commands
                // $result[] = array('type' => 'require', 'target' => array_pop($tokens));
                break;

            case 'notify':
                $action     = array('type' => 'notify');
                $priorities = array('high' => 1, 'normal' => 2, 'low' => 3);
                $vargs      = array('from', 'id', 'importance', 'options', 'message', 'method');
                $args       = array_keys($priorities);
                $action    += $this->action_arguments($tokens, $args, $vargs);

                // Here we'll convert draft-martin-sieve-notify-01 into RFC 5435
                if (!isset($action['importance'])) {
                    foreach ($priorities as $key => $val) {
                        if (isset($action[$key])) {
                            $action['importance'] = $val;
                            unset($action[$key]);
                        }
                    }
                }

                $action['options'] = (array) $action['options'];

                // Old-draft way: :method "mailto" :options "email@address"
                if (!empty($action['method']) && !empty($action['options'])) {
                    $action['method'] .= ':' . array_shift($action['options']);
                }
                // unnamed parameter is a :method in enotify extension
                else if (!isset($action['method'])) {
                    $action['method'] = array_pop($tokens);
                }

                $result[] = $action;
                break;
            }

            if ($separator == $end) {
                break;
            }
        }

        return $result;
    }
675665
public function renderReference(Token $token, TokenList $tokenList) {
        $reference = $token->get('reference');
        $alt = $token->get('alt');

        if ($reference instanceof Token) {
            $referenceToken = $reference;
        } else {
            $tokens = $tokenList->getTokensByCallback(function (Token $token) use ($reference) {
                return $token->getName() === ReferenceRule::NAME && $token->get('reference', null) === $reference;
            });
            // Get first token (if not found return full match)
            if (($referenceToken = reset($tokens)) === false) {
                return $token->get('match', '');
            }
            $referenceToken->set('used', true);
        }
        $url = $referenceToken->get('url');
        $title = $referenceToken->get('title', '');

        if (empty($title)) {
            return "<img alt=\"$alt\" src=\"$url\"/>";
        }

        return "<img alt=\"$alt\" src=\"$url\" title=\"$title\" />";
    }
19904
protected function handleUnmatchedState($content, $position, TokenList $tokenList) {
        if (!empty($content)) {
            $this->parseContent($content, $tokenList);
        }
    }
1051770
private function addTokenVectorToVector($tokenVector, $field)
    {
        foreach ($tokenVector as $token => $count) {
            $this->addTokenToVector($token, $field, $count);
        }
    }
486029
protected function handleToken($tokenType, string $content)
    {
        if ($tokenType == self::PLAIN_TEXT) {
            if (empty($content)) {
                return;
            }

            $token = [
                self::TOKEN_TYPE    => self::PLAIN_TEXT,
                self::TOKEN_CONTENT => $this->repairPHP($content)
            ];
        } else {
            $token = $this->parseToken($content);
        }

        $this->tokens[] = $token;
    }
336140
public function merge(TokenList $tokenList) {
        $tokenList->closeOpenTokens();
        $lastTokenPosition = $this->last() === null ? 0 : $this->last()
            ->getPosition();
        foreach ($tokenList->getTokens() as $token) {
            $token->setPosition($token->getPosition() + $lastTokenPosition);
            $this->tokens[] = $token;
        }
    }
1008273
private function _tokenize_rule($content, &$position)
    {
        $cond = strtolower(self::tokenize($content, 1, $position));
        if ($cond != 'if' && $cond != 'elsif' && $cond != 'else') {
            return null;
        }

        $disabled = false;
        $join     = false;
        $join_not = false;
        $length   = strlen($content);

        // disabled rule (false + comment): if false # .....
        if (preg_match('/^\s*false\s+#\s*/i', substr($content, $position, 20), $m)) {
            $position += strlen($m[0]);
            $disabled = true;
        }

        while ($position < $length) {
            $tokens    = self::tokenize($content, true, $position);
            $separator = array_pop($tokens);

            if (!empty($tokens)) {
                $token = array_shift($tokens);
            }
            else {
                $token = $separator;
            }

            $token = strtolower($token);

            if ($token == 'not') {
                $not = true;
                $token = strtolower(array_shift($tokens));
            }
            else {
                $not = false;
            }

            // we support "not allof" as a negation of allof sub-tests
            if ($join_not) {
                $not = !$not;
            }

            switch ($token) {
            case 'allof':
                $join     = true;
                $join_not = $not;
                break;

            case 'anyof':
                break;

            case 'size':
                $test = array('test' => 'size', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])
                        && preg_match('/^:(under|over)$/i', $tokens[$i])
                    ) {
                        $test['type'] = strtolower(substr($tokens[$i], 1));
                    }
                }

                $tests[] = $test;
                break;

            case 'header':
            case 'string':
            case 'address':
            case 'envelope':
                $test = array('test' => $token, 'not' => $not);

                $test['arg2'] = array_pop($tokens);
                $test['arg1'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                if ($token != 'header' && $token != 'string' && !empty($tokens)) {
                    for ($i=0, $len=count($tokens); $i<$len; $i++) {
                        if (!is_array($tokens[$i]) && preg_match('/^:(localpart|domain|all|user|detail)$/i', $tokens[$i])) {
                            $test['part'] = strtolower(substr($tokens[$i], 1));
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'body':
                $test = array('test' => 'body', 'not' => $not);

                $test['arg'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:(raw|content|text)$/i', $tokens[$i])) {
                        $test['part'] = strtolower(substr($tokens[$i], 1));

                        if ($test['part'] == 'content') {
                            $test['content'] = $tokens[++$i];
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'date':
            case 'currentdate':
                $test = array('test' => $token, 'not' => $not);

                $test['arg']  = array_pop($tokens);
                $test['part'] = array_pop($tokens);

                if ($token == 'date') {
                    $test['header']  = array_pop($tokens);
                }

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:zone$/i', $tokens[$i])) {
                        $test['zone'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:originalzone$/i', $tokens[$i])) {
                        $test['originalzone'] = true;
                    }
                }

                $tests[] = $test;
                break;

            case 'duplicate':
                $test = array('test' => $token, 'not' => $not);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])) {
                        if (preg_match('/^:(handle|header|uniqueid|seconds)$/i', $tokens[$i], $m)) {
                            $test[strtolower($m[1])] = $tokens[++$i];
                        }
                        else if (preg_match('/^:last$/i', $tokens[$i])) {
                            $test['last'] = true;
                        }
                    }
                }

                $tests[] = $test;
                break;

            case 'exists':
                $tests[] = array('test' => 'exists', 'not' => $not,
                    'arg'  => array_pop($tokens));
                break;

            case 'true':
                $tests[] = array('test' => 'true', 'not' => $not);
                break;

            case 'false':
                $tests[] = array('test' => 'true', 'not' => !$not);
                break;
            }

            // goto actions...
            if ($separator == '{') {
                break;
            }
        }

        // ...and actions block
        $actions = $this->_parse_actions($content, $position);

        if ($tests && $actions) {
            $result = array(
                'type'     => $cond,
                'tests'    => $tests,
                'actions'  => $actions,
                'join'     => $join,
                'disabled' => $disabled,
            );
        }

        return $result;
    }
159501
public static function create(array $tokens, array $content) {
        $content = self::orderBy($tokens, $content);

        return array_map(function($entry) use ($tokens) {
            $row = SelectSingleResult::create($tokens, $entry);
            return array_pop($row);
        }, $content);
    }