Implementation of “concatenation”.

@param LookaheadIterator $tokens
@param string|null $pNodeId
@return string|int|null
@throws GrammarException
435546
protected function concatenation(LookaheadIterator $tokens, &$pNodeId)
    {
        $children = [];

        // repetition() …
        $rule = $this->repetition($tokens, $pNodeId);

        if ($rule === null) {
            return null;
        }

        $children[] = $rule;
        $others = false;

        // … repetition()*
        while (null !== $r1 = $this->repetition($tokens, $pNodeId)) {
            $children[] = $r1;
            $others = true;
        }

        if ($others === false && $pNodeId === null) {
            return $rule;
        }

        $name = $this->transitionalRuleCounter++;

        $this->parsedRules[$name] = new Concatenation($name, $children);

        return $name;
    }
802121
function concatenate(node, index, parent) {
        if (
            parent &&
            (parent.type === 'ParagraphNode' || parent.type === 'RootNode') &&
            queue.indexOf(parent) === -1
        ) {
            queue.push(parent);
        }
    }
797633
private function optimize_regexp_list_tokens_to_string(&$tokens, $recursed = false)
    {
        $list = '';
        foreach ($tokens as $token => $sub_tokens) {
            $list .= $token;
            $close_entry = isset($sub_tokens['']);
            unset($sub_tokens['']);
            if (!empty($sub_tokens)) {
                $list .= '(?:' . $this->optimize_regexp_list_tokens_to_string($sub_tokens, true) . ')';
                if ($close_entry) {
                    // make sub_tokens optional
                    $list .= '?';
                }
            }
            $list .= '|';
        }
        if (!$recursed) {
            // do some optimizations
            // common trailing strings
            // BUGGY!
            //$list = preg_replace_callback('#(?<=^|\:|\|)\w+?(\w+)(?:\|.+\1)+(?=\|)#', create_function(
            //    '$matches', 'return "(?:" . preg_replace("#" . preg_quote($matches[1], "#") . "(?=\||$)#", "", $matches[0]) . ")" . $matches[1];'), $list);
            // (?:p)? => p?
            $list = preg_replace('#\(\?\:(.)\)\?#', '\1?', $list);
            // (?:a|b|c|d|...)? => [abcd...]?
            // TODO: a|bb|c => [ac]|bb
            static $callback_2;
            if (!isset($callback_2)) {
                $callback_2 = create_function('$matches', 'return "[" . str_replace("|", "", $matches[1]) . "]";');
            }
            $list = preg_replace_callback('#\(\?\:((?:.\|)+.)\)#', $callback_2, $list);
        }

        // return $list without trailing pipe
        return substr($list, 0, -1);
    }
1050791
protected function isMatchingReturn($tokens, $returnPos)
    {
        do {
            $returnPos++;
        } while ($tokens[$returnPos]['code'] === T_WHITESPACE);

        return $tokens[$returnPos]['code'] !== T_SEMICOLON;
    }
746980
public function process($tokens, $keys = array()) {

        $base_expr = "";
        $expr = array();
        $currCategory = "";

        if ($this->isStatement($keys)) {
            foreach ($tokens as $token) {

                $trim = trim($token);
                $base_expr .= $token;

                if ($trim === '') {
                    continue;
                }

                $upper = strtoupper($trim);

                switch ($upper) {

                case 'EXTENDED':
                case 'PARTITIONS':
                    return array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $token);
                    break;

                case 'FORMAT':
                    if ($currCategory === '') {
                        $currCategory = $upper;
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case '=':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::OPERATOR, 'base_expr' => $trim);
                    }
                    // else?
                    break;

                case 'TRADITIONAL':
                case 'JSON':
                    if ($currCategory === 'FORMAT') {
                        $expr[] = array('expr_type' => ExpressionType::RESERVED, 'base_expr' => $trim);
                        return array('expr_type' => ExpressionType::EXPRESSION, 'base_expr' => trim($base_expr),
                                     'sub_tree' => $expr);
                    }
                    // else?
                    break;

                default:
                // ignore the other stuff
                    break;
                }
            }
            return empty($expr) ? null : $expr;
        }

        foreach ($tokens as $token) {

            $trim = trim($token);

            if ($trim === '') {
                continue;
            }

            switch ($currCategory) {

            case 'TABLENAME':
                $currCategory = 'WILD';
                $expr[] = array('expr_type' => ExpressionType::COLREF, 'base_expr' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim));
                break;

            case '':
                $currCategory = 'TABLENAME';
                $expr[] = array('expr_type' => ExpressionType::TABLE, 'table' => $trim,
                                'no_quotes' => $this->revokeQuotation($trim), 'alias' => false, 'base_expr' => $trim);
                break;

            default:
                break;
            }
        }
        return empty($expr) ? null : $expr;
    }
1164306
protected function isShortArraySyntax(array $tokens, Node\Expr\Array_ $array)
    {
        $i = $array->getAttribute('startTokenPos');
        return is_string($tokens[$i]);
    }
792916
private function doParse(array $tokens, &$i) {

    $id = ParserUtil::nextSubstantialIncl($tokens, $i);

    if (T_USE === $id) {
      return ParserUtil::parseUseStatementGroup($tokens, $i);
    }
    elseif (T_NAMESPACE === $id) {
      return ParserUtil::parseNamespaceDeclaration($tokens, $i);
    }
    elseif (T_FINAL === $id || T_ABSTRACT === $id) {
      if (T_DOC_COMMENT === $tokens[$i - 2][0]) {
        $docComment = $tokens[$i - 2][1];
      }
      else {
        $docComment = NULL;
      }
      $modifiers = array($id => TRUE);
      while (TRUE) {
        $id = ParserUtil::nextSubstantialExcl($tokens, $i);
        $modifiers[$id] = TRUE;
        if (T_CLASS === $id || T_INTERFACE === $id || T_TRAIT === $id) {
          return $this->classLikeParser->parse($tokens, $i, $modifiers, $docComment);
        }
        elseif (T_FINAL === $id || T_ABSTRACT === $id) {
          $modifiers[$id] = TRUE;
        }
        else {
          // Something other than the above does not belong here.
          return FALSE;
        }
      }
      throw new \RuntimeException('Unreachable code.');
    }
    elseif (T_CLASS === $id || T_INTERFACE === $id || T_TRAIT === $id) {
      if (T_DOC_COMMENT === $tokens[$i - 2][0]) {
        $docComment = $tokens[$i - 2][1];
      }
      else {
        $docComment = NULL;
      }
      return $this->classLikeParser->parse($tokens, $i, array($id => TRUE), $docComment);
    }
    elseif ('#' === $id) {
      return FALSE;
    }
    else {
      return self::ignoreStatement($tokens, $i);
    }
  }
188388
public static String[] decodeLemmas(final String[] tokens,
      final Span[] preds) {
    final List<String> lemmas = new ArrayList<>();
    for (final Span span : preds) {
      String lemma = decodeShortestEditScript(
          span.getCoveredText(tokens).toLowerCase(), span.getType());
      // System.err.println("-> DEBUG: " + toks[i].toLowerCase() + " " +
      // preds[i] + " " + lemma);
      if (lemma.length() == 0) {
        lemma = "_";
      }
      lemmas.add(lemma);
    }
    return lemmas.toArray(new String[lemmas.size()]);
  }
895634
public function findNextTokenPosition($tokenId, $apos = null) {
		if ($apos === null) {
			$pos = $this->getCurrentPosition();
		} else {
			$pos = $apos;
		}
		$pos += 1; // Start from the token following the current position

		$nbTokens = count($this->tokens);
		while ($pos < $nbTokens) {
			$token = $this->tokens[$pos];

			if ($tokenId === $token->id) {
				return $pos;
			}

			$pos ++;
		}

		return null;
	}
992201
private TokenDefinition getTokenDefinition(ParseTreeNode tokenDefinition,
	    Map<String, ParseTreeNode> helpers, Map<String, ParseTreeNode> tokens)
	    throws GrammarException, TreeException {
	String tokenName = tokenDefinition.getChild("IDENTIFIER").getText();
	String pattern = createTokenDefinitionPattern(tokenDefinition, helpers,
		tokens);
	boolean ignoreCase = Boolean.valueOf((String) options
		.get("grammar.ignore-case"));
	if (tokenVisibility.get(tokenName) != null) {
	    return new TokenDefinition(tokenName, pattern,
		    tokenVisibility.get(tokenName), ignoreCase);
	} else {
	    return new TokenDefinition(tokenName, pattern, ignoreCase);
	}
    }
712975
private function parseNamespace($tokens)
    {
        $namespace = null;
        $tokenCount = count($tokens);
        for ($offset=0; $offset < $tokenCount; $offset++) {
            if (!is_array($tokens[$offset])) {
                continue;
            }

            if (T_NAMESPACE === $tokens[$offset][0]) {
                $offset++; // the next token is a whitespace

                return $this->parseNamespaceString($tokens, $offset);
            }
        }

        return $namespace;
    }
814976
def tokens2text(docgraph, token_ids):
    
    return ' '.join(docgraph.node[token_id][docgraph.ns+':token']
                    for token_id in token_ids)
222832
public function parse(array $tokens) : int
    {
        $tokensAvailable = array_flip($this->getGrammar()->getTokens());

        $length = count($tokens);

        if ($length === 0) {
            throw new Exception('Invalid Roman', Exception::INVALID_ROMAN);
        }

        foreach ($tokens as $position => $token) {
            if (! is_string($token)) {
                $exception = new Exception(
                    sprintf('Invalid token type "%s" at position %d', gettype($token), $position),
                    Exception::INVALID_TOKEN_TYPE
                );

                $exception->setPosition($position);

                throw $exception;
            }

            if (! isset($tokensAvailable[$token])) {
                $exception = new Exception(
                    sprintf('Unknown token "%s" at position %d', $token, $position),
                    Exception::UNKNOWN_TOKEN
                );

                $exception
                    ->setToken($token)
                    ->setPosition($position);

                throw $exception;
            }
        }

        return (new Automaton($this->getGrammar()))->read($tokens)->getValue();
    }
250427
protected function formatOpeningParenthesis($token, $i, array &$tokens, array &$originalTokens)
    {
        $length = 0;
        for ($j = 1; $j <= 250; ++$j) {
            if (isset($tokens[$i + $j])) {
                $next = $tokens[$i + $j];
                if ($this->parentheses->stringIsClosingParentheses($next)) {
                    $this->parentheses->writeNewInlineParentheses();
                    break;
                }

                if ($this->parentheses->invalidParenthesesTokenValue($next)
                    || $this->parentheses->invalidParenthesesTokenType($next)
                ) {
                    break;
                }

                $length += \strlen($next[Tokenizer::TOKEN_VALUE]);
            }
        }
        $this->newLine->writeNewLineForLongInlineValues($length);

        if (WhiteSpace::isPrecedingCurrentTokenOfTokenTypeWhiteSpace($originalTokens, $token)) {
            $this->formattedSql = \rtrim($this->formattedSql, ' ');
        }

        $this->newLine->addNewLineAfterOpeningParentheses();

        return $tokens;
    }
1083039
protected function catchAsStringUntilSemicolon(array $tokens, $pos) {
        $t = count($tokens);
        $catched = null;
        for ($i = $pos; $i < $t; $i++) {
            if ($tokens[$i] == ';')
                return $catched;
            else if ($tokens[$i] == '\\')
                $catched .= '\\';
            else if (is_array($tokens[$i])) {
                $catched .= $tokens[$i][1];
            }
        }
        return $catched;
    }
129726
public static function tokenHasMultipleBoundaryCharactersTogether($token, &$tokens, $i, &$originalTokens)
    {
        return $token[Tokenizer::TOKEN_TYPE] === Tokenizer::TOKEN_TYPE_BOUNDARY
        && self::tokenPreviousCharacterIsBoundary($tokens, $i)
        && self::tokenPreviousCharacterIsWhiteSpace($token, $originalTokens);
    }
716620
static function parseIdentifierList(array $tokens, &$i) {
    $list = array();
    while (TRUE) {
      $id = ParserUtil::nextSubstantialIncl($tokens, $i);
      if (T_STRING !== $id) {
        return FALSE;
      }
      $list[] = $tokens[$i][1];
      $id = ParserUtil::nextSubstantialExcl($tokens, $i);
      if (',' !== $id) {
        break;
      }
      ++$i;
    }
    return $list;
  }