Construct a single command from $cmd template and given $tokens

@param string $cmd Command template
@param array $tokens List of tokens to subsctitude in command template

@return string final command
489369
protected function getCommand($cmd, $tokens)
    {
        return Template::parse($cmd, $tokens, '%%', '%%', Template::FLAG_RECURSIVE | Template::FLAG_EMPTY_MISSING);
    }
531329
private static String[] executeShellCommand(String[] command)
  throws IOException {
    String groups = Shell.execCommand(command);
    StringTokenizer tokenizer = new StringTokenizer(groups);
    int numOfTokens = tokenizer.countTokens();
    String[] tokens = new String[numOfTokens];
    for (int i=0; tokenizer.hasMoreTokens(); i++) {
      tokens[i] = tokenizer.nextToken();
    }

    return tokens;
  }
792916
private function doParse(array $tokens, &$i) {

    $id = ParserUtil::nextSubstantialIncl($tokens, $i);

    if (T_USE === $id) {
      return ParserUtil::parseUseStatementGroup($tokens, $i);
    }
    elseif (T_NAMESPACE === $id) {
      return ParserUtil::parseNamespaceDeclaration($tokens, $i);
    }
    elseif (T_FINAL === $id || T_ABSTRACT === $id) {
      if (T_DOC_COMMENT === $tokens[$i - 2][0]) {
        $docComment = $tokens[$i - 2][1];
      }
      else {
        $docComment = NULL;
      }
      $modifiers = array($id => TRUE);
      while (TRUE) {
        $id = ParserUtil::nextSubstantialExcl($tokens, $i);
        $modifiers[$id] = TRUE;
        if (T_CLASS === $id || T_INTERFACE === $id || T_TRAIT === $id) {
          return $this->classLikeParser->parse($tokens, $i, $modifiers, $docComment);
        }
        elseif (T_FINAL === $id || T_ABSTRACT === $id) {
          $modifiers[$id] = TRUE;
        }
        else {
          // Something other than the above does not belong here.
          return FALSE;
        }
      }
      throw new \RuntimeException('Unreachable code.');
    }
    elseif (T_CLASS === $id || T_INTERFACE === $id || T_TRAIT === $id) {
      if (T_DOC_COMMENT === $tokens[$i - 2][0]) {
        $docComment = $tokens[$i - 2][1];
      }
      else {
        $docComment = NULL;
      }
      return $this->classLikeParser->parse($tokens, $i, array($id => TRUE), $docComment);
    }
    elseif ('#' === $id) {
      return FALSE;
    }
    else {
      return self::ignoreStatement($tokens, $i);
    }
  }
908437
private function getLicenseTokenIndex(array $tokens)
    {
        $licenseTokenIndex = null;

        foreach ($tokens as $index => $token) {
            if ($token[0] === T_COMMENT) {
                $licenseTokenIndex = $index;
            }

            // if we reach the class declaration then it does not have a license
            if ($token[0] === T_CLASS) {
                break;
            }
        }

        return $licenseTokenIndex;
    }
741738
protected function getInput(array $tokens)
    {
        $var = '';
        $firstToken = \array_pop($tokens);
        if (self::tokenIs($firstToken, self::T_STRING)) {
            $var = $firstToken[1];
        }

        return $var;
    }
31821
private function createId(array $tokens) {
        $idColumn = Identifier::column($tokens, new MetaData());
        return empty($this->result[$idColumn]) ? null : $this->result[$idColumn];
    }
133994
private function tokensToString(array $tokens): string
    {
        $lineCount = 0;
        $headerLines = [];
        $headerLines[] = $this->name.': ';
        $currentLine = &$headerLines[$lineCount++];

        // Build all tokens back into compliant header
        foreach ($tokens as $i => $token) {
            // Line longer than specified maximum or token was just a new line
            if (("\r\n" === $token) ||
                ($i > 0 && \strlen($currentLine.$token) > $this->lineLength)
                && 0 < \strlen($currentLine)) {
                $headerLines[] = '';
                $currentLine = &$headerLines[$lineCount++];
            }

            // Append token to the line
            if ("\r\n" !== $token) {
                $currentLine .= $token;
            }
        }

        // Implode with FWS (RFC 2822, 2.2.3)
        return implode("\r\n", $headerLines);
    }
1055562
public function _getProgramElementName(&$tokens, $key)
    {
        $name = '';
        $key++;
        while (
            $tokens[$key] && (
                $tokens[$key] == '&' || (
                    isset($tokens[$key][0]) && isset($tokens[$key][1]) && (
                    $tokens[$key][0] == T_WHITESPACE ||
                    $tokens[$key][0] == T_STRING ||
                    $tokens[$key][0] == T_NS_SEPARATOR
                )
            )
        )) {
            if (isset($tokens[$key][1])) {
                $name .= $tokens[$key][1];
            }
            $key++;
        }

        return trim($name);
    }
236128
protected function generateNamespaceContext(array & $tokens)
	{
		$namespaceContext = new NamespaceContext();
		
		for($size = count($tokens), $i = 0; $i < $size; $i++)
		{
			if(is_array($tokens[$i]))
			{
				switch($tokens[$i][0])
				{
					case T_NAMESPACE:
						
						$namespace = '';
						
						for($i++; $i < $size; $i++)
						{
							if(is_array($tokens[$i]))
							{
								switch($tokens[$i][0])
								{
									case T_WHITESPACE:
									case T_COMMENT:
									case T_DOC_COMMENT:
										continue 2;
									case T_NS_SEPARATOR:
									case T_STRING:
										$namespace .= $tokens[$i][1];
										continue 2;
								}
							}
							
							break;
						}
						
						$namespaceContext = new NamespaceContext($namespace);
						
						break;
					case T_USE:
						do
						{
							$i++;
							$this->readImport($tokens, $i, $namespaceContext);
						}
						while($tokens[$i] == ',');
						
						break;
					case T_CLASS:
					case T_INTERFACE:
					case T_TRAIT:
						// Abort namespace detection as soon as code declaring a type is reached.
						break 2;
				}
			}
		}
		
		return $namespaceContext;
	}
1182010
def get_command_templates(command_tokens, file_tokens=[], path_tokens=[],
    job_options=[]):
    
    files = get_files(file_tokens)
    paths = get_paths(path_tokens)
    job_options = get_options(job_options)

    templates = _get_command_templates(command_tokens, files, paths,
        job_options)

    for command_template in templates:
        command_template._dependencies = _get_prelim_dependencies(
            command_template, templates)
    return templates
530777
private static function getName($key, array $tokens): ?string
    {
        $class      = null;
        $tokenCount = \count($tokens);

        for ($i = $key; $i < $tokenCount; $i++) {
            if (\is_array($tokens[$i])) {
                if ($tokens[$i][0] === \T_STRING) {
                    $class .= $tokens[$i][1];
                } elseif ($tokens[$i][0] === \T_WHITESPACE) {
                    return $class;
                }
            }
        }

        return null;
    }
172580
def _command_and_args(tokens: List[str]) -> Tuple[str, str]:
        
        command = ''
        args = ''

        if tokens:
            command = tokens[0]

        if len(tokens) > 1:
            args = ' '.join(tokens[1:])

        return command, args
670006
public function pushOneDoc($fields, $cmd) {
        $cmdName = Constant::get('DOC_KEY_CMD');
        $fieldName = Constant::get('DOC_KEY_FIELDS');
        $this->docs[] = array($cmdName => $cmd, $fieldName => $fields);
    }
348029
private static function getSegmentFromToken($token, $key, array $tokens)
    {
        $relation_tokens = array('>', '~', '+');

        if (\in_array($token, $relation_tokens, true)) { // not a segment
            return false;
        }

        $segment = (object) array(
            'selector' => '',
            'relation_token' => false,
            'attribute_filters' => array(),
            'pseudo_filters' => array()
        );

        if (isset($tokens[$key-1]) && \in_array($tokens[$key-1], $relation_tokens, true)) { // get relationship token
            $segment->relation_token = $tokens[$key-1];
        }

        if (ctype_alpha($token)) { // simple element selector
            $segment->selector = $token;
            return $segment;
        }

        $char_tmp_replaced = false;
        if (strpos($token, '\\') !== false) {
            $token = preg_replace_callback( // temporary replace escaped characters
                '#(\\\\)(.{1})#',
                function ($matches) {
                    return 'ESCAPED'. \ord($matches[2]);
                },
                $token
            );
            $char_tmp_replaced = true;
        }

        $segment->selector = $token;

        while (preg_match('/(.*)\:(not|contains|has)\((.+)\)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2].'('.$matches[3].')';
        }

        while (preg_match('/(.*)\:([a-z][a-z\-]+)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\[([^]]+)\]$/', $segment->selector, $matches)) { // attribute selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\.([^\.\#]+)$/i', $segment->selector, $matches)) { // class selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'class~="'.$matches[2].'"';
        }

        while (preg_match('/(.*)\#([^\.\#]+)$/i', $segment->selector, $matches)) { // id selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'id="'.$matches[2].'"';
        }

        if ($char_tmp_replaced) { // restore temporary replaced characters
            $set_escape_back = function (string $str) {
                return preg_replace_callback(
                    '#(ESCAPED)([0-9]{1,3})#',
                    function ($matches) {
                        return \chr($matches[2]);
                    },
                    $str
                );
            };

            $segment->selector = $set_escape_back($segment->selector);

            foreach ($segment->attribute_filters as &$attr_filter) {
                $attr_filter = $set_escape_back($attr_filter);
            }
        }

        return $segment;
    }
148964
public static function replace(array $tokens, $newTable) {
        HashMap::assert($tokens, 'tokens');

        $operation = SqlOperation::create($tokens);
        $firstKey  = $operation === SqlOperations::DELETE || $operation === SqlOperations::SELECT ? 'FROM' : strtoupper($operation);

        $tokens[$firstKey][$operation === SqlOperations::INSERT ? 1 : 0]['no_quotes']['parts'][0] = $newTable;

        return $tokens;
    }
116001
public static function transpileMatchRegex(array $tokens)
    {
        $regex = [];

        foreach ($tokens as $token) {
            $var = $token instanceof Variable ? $token : ($token instanceof Delimiter ? $token->next : null);

            if (null !== $var && $var instanceof Variable && null !== ($optgrp = self::makeOptGrp($var))) {
                $regex[] = $optgrp;
                break;
            }

            $regex[] = $token;
        }

        return implode('', $regex);
    }
958891
public function translate(array $tokens)
    {
        $this->operatorStack = new SplStack();
        $this->outputQueue = new SplQueue();
        foreach($tokens as $token) {
            switch ($token->getType()) {
                case Token::T_OPERAND:
                    $this->outputQueue->enqueue($token);
                    break;
                case Token::T_OPERATOR:
                    $o1 = $token;
                    while($this->hasOperatorInStack()
                            && ($o2 = $this->operatorStack->top())
                            && $o1->hasLowerPriority($o2))
                    {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }

                    $this->operatorStack->push($o1);
                    break;
                case Token::T_LEFT_BRACKET:
                    $this->operatorStack->push($token);
                    break;
                case Token::T_RIGHT_BRACKET:
                    while((!$this->operatorStack->isEmpty()) && (Token::T_LEFT_BRACKET != $this->operatorStack->top()->getType())) {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }
                    if($this->operatorStack->isEmpty()) {
                        throw new InvalidArgumentException(sprintf('Mismatched parentheses: %s', implode(' ',$tokens)));
                    }
                    $this->operatorStack->pop();
                    break;
                default:
                    throw new InvalidArgumentException(sprintf('Invalid token detected: %s', $token));
                    break;
            }
        }
        while($this->hasOperatorInStack()) {
            $this->outputQueue->enqueue($this->operatorStack->pop());
        }

        if(!$this->operatorStack->isEmpty()) {
            throw new InvalidArgumentException(sprintf('Mismatched parenthesis or misplaced number: %s', implode(' ',$tokens)));
        }

        return iterator_to_array($this->outputQueue);
    }