Check all the tokens for a single source
192142
public function checkAction()
    {
        $sourceId = $this->getSourceId();
        $where    = $this->db->quoteInto('gto_id_survey IN (SELECT gsu_id_survey FROM gems__surveys WHERE gsu_id_source = ?)', $sourceId);

        $batch = $this->loader->getTracker()->recalculateTokens('sourceCheck' . $sourceId, $this->currentUser->getUserId(), $where);

        $title = sprintf($this->_('Checking all surveys in the %s source for answers.'),
                    $this->db->fetchOne("SELECT gso_source_name FROM gems__sources WHERE gso_id_source = ?", $sourceId));
        $this->_helper->batchRunner($batch, $title, $this->accesslog);

        $this->addSnippet('Survey\\CheckAnswersInformation',
                'itemDescription', $this->_('This task checks all tokens using this source for answers .')
                );
    }
202000
function checkForSpaces(node) {
            const tokens = node.type === "UpdateExpression" && !node.prefix
                ? sourceCode.getLastTokens(node, 2)
                : sourceCode.getFirstTokens(node, 2);
            const firstToken = tokens[0];
            const secondToken = tokens[1];

            if ((node.type === "NewExpression" || node.prefix) && firstToken.type === "Keyword") {
                checkUnaryWordOperatorForSpaces(node, firstToken, secondToken, firstToken.value);
                return;
            }

            const operator = node.prefix ? tokens[0].value : tokens[1].value;

            if (overrideExistsForOperator(operator)) {
                if (overrideEnforcesSpaces(operator)) {
                    verifyNonWordsHaveSpaces(node, firstToken, secondToken);
                } else {
                    verifyNonWordsDontHaveSpaces(node, firstToken, secondToken);
                }
            } else if (options.nonwords) {
                verifyNonWordsHaveSpaces(node, firstToken, secondToken);
            } else {
                verifyNonWordsDontHaveSpaces(node, firstToken, secondToken);
            }
        }
482331
def name_check(self, original, loc, tokens):
        """"""
        internal_assert(len(tokens) == 1, "invalid name tokens", tokens)
        if self.strict:
            self.unused_imports.discard(tokens[0])
        if tokens[0] == "exec":
            return self.check_py("3", "exec function", original, loc, tokens)
        elif tokens[0].startswith(reserved_prefix):
            raise self.make_err(CoconutSyntaxError, "variable names cannot start with reserved prefix " + reserved_prefix, original, loc)
        else:
            return tokens[0]
1362214
private static boolean areTokensValid(Token... tokens)
    {
        for (Token token : tokens)
        {
            if (!isTokenValid(token))
                return false;
        }
        return true;
    }
84249
function process_token(token, index, tokens) {
	var last = (index === (tokens.length - 1));
	if (token[0] === "+") return process_single(token, last);
	else if (token[0] === "#") return process_multi(token, last);
	else return process_raw(token, last);
}
477775
def check_state(self, token):
        
        # print "zip", self.zip
        if len(token) == 2 and self.state is None:
            if token.capitalize() in self.parser.states.keys():
                self.state = self._clean(self.parser.states[token.capitalize()])
                return True
            elif token.upper() in self.parser.states.values():
                self.state = self._clean(token.upper())
                return True
        if self.state is None and self.street_suffix is None and len(self.comma_separated_address) > 1:
            if token.capitalize() in self.parser.states.keys():
                self.state = self._clean(self.parser.states[token.capitalize()])
                return True
            elif token.upper() in self.parser.states.values():
                self.state = self._clean(token.upper())
                return True
        return False
422370
def check_token(token: str) -> bool:
    
    if any(x.isspace() for x in token):
        raise exceptions.ValidationError('Token is invalid!')

    left, sep, right = token.partition(':')
    if (not sep) or (not left.isdigit()) or (len(left) < 3):
        raise exceptions.ValidationError('Token is invalid!')

    return True
624411
private function _getAllTokens($source) {
		$newTokens = array();

		// Ugly trick
		// Reset the error array by calling an undefined variable
		set_error_handler('var_dump', 0);
		@$errLastResetUndefinedVar;
		restore_error_handler();

		// Get the tokens
		$tokens = @token_get_all($source);

		// Check for parsing errors
		$parsingErrors = error_get_last();
		if (!empty($parsingErrors) && $parsingErrors["type"] === 128) {
			throw new Exception($parsingErrors["message"]);
		}

		// Check each token and transform into an Object
		foreach ($tokens as $token) {
			$isTokenArray = is_array($token);

			$tokenID = $isTokenArray ? $token[0] : T_UNKNOWN;
			$tokenText = $isTokenArray ? $token[1] : $token;

			// Manage T_OPEN_TAG when php.ini setting short_open_tag is Off.
			if ($this->shortOpenTagOff && $tokenID === T_INLINE_HTML) {

				$startPos = strpos($tokenText, SHORT_OPEN_TAG);
				$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));

				// Extract the content of the short_open_tag
				while (strlen($tokenText) > 2 && $startPos !== false && $endPos !== false) {

					// Parse the beginning of the text
					$beforeText = substr($tokenText, 0, $startPos);
					$this->_identifyTokens($beforeText, $tokenID);

					// The open tag
					$openTag = new TokenInfo();
					$openTag->id = T_OPEN_TAG;
					$openTag->text = SHORT_OPEN_TAG;
					$this->tokenNumber ++;
					$openTag->position = $this->tokenNumber;
					$openTag->line = $this->lineNumber;
					$this->newTokens[] = $openTag;

					// Tokenize the content
					$inlineText = substr($tokenText, $startPos + strlen(SHORT_OPEN_TAG), $endPos - $startPos);
					$inlineText = substr($inlineText, 0, -strlen(CLOSE_TAG));

					$inline = $this->_getAllTokens(OPEN_TAG . " " . $inlineText);

					array_shift($inline); // remove <?php
					$this->_identifyTokens($newTokens, $inline);

					// Add the close tag
					$closeTag = new TokenInfo();
					$closeTag->id = T_CLOSE_TAG;
					$closeTag->text = CLOSE_TAG;
					$this->tokenNumber ++;
					$closeTag->position = $this->tokenNumber;
					$closeTag->line = $this->lineNumber;
					$this->newTokens[] = $closeTag;

					// text = the remaining text
					$tokenText = substr($tokenText, $endPos + strlen(SHORT_OPEN_TAG));

					$startPos = strpos($tokenText, SHORT_OPEN_TAG);
					$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));
				}
			}

			// Identify the tokens
			$this->_identifyTokens($tokenText, $tokenID);
		}

		return $this->newTokens;
	}
1207287
function(source) {
    var proto = Lexer.prototype;
    var tokens = source.match(proto.regs.token);
    for (var i = 0, l = tokens.length; i < l; i ++) {
        if (proto.regs.strip.test(tokens[i])) {
            tokens.splice(i, 1);
        }
    }
    for (var n in proto) tokens[n] = proto[n];
    tokens.index = 0;
    return tokens;
}
494275
function extractToken(token) {
    var regex = /^Bearer\s(\S+)$/i;
    var result = regex.exec(token);
    return result ? result[1] : false;
}
246735
def grep_full_py_identifiers(tokens):
    
    global py_keywords
    tokens = list(tokens)
    i = 0
    while i < len(tokens):
        token_type, token = tokens[i]
        i += 1
        if token_type != "id":
            continue
        while i+1 < len(tokens) and tokens[i] == ("op", ".") and tokens[i+1][0] == "id":
            token += "." + tokens[i+1][1]
            i += 2
        if token == "":
            continue
        if token in py_keywords:
            continue
        if token[0] in ".0123456789":
            continue
        yield token
876794
def check_for_eni_source():
    '''  '''

    with open('/etc/network/interfaces', 'r') as eni:
        for line in eni:
            if line == 'source /etc/network/interfaces.d/*':
                return
    with open('/etc/network/interfaces', 'a') as eni:
        eni.write('\nsource /etc/network/interfaces.d/*')
912130
def _check_generic_pos(self, *tokens):
        """"""
        for record in self.records:
            if all(token in record.message for token in tokens):
                return

        # didn't exit, all tokens are not present in the same record
        msgs = ["Tokens {} not found, all was logged is...".format(tokens)]
        for record in self.records:
            msgs.append("    {:9s} {!r}".format(record.levelname, record.message))
        self.test_instance.fail("\n".join(msgs))
189833
def get_all_templates(server, token):
    
    method = 'GET'
    uri = 'https://' + server + '/template'
    connect.tonicdns_client(uri, method, token, data=False)
1225011
def check_token_file(self):
        
        # TODO: remove this method in a future release
        warnings.warn('This method will be removed in future versions',
                      DeprecationWarning)
        return self.token_backend.check_token() if hasattr(self.token_backend, 'check_token') else None
778472
private void findTokens() {
        mergedTokens = new TokenList();
        variable = null;
        state = TokenizerState.START;
        index = -1;

        while (index < rawTokens.length) {
            switch (state) {
            case START:
                next(TokenizerState.OUTSIDE_VARIABLE_CONTEXT);
                break;
            case OUTSIDE_VARIABLE_CONTEXT:
                handleTokenOutsideOfVariableContext();
                break;
            case INSIDE_VARIABLE_CONTEXT:
                handleTokenInsideOfVariableContext();
                break;
            default:
                break;
            }
        }
    }
792846
private void checkTokenCollectionIndex() throws MtasParserException {
    if (tokenCollectionIndex.size() != tokenCollection.size()) {
      MtasToken token;
      Integer maxId = null;
      Integer minId = null;
      tokenCollectionIndex.clear();
      for (Entry<Integer, MtasToken> entry : tokenCollection.entrySet()) {
        token = entry.getValue();
        maxId = ((maxId == null) ? entry.getKey()
            : Math.max(maxId, entry.getKey()));
        minId = ((minId == null) ? entry.getKey()
            : Math.min(minId, entry.getKey()));
        if (token.getId() == null) {
          throw new MtasParserException(
              "no id for token (" + token.getValue() + ")");
        } else if ((token.getPositionStart() == null)
            || (token.getPositionEnd() == null)) {
          throw new MtasParserException("no position for token with id "
              + token.getId() + " (" + token.getValue() + ")");
        } else if (token.getValue() == null || (token.getValue().equals(""))) {
          throw new MtasParserException(
              "no value for token with id " + token.getId());
        } else if (token.getPrefix() == null
            || (token.getPrefix().equals(""))) {
          throw new MtasParserException(
              "no prefix for token with id " + token.getId());
        } else if ((token.getParentId() != null)
            && !tokenCollection.containsKey(token.getParentId())) {
          throw new MtasParserException(
              "missing parentId for token with id " + token.getId());
        } else if ((token.getOffsetStart() == null)
            || (token.getOffsetEnd() == null)) {
          throw new MtasParserException("missing offset for token with id "
              + token.getId() + " (" + token.getValue() + ")");
        }
        tokenCollectionIndex.add(entry.getKey());
      }
      if ((tokenCollection.size() > 0)
          && ((minId > 0) || ((1 + maxId - minId) != tokenCollection.size()))) {
        throw new MtasParserException("missing ids");
      }
      Collections.sort(tokenCollectionIndex, getCompByName());
    }
  }