Helper method to return all tokens in an extent.

        This functionality is needed multiple places in this module. We define
        it here because it seems like a logical place.
324856
def get_tokens(tu, extent):
        
        tokens_memory = POINTER(Token)()
        tokens_count = c_uint()

        conf.lib.clang_tokenize(tu, extent, byref(tokens_memory),
                byref(tokens_count))

        count = int(tokens_count.value)

        # If we get no tokens, no memory was allocated. Be sure not to return
        # anything and potentially call a destructor on nothing.
        if count < 1:
            return

        tokens_array = cast(tokens_memory, POINTER(Token * count)).contents

        token_group = TokenGroup(tu, tokens_memory, tokens_count)

        for i in xrange(0, count):
            token = Token()
            token.int_data = tokens_array[i].int_data
            token.ptr_data = tokens_array[i].ptr_data
            token._tu = tu
            token._group = token_group

            yield token
1010767
def _token_extensions(self):
        
        token_provider = self.config['sasl_oauth_token_provider']

        # Only run if the #extensions() method is implemented by the clients Token Provider class
        # Builds up a string separated by \x01 via a dict of key value pairs
        if callable(getattr(token_provider, "extensions", None)) and len(token_provider.extensions()) > 0:
            msg = "\x01".join(["{}={}".format(k, v) for k, v in token_provider.extensions().items()])
            return "\x01" + msg
        else:
            return ""
962213
def get_token_and_data(self, data):
        '''
        
        '''
        token = ''
        for c in data:
            if c != ')':
                token = token + c
            else:
                break;

        return token, data.lstrip(token + '):')
248384
def get_tokens(self, filters={}):
        
        uri = 'tokens'

        if filters:
            filters = {'filters': json.dumps(filters)}
            uri += "?%s" % urllib.urlencode(filters, True)

        resp, body = self.get(uri)
        self.expected_success(200, resp.status)
        body = json.loads(body)
        return rest_client.ResponseBody(resp, body)
118350
def fetch(index, tokens):
    
    if len(tokens) == 0:
        return set()
    return set.intersection(*[set(index.get(token, [])) for token in tokens])
254061
function parseArgumentList() {
                    var token, expr, args = [];

                    while (true) {
                        expr = parseExpression();
                        if (typeof expr === 'undefined') {
                            break;
                        }
                        args.push(expr);
                        token = lexer.peek();
                        if (!matchOp(token, ',')) {
                            break;
                        }
                        lexer.next();
                    }

                    return args;
                }
730786
def all_expressions(self):
        
        if self._all_expressions is None:
            self._all_expressions = self._explore_functions(lambda x: x.expressions)
        return self._all_expressions
1128883
def visitObjectExpr(self, ctx: jsgParser.ObjectExprContext):
        
        if not self._name:
            self._name = self._context.anon_id()
        if ctx.membersDef():
            self.visitChildren(ctx)
        elif ctx.MAPSTO():
            if ctx.LEXER_ID_REF():
                self._map_name_type = as_token(ctx)
            # Any and absent mean the same thing
            self._map_valuetype = JSGValueType(self._context, ctx.valueType())
            if ctx.ebnfSuffix():
                self._map_ebnf = JSGEbnf(self._context, ctx.ebnfSuffix())
390822
def tokens(self):
        
        if not self._tokens:
            self._tokens = list(self.tokenise(self.route))
        return self._tokens
579169
def get_tokens(tokens):
    
    tokens = tokens[1:-1]
    ret = []
    start = 0
    stack = 0
    for i in range(len(tokens)):
        if tokens[i] == '(':
            if stack == 0:
                start = i
            stack += 1
        elif tokens[i] == ')':
            stack -= 1
            if stack < 0:
                raise Exception('Bracket mismatch: ' + str(tokens))
            if stack == 0:
                ret.append(get_tokens(tokens[start:i + 1]))
        else:
            if stack == 0:
                ret.append(tokens[i])
    if stack != 0:
        raise Exception('Bracket mismatch: ' + str(tokens))
    return ret
802771
def get_all(self):
        
        self.optional_params = {}
        self.endpoints = self._ENDPOINTS[:10]
        json_data = self.fetch(fmt_p=no_pandas)
        self.endpoints = self._ENDPOINTS[10:20]
        json_data_2 = self.fetch(fmt_p=no_pandas)
        for symbol in self.symbols:
            if symbol not in json_data:
                raise IEXSymbolError(symbol)
            json_data[symbol].update(json_data_2[symbol])
        return json_data[self.symbols[0]] if self.n_symbols == 1 else json_data
855073
def extent(self):
		
		if not len(self):
			raise ValueError("empty list")
		min, max = self[0]
		for lo, hi in self:
			if min > lo:
				min = lo
			if max < hi:
				max = hi
		return segment(min, max)
669487
function checkExtend1(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkExtendSelector(i)) i += l;
  else return 0;

  if (tokens[i + 1] &&
      tokens[i + 1].value === 'extend' &&
      (l = checkPseudoc(i))) i += l;
  else return 0;

  if (l = checkSC(i)) i += l;

  if (l = checkBlock(i)) i += l;
  else return 0;

  return i - start;
}
468065
def all(self):
        
        result = self.invoke()
        if self.resolve_links:
            result.resolve_links()

        return result
621485
def tokens(self, instance):
        
        if not instance.pk:
            # e.g.: Use will create a new TOTP entry
            return "-"

        totp = TOTP(instance.bin_key, instance.step, instance.t0, instance.digits)

        tokens = []
        for offset in range(-instance.tolerance, instance.tolerance + 1):
            totp.drift = instance.drift + offset
            tokens.append(totp.token())

        return " ".join(["%s" % token for token in tokens])
1129260
function tokensToRegExp (tokens, keys, options) {
  if (!isarray(keys)) {
    options = /** @type {!Object} */ (keys || options);
    keys = [];
  }

  options = options || {};

  var strict = options.strict;
  var end = options.end !== false;
  var route = '';

  // Iterate over the tokens and create our regexp string.
  for (var i = 0; i < tokens.length; i++) {
    var token = tokens[i];

    if (typeof token === 'string') {
      route += escapeString(token);
    } else {
      var prefix = escapeString(token.prefix);
      var capture = '(?:' + token.pattern + ')';

      keys.push(token);

      if (token.repeat) {
        capture += '(?:' + prefix + capture + ')*';
      }

      if (token.optional) {
        if (!token.partial) {
          capture = '(?:' + prefix + '(' + capture + '))?';
        } else {
          capture = prefix + '(' + capture + ')?';
        }
      } else {
        capture = prefix + '(' + capture + ')';
      }

      route += capture;
    }
  }

  var delimiter = escapeString(options.delimiter || '/');
  var endsWithDelimiter = route.slice(-delimiter.length) === delimiter;

  // In non-strict mode we allow a slash at the end of match. If the path to
  // match already ends with a slash, we remove it for consistency. The slash
  // is valid at the end of a path match, not in the middle. This is important
  // in non-ending mode, where "/test/" shouldn't match "/test//route".
  if (!strict) {
    route = (endsWithDelimiter ? route.slice(0, -delimiter.length) : route) + '(?:' + delimiter + '(?=$))?';
  }

  if (end) {
    route += '$';
  } else {
    // In non-ending mode, we need the capturing groups to match as much as
    // possible by using a positive lookahead to the end or next path segment.
    route += strict && endsWithDelimiter ? '' : '(?=' + delimiter + '|$)';
  }

  return attachKeys(new RegExp('^' + route, flags(options)), keys)
}
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens