Returns a token if position is in [start-of-leading-trivia, end)
548326
function getTokenAtPosition(sourceFile, position, includeJsDocComment) {
        if (includeJsDocComment === void 0) { includeJsDocComment = false; }
        return getTokenAtPositionWorker(sourceFile, position, /*allowPositionInLeadingTrivia*/ true, /*includeItemAtEndPosition*/ undefined, includeJsDocComment);
    }
332614
def on_position_changed(self, position):
        """"""
        if not self._lyric:
            return

        pos = find_previous(position*1000 + 300, self._pos_list)
        if pos is not None and pos != self._pos:
            self.current_sentence = self._pos_s_map[pos]
            self._pos = pos
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }
226369
def find_token(self, start_token, tok_type, tok_str=None, reverse=False):
    
    t = start_token
    advance = self.prev_token if reverse else self.next_token
    while not match_token(t, tok_type, tok_str) and not token.ISEOF(t.type):
      t = advance(t, include_extra=True)
    return t
1192213
function endsinArr(token, suffixes) {
  var i, longest = '';
  for (i = 0; i < suffixes.length; i++) {
    if (endsin(token, suffixes[i]) && suffixes[i].length > longest.length)
      longest = suffixes[i];
  }

  return longest;
}
554218
private int parseSpecialPosition(int i, StringBuilder str) throws ParseException {
        int j = readWords(i + 1, rawBuilder);
        if(j > i && rules.charAt(j) == 0x5d && rawBuilder.length() != 0) {  // words end with ]
            ++j;
            String raw = rawBuilder.toString();
            str.setLength(0);
            for(int pos = 0; pos < positions.length; ++pos) {
                if(raw.equals(positions[pos])) {
                    str.append(POS_LEAD).append((char)(POS_BASE + pos));
                    return j;
                }
            }
            if(raw.equals("top")) {
                str.append(POS_LEAD).append((char)(POS_BASE + Position.LAST_REGULAR.ordinal()));
                return j;
            }
            if(raw.equals("variable top")) {
                str.append(POS_LEAD).append((char)(POS_BASE + Position.LAST_VARIABLE.ordinal()));
                return j;
            }
        }
        setParseError("not a valid special reset position");
        return i;
    }
929772
function getStartWithSpaces(token, sourceCode) {
    const text = sourceCode.text;
    let start = token.range[0];

    // If the previous token is a line comment then skip this step to avoid commenting this token out.
    {
        const prevToken = sourceCode.getTokenBefore(token, { includeComments: true });

        if (prevToken && prevToken.type === "Line") {
            return start;
        }
    }

    // Detect spaces before the token.
    while (ANY_SPACE.test(text[start - 1] || "")) {
        start -= 1;
    }

    return start;
}
668514
def split_leading_comment(inputstring):
    """"""
    if inputstring.startswith("#"):
        comment, rest = inputstring.split("\n", 1)
        return comment + "\n", rest
    else:
        return "", inputstring
665692
def is_potential_flag(token: str, parser: argparse.ArgumentParser) -> bool:
    """"""
    # if it's an empty string, it was meant to be a positional
    if not token:
        return False

    # if it doesn't start with a prefix, it was meant to be positional
    if not token[0] in parser.prefix_chars:
        return False

    # if it's just a single character, it was meant to be positional
    if len(token) == 1:
        return False

    # if it looks like a negative number, it was meant to be positional
    # unless there are negative-number-like options
    if parser._negative_number_matcher.match(token):
        if not parser._has_negative_number_optionals:
            return False

    # if it contains a space, it was meant to be a positional
    if ' ' in token:
        return False

    # Looks like a flag
    return True
303299
function checkAtruler(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkAtkeyword(i)) i += l;
  else return 0;

  if (l = checkTsets(i)) i += l;

  if (i < tokensLength && tokens[i].type === TokenType.LeftCurlyBracket) i++;
  else return 0;

  if (l = checkAtrulers(i)) i += l;

  if (i < tokensLength && tokens[i].type === TokenType.RightCurlyBracket) i++;
  else return 0;

  return i - start;
}
992114
private static String highlightToken(String line, Token token)
    {
        String newLine = insertChar(line, getLastCharPositionInLine(token), ']');
        return insertChar(newLine, token.getCharPositionInLine(), '[');
    }
603189
def _getPOS( self, token, onlyFirst = True ):
        ''' 
        '''
        if onlyFirst:
            return token[ANALYSIS][0][POSTAG]
        else:
            return [ a[POSTAG] for a in token[ANALYSIS] ]
572480
function getR1(token) {
    var match = token.match(/[aeiouyæåø]{1}[^aeiouyæåø]([A-Za-z0-9_æøåÆØÅäÄöÖüÜ]+)/);

    if (match) {
        var preR1Length = match.index + 2;

        if (preR1Length < 3 && preR1Length > 0) {
            return token.slice(3);
        } else if (preR1Length >= 3) {
            return match[1];
        } else {
            return token;
        }
    }

    return null;
}
165948
function checkEscapedString(i) {
  const start = i;
  let l;

  if (i >= tokensLength) return 0;

  if (tokens[i].type === TokenType.Tilde && (l = checkString(i + 1)))
    return i + l - start;
  else return 0;
}
667653
def get_w605_position(tokens):
    """"""
    # TODO: When this PR(*) change is released, use pos of pycodestyle
    # *: https://github.com/PyCQA/pycodestyle/pull/747
    valid = [
        '\n', '\\', '\'', '"', 'a', 'b', 'f', 'n', 'r', 't', 'v',
        '0', '1', '2', '3', '4', '5', '6', '7', 'x',

        # Escape sequences only recognized in string literals
        'N', 'u', 'U',
    ]

    for token_type, text, start_pos, _end_pos, _line in tokens:
        if token_type == tokenize.STRING:
            quote = text[-3:] if text[-3:] in ('"""', "'''") else text[-1]
            # Extract string modifiers (e.g. u or r)
            quote_pos = text.index(quote)
            prefix = text[:quote_pos].lower()
            start = quote_pos + len(quote)
            string = text[start:-len(quote)]

            if 'r' not in prefix:
                pos = string.find('\\')
                while pos >= 0:
                    pos += 1
                    if string[pos] not in valid:
                        yield (
                            # No need to search line, token stores position
                            start_pos[1],
                            "W605 invalid escape sequence '\\%s'" %
                            string[pos],
                        )
                    pos = string.find('\\', pos + 1)
618908
def _good_token(self, token, offset, start=None):
        """"""
        if start is None:
            start = self.offset
        try:
            comment_index = self.source.rindex('#', start, offset)
        except ValueError:
            return True
        try:
            new_line_index = self.source.rindex('\n', start, offset)
        except ValueError:
            return False
        return comment_index < new_line_index
367189
def _check_endings(self):
        """"""
        if self.slug.startswith("/") and self.slug.endswith("/"):
            raise InvalidSlugError(
                _("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(self.slug.strip("/"))))
        elif self.slug.startswith("/"):
            raise InvalidSlugError(
                _("Invalid slug. Did you mean {}, without the leading slash?".format(self.slug.strip("/"))))
        elif self.slug.endswith("/"):
            raise InvalidSlugError(
                _("Invalid slug. Did you mean {}, without the trailing slash?".format(self.slug.strip("/"))))