`token_string` should be the string representation from the server.
555449
def from_string(cls, token_string):
        
        # unhexlify works fine with unicode input in everythin but pypy3, where it Raises "TypeError: 'str' does not support the buffer interface"
        if isinstance(token_string, six.text_type):
            token_string = token_string.encode('ascii')
        # The BOP stores a hex string
        return cls(unhexlify(token_string))
457250
private String consumeTokenString() throws JsonParserException {
    reusableBuffer.setLength(0);
    while (true) {
      char c = stringChar();

      switch (c) {
      case '\"':
        return reusableBuffer.toString();
      case '\\':
        int escape = advanceChar();
        switch (escape) {
        case -1:
          throw createParseException(null, "EOF encountered in the middle of a string escape", false);
        case 'b':
          reusableBuffer.append('\b');
          break;
        case 'f':
          reusableBuffer.append('\f');
          break;
        case 'n':
          reusableBuffer.append('\n');
          break;
        case 'r':
          reusableBuffer.append('\r');
          break;
        case 't':
          reusableBuffer.append('\t');
          break;
        case '"':
        case '/':
        case '\\':
          reusableBuffer.append((char)escape);
          break;
        case 'u':
          reusableBuffer.append((char)(stringHexChar() << 12 | stringHexChar() << 8 //
              | stringHexChar() << 4 | stringHexChar()));
          break;
        default:
          throw createParseException(null, "Invalid escape: \\" + (char)escape, false);
        }
        break;
      default:
        reusableBuffer.append(c);
      }
    }
  }
293696
def get_quoted_string(value):
    
    quoted_string = QuotedString()
    if value and value[0] in CFWS_LEADER:
        token, value = get_cfws(value)
        quoted_string.append(token)
    token, value = get_bare_quoted_string(value)
    quoted_string.append(token)
    if value and value[0] in CFWS_LEADER:
        token, value = get_cfws(value)
        quoted_string.append(token)
    return quoted_string, value
1047267
def t_php_STRING(t):
    '
    t.type = reserved_map.get(t.value.upper(), 'STRING')
    return t
557459
def _normalize_string(raw_str):
  
  return " ".join(
      token.strip()
      for token in tokenizer.encode(text_encoder.native_to_unicode(raw_str)))
1340367
def tokenize(self):
        

        self.token_list = []
        ps = self.parse_string.strip()

        i = 0
        last_token = None

        while i < len(ps) and ps[i].isspace():
            i += 1

        while i < len(ps):
            token = ''

            if ps[i].isalpha():
                while i < len(ps) and (ps[i].isalnum() or ps[i] == '_'):
                    token += ps[i]
                    i += 1
            elif ps[i].isdigit():
                while i < len(ps) and (ps[i].isdigit() or
                                       ps[i] == '.' or
                                       ps[i] == 'e' or
                                       ps[i] == 'E' or
                                       (ps[i] == '+' and (ps[i-1] == 'e' or ps[i-1] == 'E')) or
                                       (ps[i] == '-' and (ps[i-1] == 'e' or ps[i-1] == 'E'))):
                    token += ps[i]
                    i += 1
            elif ps[i] == '.':
                if ps[i+1].isdigit():
                    while i < len(ps) and (ps[i].isdigit() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
                else:
                    while i < len(ps) and (ps[i].isalpha() or ps[i] == '.'):
                        token += ps[i]
                        i += 1
            else:
                token += ps[i]
                i += 1

            if token == '-' and \
               (last_token == None or last_token == '(' or self.is_op(last_token)):
                token = '~'

            self.token_list += [token]
            last_token = token

            while i < len(ps) and ps[i].isspace():
                i += 1
599554
protected function getTokenString()
    {
        $result = Request::post('_token', '');
        if ('' == $result) {
            $result = Request::get('_token', '');
        }

        if ('' == $result) {
            $result = isset($_SERVER['X-CSRF-TOKEN']) ? $_SERVER['X-CSRF-TOKEN'] : '';
        }

        return $result;
    }
437414
def process_tokens(self, tokens):
        
        """
        for (tok_type, token, (start_row, _), _, _) in tokens:
            if tok_type == tokenize.STRING:
                self._check_string(token, start_row)
574598
function parseRstr(rstr){
	var startOfAssertion = rstr.indexOf('<Assertion ');
	var endOfAssertion = rstr.indexOf('</Assertion>') + '</Assertion>'.length;
	var token = rstr.substring(startOfAssertion, endOfAssertion);
	return token;
}
1091794
private static String readToken(Buffer buffer) {
    try {
      long tokenSize = buffer.indexOfElement(TOKEN_DELIMITERS);
      if (tokenSize == -1L) tokenSize = buffer.size();

      return tokenSize != 0L
          ? buffer.readUtf8(tokenSize)
          : null;
    } catch (EOFException e) {
      throw new AssertionError();
    }
  }
753616
def tokenize(self, string):
        ''''''
        pos = 0
        while pos < len(string):
            m = SKIP_RE.match(string, pos=pos)
            if m:
                skip_lines = m.group(0).split('\n')
                if len(skip_lines) > 1:
                    self.row += len(skip_lines) - 1
                    self.column = 1 + len(skip_lines[-1])
                else:
                    self.column += len(skip_lines[0])

                pos = m.end()
                continue

            for cls, type, regex in self.token_map:
                m = regex.match(string, pos=pos)
                if m:
                    yield cls(type, m.group(0),
                              self.filename, self.row, self.column)
                    self.column += len(m.group(0))
                    pos = m.end()
                    break
            else:
                raise ConfigParseError(
                    "Couldn't load config in %r row %d, column %d: %r" %
                    (self.filename, self.row, self.column,
                     string[pos:pos+20]))
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
1076911
def run_string(self, string, metadata=None):
        """"""
        token = Token(string, metadata)
        return [str(tkn) for tkn in self.run([token])]
1112956
function removeTokenFromString(token, string) {
        var tokens = split(string),
            index = tokens.indexOf(token);
        if (index !== -1) {
          tokens.splice(index, 1);
        }
        return tokens.join(' ');
      }
1284752
def _FormatDataToken(self, token_data):
    
    format_string = bsmtoken.BSM_TOKEN_DATA_PRINT.get(
        token_data.data_format, 'UNKNOWN')

    if token_data.data_format == 4:
      data = bytes(bytearray(token_data.data)).split(b'\x00')[0]
      data = data.decode('utf-8')
    else:
      data = ''.join(['{0:02x}'.format(byte) for byte in token_data.data])
    return {
        'format': format_string,
        'data': data}
556159
def _FormatOpaqueToken(self, token_data):
    
    data = ''.join(['{0:02x}'.format(byte) for byte in token_data.data])
    return {'data': data}
1277621
function decodeToken (tokenStr) {
  if (!tokenStr || tokenStr === '') return null
  try {
    return jwtDecode(tokenStr)
  } catch (error) {
    logger.error({
      description: 'Error decoding token.',
      error, func: 'decodeToken', file: 'token'
    })
    return null
  }
}