Converts an escaped token string to a list of subtoken strings.

        Args:
          escaped_token: An escaped token as a unicode string.
        Returns:
          A list of subtokens as unicode strings.
482382
def _escaped_token_to_subtoken_strings(self, escaped_token):
        
        # NOTE: This algorithm is greedy; it won't necessarily produce the "best"
        # list of subtokens.
        ret = []
        start = 0
        token_len = len(escaped_token)
        while start < token_len:
            for end in xrange(min(token_len, start + self._max_subtoken_len), start, -1):
                subtoken = escaped_token[start:end]
                if subtoken in self._all_subtoken_strings:
                    ret.append(subtoken)
                    start = end
                    break

            else:  # Did not break
                # If there is no possible encoding of the escaped token then one of the
                # characters in the token is not in the alphabet. This should be
                # impossible and would be indicative of a bug.
                assert False, "Token substring not found in subtoken vocabulary."

        return ret
6008
def decode(tokens):
  
  token_is_alnum = [t[0] in _ALPHANUMERIC_CHAR_SET for t in tokens]
  ret = []
  for i, token in enumerate(tokens):
    if i > 0 and token_is_alnum[i - 1] and token_is_alnum[i]:
      ret.append(u" ")
    ret.append(token)
  return "".join(ret)
1183690
def _subtoken_ids_to_tokens(self, subtokens):
    
    concatenated = "".join(
        [self._subtoken_id_to_subtoken_string(s) for s in subtokens])
    split = concatenated.split("_")
    ret = []
    for t in split:
      if t:
        unescaped = _unescape_token(t + "_")
        if unescaped:
          ret.append(unescaped)
    return ret
1320730
def make_tokens(parse_buffer):
    
    tokens = []
    for token_type, result in parse_buffer:
        token = token_type(result)
        if token is not None:
            tokens.append(token)
    return tokens
539793
def _subtoken_id_to_subtoken_string(self, subtoken):
    """"""
    if 0 <= subtoken < self.vocab_size:
      return self._all_subtoken_strings[subtoken]
    return u""
273190
def _get_tokens(morph_fd):
    '''
    '''
    for line in morph_fd:
        line = line.rstrip()   # remove \r\n
        line = line.split(';', 1)[0]  # strip comments
        squash_token = []  # quoted strings get squashed into one token

        if '<(' in line:  # skip spines, which exist on a single line
            assert ')>' in line, 'Missing end of spine'
            continue

        for token in line.replace('(', ' ( ').replace(')', ' ) ').split():
            if squash_token:
                squash_token.append(token)
                if token.endswith('"'):
                    token = ' '.join(squash_token)
                    squash_token = []
                    yield token
            elif token.startswith('"') and not token.endswith('"'):
                squash_token.append(token)
            else:
                yield token
1201727
def tokenize_string(source):
    
    line_reader = StringIO(source).readline
    token_generator = tokenize.generate_tokens(line_reader)

    # Loop over all tokens till a backtick (`) is found.
    # Then, take all tokens till the next backtick to form a backtick quoted
    # string.
    for toknum, tokval, _, _, _ in token_generator:
        if tokval == '`':
            tokval = " ".join(it.takewhile(
                lambda tokval: tokval != '`',
                map(operator.itemgetter(1), token_generator)))
            toknum = _BACKTICK_QUOTED_STRING
        yield toknum, tokval
669912
def _parse_dash_escaped_line(dash_escaped_line: str) -> str:
        
        text = str()
        regex_dash_escape_prefix = compile('^' + DASH_ESCAPE_PREFIX)
        # if prefixed by a dash escape prefix...
        if regex_dash_escape_prefix.match(dash_escaped_line):
            # remove dash '-' (0x2D) and space ' ' (0x20) prefix
            text += dash_escaped_line[2:]

        return text
507120
def decode_escapes(s):
    ''''''
    def decode_match(match):
        return codecs.decode(match.group(0), 'unicode-escape')

    return ESCAPE_SEQUENCE_RE.sub(decode_match, s)
1070446
def tokenize_sents(string):
    
    string = six.text_type(string)

    spans = []
    for match in re.finditer('[^\s]+', string):
        spans.append(match)
    spans_count = len(spans)

    rez = []
    off = 0

    for i in range(spans_count):
        tok = string[spans[i].start():spans[i].end()]
        if i == spans_count - 1:
            rez.append(string[off:spans[i].end()])
        elif tok[-1] in ['.', '!', '?', '…', '»']:
            tok1 = tok[re.search('[.!?…»]', tok).start()-1]
            next_tok = string[spans[i + 1].start():spans[i + 1].end()]
            if (next_tok[0].isupper()
                and not tok1.isupper()
                and not (tok[-1] != '.'
                         or tok1[0] == '('
                         or tok in ABBRS)):
                rez.append(string[off:spans[i].end()])
                off = spans[i + 1].start()

    return rez
860170
def _process_tokens(self, char):
        

        if (char in self.WHITESPACE or char == self.COMMENT_START or
                char in self.QUOTES or char in self.TOKENS):

            add_token = True

            # escaped chars, keep going
            if char == self.SPACE or char in self.TOKENS:
                if self._escaped:
                    add_token = False

            # start of comment
            elif char == self.COMMENT_START:
                self._state = self.ST_COMMENT

            # start of quoted string
            elif char in self.QUOTES:
                if self._escaped:
                    # escaped, keep going
                    add_token = False

                else:
                    self._state = self.ST_STRING
                    self._last_quote = char  # store for later quote matching

            if add_token:
                # store token
                self._new_token()

                if char in self.TOKENS:
                    # store char as a new token
                    self._new_token([char])

                return  # skip adding token char

        self._token_chars.append(char)
597908
def escape_args(cls, *args):
        
        escaped_args = [shlex.quote(str(arg)) for arg in args]

        return " ".join(escaped_args)
919996
def reconstruct_text(tokens: List[Token]) -> str:
        
        return "".join([x.text_with_ws for x in tokens])
673055
def escape_yaml(raw_str: str) -> str:
    
    escape_list = [char for char in raw_str if char in ['!', '{', '[']]
    if len(escape_list) == 0:
        return raw_str

    str_quotes = '"'
    i_str_quotes = "'"
    if str_quotes in raw_str and str_quotes not in raw_str[1:-1]:
        return raw_str

    if str_quotes in raw_str[1:-1]:
        raw_str = i_str_quotes + raw_str + i_str_quotes
    else:
        raw_str = str_quotes + raw_str + str_quotes
    return raw_str
16752
def span_tokenize_sents(self, strings):
        
        raise NotImplementedError("span_tokenizer and span_tokenzie_sents not yet implemented. ;)")
        for s in strings:
            yield list(self.span_tokenize(s))
429244
def unpack_token(token):
    
    if isinstance(token, (unicode,str)):
        return _decode_token_compact(token)

    else:
        return _decode_token_json(token)
1338714
def _tokens_to_subtoken(self, tokens):
        
        ret = []
        for token in tokens:
            ret.extend(
                self._escaped_token_to_subtoken_strings(_escape_token(token, self._alphabet)))
        return ret