Generates the 1-gram to n-grams tuples of the list of tokens
960505
def _generate_ngrams_with_context(self, tokens: List[Token]) -> chain:
        """"""
        chained_ngrams_iter = self._generate_ngrams_with_context_helper(iter(tokens), 1)
        for n in range(2, self._ngrams + 1):
            ngrams_iter = tee(tokens, n)
            for j in range(1, n):
                for k in range(j):
                    next(ngrams_iter[j], None)
            ngrams_iter_with_context = self._generate_ngrams_with_context_helper(zip(*ngrams_iter), n)
            chained_ngrams_iter = chain(chained_ngrams_iter, ngrams_iter_with_context)
        return chained_ngrams_iter
1356764
def iter_tokens(cls, blob):
    
    readline_func = io.StringIO(blob.decode('utf-8')).readline
    return tokenize.generate_tokens(readline_func)
1153714
def to_ngrams(self, terms):
        ''' 
        '''
        if len(terms) <= self.n:
            return terms
        if self.n == 1:
            n_grams = [[term] for term in terms]
        else:
            n_grams = []
            for i in range(0,len(terms)-self.n+1):
                n_grams.append(terms[i:i+self.n])
        return n_grams
1223618
def chunks_of(max_chunk_size, list_to_chunk):
    
    for i in range(0, len(list_to_chunk), max_chunk_size):
        yield list_to_chunk[i:i + max_chunk_size]
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
1143147
def generate_flat_data(self):
    
    # Construct the string statements.
    all_statements = []
    all_targets = []
    self.sequence_sizes_in = []
    self.sequence_sizes_out = []
    for _ in six.moves.range(self._batch_size):
      length, nest = self.curriculum_obj.fetch()
      seq_size_in = self._max_seq_length
      # Generate batch within max length.
      is_valid_sample = False
      tries_remaining = 10
      while not is_valid_sample:
        value, code = generate_code(length, nest, self._ops)
        tokens_in, seq_size_in = self.tokenize(
            code, self._max_seq_length, self._token_by_char)
        tokens_out, seq_size_out = self.tokenize(
            value, self._max_seq_length, self._token_by_char)
        is_valid_sample = self._max_seq_length >= seq_size_in
        if is_valid_sample:
          self.sequence_sizes_in.append(seq_size_in)
          self.sequence_sizes_out.append(seq_size_out)
        if tries_remaining == 0:
          raise ValueError("Could not generate a sample below the allowable "
                           "maximum, consider reducing either max_length or "
                           "max_nest.")
        else:
          tries_remaining -= 1
      all_statements += tokens_in
      all_targets += tokens_out
    # Store the flattened data.
    self.flat_data = np.array(all_statements, dtype=np.int64)
    self.num_tokens = self.flat_data.shape[0]
    self.flat_targets = np.array(all_targets, dtype=np.int64)
    self.num_tokens_target = self.flat_targets.shape[0]
    self.start_token = np.array(self.tokenize(
        [get_start_token()], 1)[0], dtype=np.int64)
    self.end_token = np.array(self.tokenize(
        [get_end_token()], 1)[0], dtype=np.int64)
373827
def tokenize_block(iterable, token_types):
    
    lines = FileWrapper(iterable)
    parse_buffer = ParseBuffer()
    line = lines.peek()
    while line is not None:
        for token_type in token_types:
            if token_type.start(line):
                result = token_type.read(lines)
                if result is not None:
                    parse_buffer.append((token_type, result))
                    break
        else:  # unmatched newlines
            next(lines)
            parse_buffer.loose = True
        line = lines.peek()
    return parse_buffer
25308
def tokens(self):
        """"""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
951970
def ngram(n, iter_tokens):
    
    z = len(iter_tokens)
    return (iter_tokens[i:i+n] for i in range(z-n+1))
306434
def vectorize(self, token_list):
        '''
        
        '''
        vector_list = [self.__collection.tf_idf(token, self.__collection) for token in token_list]
        return vector_list
826595
def _generate_tokens(pat: GenericAny, text: str) -> Iterator[Token]:
    
    rules = _TYPE_RULES
    keys = _TYPE_KEYS
    groupindex = pat.groupindex
    scanner = pat.scanner(text)
    for m in iter(scanner.match, None):
        lastgroup = m.lastgroup
        func = rules[keys[groupindex[lastgroup] - 1]]
        if func is not None:
            yield func(m.group(lastgroup))
448465
def list(self):
        """"""
        start = self.tokens.matched.start

        if self.tokens.accept(common_grammar.rbracket):
            return ast.Tuple(start=start, end=self.tokens.matched.end,
                             source=self.original)

        elements = [self.expression()]

        while self.tokens.accept(common_grammar.comma):
            elements.append(self.expression())

        self.tokens.expect(common_grammar.rbracket)
        return ast.Tuple(*elements, start=start, end=self.tokens.matched.end,
                         source=self.original)
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
295754
def chunks(self, chunk_size: int) -> Generator['TranslatorInput', None, None]:
        

        if len(self.tokens) > chunk_size and self.constraints is not None:
            logger.warning(
                'Input %s has length (%d) that exceeds max input length (%d), '
                'triggering internal splitting. Placing all target-side constraints '
                'with the first chunk, which is probably wrong.',
                self.sentence_id, len(self.tokens), chunk_size)

        for chunk_id, i in enumerate(range(0, len(self), chunk_size)):
            factors = [factor[i:i + chunk_size] for factor in self.factors] if self.factors is not None else None
            # Constrained decoding is not supported for chunked TranslatorInputs. As a fall-back, constraints are
            # assigned to the first chunk
            constraints = self.constraints if chunk_id == 0 else None
            pass_through_dict = self.pass_through_dict if chunk_id == 0 else None
            yield TranslatorInput(sentence_id=self.sentence_id,
                                  tokens=self.tokens[i:i + chunk_size],
                                  factors=factors,
                                  restrict_lexicon=self.restrict_lexicon,
                                  constraints=constraints,
                                  avoid_list=self.avoid_list,
                                  pass_through_dict=pass_through_dict)
700446
def snli_token_generator(tmp_dir, train, vocab_size):
  """"""
  _download_and_parse_dataset(tmp_dir, train)

  symbolizer_vocab = _get_or_generate_vocab(
      tmp_dir, 'vocab.subword_text_encoder', vocab_size)

  file_name = 'train' if train else 'dev'
  data_file = os.path.join(tmp_dir, file_name + '.txt')
  with tf.gfile.GFile(data_file, mode='r') as f:
    for line in f:
      sent1, sent2, label = line.strip().split('\t')
      sent1_enc = symbolizer_vocab.encode(sent1)
      sent2_enc = symbolizer_vocab.encode(sent2)

      inputs = sent1_enc + [_SEP] + sent2_enc + [_EOS]
      yield {
          'inputs': inputs,
          'targets': [_LABEL_TO_ID[label]],
      }
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens