Generator to feed tokenized documents (treating each line as a document).
968125
def doc_stream(path):
    
    with open(path, 'r') as f:
        for line in f:
            if line.strip():
                yield line
1299592
def read_content(path: str, limit: Optional[int] = None) -> Iterator[List[str]]:
    
    with smart_open(path) as indata:
        for i, line in enumerate(indata):
            if limit is not None and i == limit:
                break
            yield list(get_tokens(line))
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
195117
def _doc2vec_doc_stream(paths, n, tokenizer=word_tokenize, sentences=True):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)

                # We do minimal pre-processing here so the model can learn
                # punctuation
                line = line.lower()

                if sentences:
                    for sent in sent_tokenize(line):
                        tokens = tokenizer(sent)
                        yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
                else:
                    tokens = tokenizer(line)
                    yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
783490
def pos_tokenize(self: object, fileids: str):
        
        for para in self.paras(fileids):
            yield [
                self.pos_tagger(word_tokenize(sent))
                for sent in sent_tokenize(para)
            ]
946186
def txt_line_iterator(txt_path):
  """"""
  with tf.gfile.Open(txt_path) as f:
    for line in f:
      yield line.strip()
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
1366488
def process_text(self, kind, context):
		""""""
		
		result = None
		
		while True:
			chunk = yield None
			
			if chunk is None:
				if result:
					yield result.clone(line=repr(result.line))
				
				return
			
			if not result:
				result = chunk
				continue
			
			result.line += chunk.line
997914
def tokenize(self, path):
        """"""
        assert os.path.exists(path)
        # Add words to the dictionary
        with open(path, 'r') as f:
            tokens = 0
            for line in f:
                words = line.split() + ['<eos>']
                tokens += len(words)
                for word in words:
                    self.dictionary.add_word(word)

        # Tokenize file content
        with open(path, 'r') as f:
            ids = torch.LongTensor(tokens)
            token = 0
            for line in f:
                words = line.split() + ['<eos>']
                for word in words:
                    ids[token] = self.dictionary.word2idx[word]
                    token += 1

        return ids
1124350
def train_idf(tokens_stream, out=None, **kwargs):
    
    idfs = parallel(count_idf, tokens_stream, n_jobs=-1)
    N = len(idfs) # n docs
    idf = merge(idfs)

    for k, v in idf.items():
        idf[k] = math.log(N/v)
        # v ~= N/(math.e ** idf[k])

    # Keep track of N to update IDFs
    idf['_n_docs'] = N

    if out is not None:
        with open(out, 'w') as f:
            json.dump(idf, f)

    return idf
834699
def process_document(self, doc: Document):
        

        text_segments = doc.select_segments("lexisnexis.doc_description")
        for text_segment in text_segments:
            split_sentences = doc.extract(self.sentence_extractor, text_segment)
            doc.store(split_sentences, 'split_sentences')
        # for t, u in zip(text_to_be_split, units_of_text):
        #     split_sentences = doc.extract(self.sentence_extractor, t)
        #     u.store(split_sentences, "split_sentences")
        return list()
188482
def words(self: object, fileids: str):
        
        for sent in self.sents(fileids):
            for token in word_tokenize(sent):
                yield token
381386
def _create_get_processed_line_func(self, cli, document):
        
        def transform(lineno, tokens):
            " Transform the tokens for a given line number. "
            source_to_display_functions = []
            display_to_source_functions = []

            # Get cursor position at this line.
            if document.cursor_position_row == lineno:
                cursor_column = document.cursor_position_col
            else:
                cursor_column = None

            def source_to_display(i):
                """ Translate x position from the buffer to the x position in the
                processed token list. """
                for f in source_to_display_functions:
                    i = f(i)
                return i

            # Apply each processor.
            for p in self.input_processors:
                transformation = p.apply_transformation(
                    cli, document, lineno, source_to_display, tokens)
                tokens = transformation.tokens

                if cursor_column:
                    cursor_column = transformation.source_to_display(cursor_column)

                display_to_source_functions.append(transformation.display_to_source)
                source_to_display_functions.append(transformation.source_to_display)

            def display_to_source(i):
                for f in reversed(display_to_source_functions):
                    i = f(i)
                return i

            return _ProcessedLine(tokens, source_to_display, display_to_source)

        def create_func():
            get_line = self._get_tokens_for_line_func(cli, document)
            cache = {}

            def get_processed_line(i):
                try:
                    return cache[i]
                except KeyError:
                    processed_line = transform(i, get_line(i))
                    cache[i] = processed_line
                    return processed_line
            return get_processed_line

        return create_func()
715002
def iter_token_lines(tokenlist):
    
    line = []
    for token, c in explode_tokens(tokenlist):
        line.append((token, c))

        if c == '\n':
            yield line
            line = []

    yield line
1197978
def tokenize(self, docs):
        
        if self.n_jobs == 1:
            return [self._tokenize(doc) for doc in docs]
        else:
            return parallel(self._tokenize, docs, self.n_jobs)
815133
def transform_source(text):
    ''''''
    toks = tokenize.generate_tokens(StringIO(text).readline)
    result = []
    where_clause = False
    for toktype, tokvalue, _, _, _ in toks:
        if toktype == tokenize.NAME and tokvalue == "where":
            where_clause = True
        elif where_clause and toktype == tokenize.DEDENT:
            where_clause = False
            continue

        if not where_clause:
            result.append((toktype, tokvalue))
    return tokenize.untokenize(result)
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())