Parse documents from a .txt file assuming 1 document per line, yielding lists of filtered tokens
326635
def get_texts(self):
        
        with self.getstream() as text_stream:
            for i, line in enumerate(text_stream):
                line = to_unicode(line)
                line = (TweetCorpus.case_normalizer or passthrough)(line)
                # line = self.case_normalizer(line)
                if self.mask is not None and not self.mask[i]:
                    continue
                ngrams = []
                for ng in tokens2ngrams((TweetCorpus.tokenizer or str.split)(line), n=self.num_grams):
                    if self.ignore_matcher(ng):
                        continue
                    ngrams += [ng]
                if not (i % 1000):
                    print(line)
                    print(ngrams)
                yield ngrams
1065191
def tokenizer(text):
    

    stream = deque(text.split("\n"))

    while len(stream) > 0:
        line = stream.popleft()

        if line.startswith("#METABOLOMICS WORKBENCH"):
            yield KeyValue("#METABOLOMICS WORKBENCH", "\n")
            yield KeyValue("HEADER", line)

            for identifier in line.split(" "):
                if ":" in identifier:
                    key, value = identifier.split(":")
                    yield KeyValue(key, value)

        elif line.startswith("#ANALYSIS TYPE"):
            yield KeyValue("HEADER", line)

        elif line.startswith("#SUBJECT_SAMPLE_FACTORS:"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue("#SUBJECT_SAMPLE_FACTORS", "\n")

        elif line.startswith("#"):
            yield KeyValue("#ENDSECTION", "\n")
            yield KeyValue(line.strip(), "\n")

        elif line.startswith("SUBJECT_SAMPLE_FACTORS"):
            key, subject_type, local_sample_id, factors, additional_sample_data = line.split("\t")
            # factors = [dict([[i.strip() for i in f.split(":")]]) for f in factors.split("|")]
            yield SubjectSampleFactors(key.strip(), subject_type, local_sample_id, factors, additional_sample_data)

        elif line.endswith("_START"):
            yield KeyValue(line, "\n")

            while not line.endswith("_END"):
                line = stream.popleft()
                if line.endswith("_END"):
                    yield KeyValue(line.strip(), "\n")
                else:
                    data = line.split("\t")
                    yield KeyValue(data[0], tuple(data))

        else:
            if line:
                if line.startswith("MS:MS_RESULTS_FILE") or line.startswith("NM:NMR_RESULTS_FILE"):
                    try:
                        key, value, extra = line.split("\t")
                        extra_key, extra_value = extra.strip().split(":")
                        yield KeyValueExtra(key.strip()[3:], value, extra_key, extra_value)
                    except ValueError:
                        key, value = line.split("\t")
                        yield KeyValue(key.strip()[3:], value)
                else:
                    try:
                        key, value = line.split("\t")
                        if ":" in key:
                            if key.startswith("MS_METABOLITE_DATA:UNITS"):
                                yield KeyValue(key.strip(), value)
                            else:
                                yield KeyValue(key.strip()[3:], value)
                        else:
                            yield KeyValue(key.strip(), value)
                    except ValueError:
                        print("LINE WITH ERROR:\n\t", repr(line))
                        raise

    yield KeyValue("#ENDSECTION", "\n")
    yield KeyValue("!#ENDFILE", "\n")
468530
def _phrase_doc_stream(paths, n, tokenizer=word_tokenize):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)
                for sent in sent_tokenize(line.lower()):
                    tokens = tokenizer(sent)
                    yield tokens
1102119
def _document_lines(self, text):
        
        inlines = text.split('\n')
        doc_lines = [(re.sub(r'^ +| +$', '', line), num)
                     for line, num
                     in zip(inlines, xrange(1, len(inlines) + 1))]
        return doc_lines
635380
def parse_document(lexer: Lexer) -> DocumentNode:
    """"""
    start = lexer.token
    return DocumentNode(
        definitions=many_nodes(lexer, TokenKind.SOF, parse_definition, TokenKind.EOF),
        loc=loc(lexer, start),
    )
1342852
def parse_doc(doc):
    """"""
    word_spans = []
    sentence_spans = []
    sentence_chunks = doc.split('\n\n')
    sentences = []
    for chunk in sentence_chunks:
        sent_texts, sent_labels = get_texts_and_labels(chunk.strip())
        sentences.append(list(zip(sent_texts, sent_labels)))
    return sentences
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
322562
def process_document(self, doc):
        

        segment = doc.select_segments("target_text")[0]

        for e in self.e_list:
            res = doc.extract(e, segment)
            doc.store(res, e.name)
        return list()
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
434095
function parse(contents, retained, parser) {
    var errors = [], data = [], lines = 0;

    // optional parser
    if (!parser) parser = JSON;

    // process each line of the file
    contents.toString().split('\n').forEach(function(line) {
        if (!line) return;
        lines++;
        try {
            data.push(parser.parse(line));
        } catch (e) {
            e.line = line;
            errors.push(e);
        }
    });

    // if no errors/data, don't return an empty array
    var ret = {};
    if (errors.length) ret.errors = errors;
    if (data.length) ret.data = data;

    // if every line failed, probably not line-delimited
    if (errors.length == lines) ret.errors = retained;

    return ret;
}
986150
def strip_docstrings(tokens):
    
    stack = []
    state = 'wait_string'
    for t in tokens:
        typ = t[0]
        if state == 'wait_string':
            if typ in (tokenize.NL, tokenize.COMMENT):
                yield t
            elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING):
                stack.append(t)
            elif typ == tokenize.NEWLINE:
                stack.append(t)
                start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
                for i in range(start_line, end_line):
                    yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
                for t in stack:
                    if t[0] in (tokenize.DEDENT, tokenize.INDENT):
                        yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4]
                del stack[:]
            else:
                stack.append(t)
                for t in stack: yield t
                del stack[:]
                state = 'wait_newline'
        elif state == 'wait_newline':
            if typ == tokenize.NEWLINE:
                state = 'wait_string'
            yield t
769684
def parse(filename, MAX_TERM_COUNT=1000):
    
    with open(filename, "r") as f:

        termId = None
        name = None
        desc = None
        parents = []

        termCount = 0
        for l in f.readlines():
            if l.startswith("id:"):
                termId = l.strip()[4:]
            if l.startswith("name:"):
                name = l.strip()[6:]
            elif l.startswith("def:"):
                desc = l.strip()[5:]
            elif l.startswith("is_a:"):
                pid = l.strip()[6:].split(" ", 1)[0]
                parents.append(pid)
            if len(l) == 1:     # newline
                # save
                if termId is not None and name is not None:
                    terms[termId] = {'name': name, 'desc': desc,
                                     'parents': parents[:], 'children': []}
                    termId = None
                    name = None
                    parents = []
                    termCount += 1
                    if MAX_TERM_COUNT is not None and \
                       termCount > MAX_TERM_COUNT:
                        break

    count = 0
    for tid, tdict in terms.items():
        count += 1      # purely for display
        for p in tdict['parents']:
            if p in terms.keys():
                terms[p]['children'].append(tid)

    # Get unique term IDs for Tag Groups.
    tagGroups = set()
    for tid, tdict in terms.items():
        # Only create Tags for GO:terms that are 'leafs' of the tree
        if len(tdict['children']) == 0:
            for p in tdict['parents']:
                tagGroups.add(p)

    return tagGroups, terms
115326
def convert_timestamps(files):
    ''''''

    sentences = []

    for f in files:

        if not f.endswith('.transcription.txt'):
            f = f + '.transcription.txt'

        if os.path.exists(f) is False:
            continue

        with open(f, 'r') as infile:
            lines = infile.readlines()

        lines = [re.sub(r'\(.*?\)', '', l).strip().split(' ') for l in lines]
        lines = [l for l in lines if len(l) == 4]

        seg_start = -1
        seg_end = -1

        for index, line in enumerate(lines):
            word, start, end, conf = line
            if word == '<s>' or word == '<sil>' or word == '</s>':
                if seg_start == -1:
                    seg_start = index
                    seg_end = -1
                else:
                    seg_end = index

                if seg_start > -1 and seg_end > -1:
                    words = lines[seg_start+1:seg_end]
                    start = float(lines[seg_start][1])
                    end = float(lines[seg_end][1])
                    if words:
                        sentences.append({'start': start, 'end': end, 'words': words, 'file': f})
                    if word == '</s>':
                        seg_start = -1
                    else:
                        seg_start = seg_end
                    seg_end = -1

    return sentences
444686
def get_tokens(line: str) -> Iterator[str]:
    
    for token in line.rstrip().split():
        if len(token) > 0:
            yield token
656214
def parse(self, lines, root):
    

    lst = IgnoreList(root)
    lst.parse(lines)
    self.append(lst)