Iterate through a DOCX file and yield text.

    Args:
        doc: DOCX document to process
        config: :class:`TextProcessingConfig` control object

    Yields:
        pieces of text (paragraphs)
422476
def docx_docx_gen_text(doc: DOCX_DOCUMENT_TYPE,
                       config: TextProcessingConfig) -> Iterator[str]:
    # only called if docx loaded
    
    if in_order:
        for thing in docx_docx_iter_block_items(doc):
            if isinstance(thing, docx.text.paragraph.Paragraph):
                yield docx_process_simple_text(thing.text, config.width)
            elif isinstance(thing, docx.table.Table):
                yield docx_process_table(thing, config)
    else:
        for paragraph in doc.paragraphs:
            yield docx_process_simple_text(paragraph.text, config.width)
        for table in doc.tables:
            yield docx_process_table(table, config)
280833
def parse_args(self, doc, argv):
        
        # first a little sneak peak if we have a generator
        arguments = docopt(doc, argv=argv, help=False)
        if arguments.get('<generator>'):
            name = arguments['<generator>']
            generator = self._get_generator(name)

            if hasattr(generator, 'DOC'):
                # register help for generator!
                # this runs after the generator was loaded so we have to
                # prepend the cmd!
                def _banana_help(args, router):
                    print(doc)

                cmd.register(lambda args: args.get('--help'), name,
                             _banana_help)
                doc = generator.DOC
                arguments = docopt(doc, argv=argv, help=False)

            # register generator '--version' cmd
            version = 'not provided by %s generator' % name
            if hasattr(generator, '__version__'):
                version = generator.__version__

            def _banana_version(args, router):
                print(version)

            cmd.register(lambda args: args.get('--version'), name,
                         _banana_version)

            # register generator interactive mode (last cmd for this generator)
            def _banana_run(args, router):
                router.navigate('run', name)
                router.navigate('exit')
            cmd.register(lambda args: True, name, _banana_run)

        return arguments
1319098
def apply(self, doc):
        
        if not isinstance(doc, Document):
            raise TypeError(
                "Input Contexts to MentionParagraphs.apply() must be of type Document"
            )

        for paragraph in doc.paragraphs:
            yield TemporaryParagraphMention(paragraph)
1311787
def _do_document(self, node):
        ''''''

        self.documentOrder = _LesserElement
        for child in node.childNodes:
            if child.nodeType == Node.ELEMENT_NODE:
                self.documentOrder = _Element        # At document element
                self._do_element(child)
                self.documentOrder = _GreaterElement # After document element
            elif child.nodeType == Node.PROCESSING_INSTRUCTION_NODE:
                self._do_pi(child)
            elif child.nodeType == Node.COMMENT_NODE:
                self._do_comment(child)
            elif child.nodeType == Node.DOCUMENT_TYPE_NODE:
                pass
            else:
                raise TypeError, str(child)
417664
def yield_deps(doc):
    

    this_node = DependencyNode(doc, None)

    for r in doc.references():

        other_node = DependencyNode(r, this_node)

        yield (this_node, other_node)

        if isinstance(other_node, PackageDependencyNode):
            yield from yield_deps(other_node.doc)
198386
def docs(self: object, fileids:str):
        

        for path, encoding in self.abspaths(fileids, include_encoding=True):
            with codecs.open(path, 'r', encoding=encoding) as f:
                yield f.read()
323099
def _generate(self):
        """"""
        doc_count = 0
        for fp in self.all_files:
            for doc in self._get_docs_for_path(fp):
                yield doc
                doc_count += 1
                if doc_count >= self.max_docs:
                    return
1102119
def _document_lines(self, text):
        
        inlines = text.split('\n')
        doc_lines = [(re.sub(r'^ +| +$', '', line), num)
                     for line, num
                     in zip(inlines, xrange(1, len(inlines) + 1))]
        return doc_lines
1037592
def generate_docstr(element, indent='', wrap=None):
    
    result = []

    txt = element.text and element.text.rstrip()
    if txt:
        result.append(_reindent(txt, indent))
        result.append(indent)

    for d in element.findall('doc') + element.findall('rule'):
        docval = ''.join(d.textlist()).rstrip()
        if not docval:
            continue
        reformat = True
        if 'name' in d.attrib:
            result.append(indent + d.attrib['name'].upper() + ':')
            result.append(indent)
            extra_indent = '    '
            if d.attrib['name'] == 'grammar':
                reformat = False # Don't want re-indenting to mess this up
        elif d.tag == 'rule':
            result.append(indent + 'RULE:')
            result.append(indent)
            extra_indent = '    '
        else:
            extra_indent = ''
        result.append(_reindent(docval, indent + extra_indent, reformat))
        result.append(indent)

    fields = element.findall('field')
    if fields:
        result.append(indent + 'PARAMETERS:')
        for f in fields:
            result.append(indent + '    ' + _fixup_field_name(f) + ': ' + _field_type(f))
            field_docs = generate_docstr(f, indent + '        ')
            if field_docs:
                result.append(indent)
                result.append(field_docs)
            result.append(indent)

    if not result:
        return None

    if wrap is not None:
        result = [wrap] + result + [wrap]

    return '\n'.join(x.rstrip() for x in result) + '\n'
109363
def _process_doc(self, doc):
        
        # Prep
        doc = doc.lower()
        tagged_tokens = [(t, penn_to_wordnet(t.tag_)) for t in spacy(doc, tag=True, parse=False, entity=False)]
        tokens = [t for t, tag in tagged_tokens]
        term_concept_map = self._disambiguate_doc(tagged_tokens)
        concept_weights = self._weight_concepts(tokens, term_concept_map)

        # Compute core semantics
        lexical_chains = self._lexical_chains(doc, term_concept_map)
        core_semantics = self._core_semantics(lexical_chains, concept_weights)
        core_concepts = [c for chain in core_semantics for c in chain]

        return [(con, concept_weights[con]) for con in core_concepts]
1150591
def sentence_iterator(self, file_path: str) -> Iterator[OntonotesSentence]:
        
        for document in self.dataset_document_iterator(file_path):
            for sentence in document:
                yield sentence
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
603150
public void processDocument(BufferedReader document) throws IOException {
        Queue<String> prevWords = new ArrayDeque<String>(windowSize);
        Queue<String> nextWords = new ArrayDeque<String>(windowSize);

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        String focusWord = null;

        // Prefetch the first windowSize words.  As soon as a word enters the
        // nextWords buffer increase its occurrence count.
        for (int i = 0; i < windowSize && documentTokens.hasNext(); ++i) 
            nextWords.offer(documentTokens.next());        
        
        while (!nextWords.isEmpty()) {
            
            focusWord = nextWords.remove();

            // shift over the window to the next word
            if (documentTokens.hasNext()) {
                String windowEdge = documentTokens.next(); 
                nextWords.offer(windowEdge);
            }    
                
            // Don't bother calculating the semantics for empty tokens
            // (i.e. words that were filtered out)
            if (!focusWord.equals(IteratorFactory.EMPTY_TOKEN)) {
                SemanticVector focusMeaning = getSemanticVector(focusWord);

                // Sum up the index vector for all the surrounding words.  If
                // permutations are enabled, permute the index vector based on
                // its relative position to the focus word.
                int permutations = -(prevWords.size());        
                for (String word : prevWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                            
                    updateSemantics(focusMeaning, word, iv);
                }
                
                // Repeat for the words in the forward window.
                permutations = 1;
                for (String word : nextWords) {
                    // Skip the addition of any words that are excluded from the
                    // filter set.  Note that by doing the exclusion here, we
                    // ensure that the token stream maintains its existing
                    // ordering, which is necessary when permutations are taken
                    // into account.
                    if (word.equals(IteratorFactory.EMPTY_TOKEN)) {
                        ++permutations;
                        continue;
                    }
                    
                    TernaryVector iv = wordToIndexVector.get(word);
                    if (usePermutations) {
                        iv = permutationFunc.permute(iv, permutations);
                        ++permutations;
                    }
                    
                    updateSemantics(focusMeaning, word, iv);
                }
            }

            // Last put this focus word in the prev words and shift off the
            // front of the previous word window if it now contains more words
            // than the maximum window size
            prevWords.offer(focusWord);

            // Increment the frequency count for the word now that it has been
            // seen and processed.
            Integer count = wordToOccurrences.get(focusWord);
            wordToOccurrences.put(focusWord, (count == null) ? 1 : count + 1);

            if (prevWords.size() > windowSize) {
                prevWords.remove();
            }
        }    

        document.close();
    }
955102
def fold_text_ctx(self, line):
        
        if not self.dom_term:
            self.__call__(line)
            yield
            return
        self.lines, old_lines = [], self.lines  # overwrite self.lines
        yield  # collect output (in new self.lines)
        self.lines, new_lines = old_lines, self.lines  # recover self.lines
        hidden_text = "".join(new_lines)
        import io
        output_buf = io.StringIO()
        prefix = ""
        while line[:1] == " ":
            prefix += " "
            line = line[1:]
        self.dom_term.fold_text(line, hidden=hidden_text, file=output_buf, align=len(prefix))
        output_text = prefix[1:] + output_buf.getvalue()
        self.lines.append(output_text)
1004811
def parse_doc(f, ctx=None, overwrite=False):
    
    lg = LineGetter(f, comment_marker=("#", ";"), strip=False)
    cfg = ConfigParser(ctx).parse_doc(lg)
    set_defaults(cfg)
    if overwrite:
        squash(cfg)
    return cfg
1180403
def apply(self, doc):
        
        if not isinstance(doc, Document):
            raise TypeError(
                "Input Contexts to MentionSections.apply() must be of type Document"
            )

        for section in doc.sections:
            yield TemporarySectionMention(section)
331563
def simple_preprocess(doc, deacc=False, min_len=2, max_len=15):
    
    tokens = [
        token for token in tokenize(doc, lower=True, deacc=deacc, errors='ignore')
        if min_len <= len(token) <= max_len and not token.startswith('_')
    ]
    return tokens