Analyze text and return pretty format.

        Args:
            text: string, the input text.
            tokenizer: Tokenize input sentence. Default tokenizer is `str.split`.

        Returns:
            res: dict.
426064
def analyze(self, text, tokenizer=str.split):
        
        if not self.tagger:
            self.tagger = Tagger(self.model,
                                 preprocessor=self.p,
                                 tokenizer=tokenizer)

        return self.tagger.analyze(text)
264674
def parse(text: str, style: Style = Style.auto) -> Docstring:
    

    if style != Style.auto:
        return _styles[style](text)
    rets = []
    for parse_ in _styles.values():
        try:
            rets.append(parse_(text))
        except ParseError as e:
            exc = e
    if not rets:
        raise exc
    return sorted(rets, key=lambda d: len(d.meta), reverse=True)[0]
1284404
def text(self, text):
        
        ret = u"".join(text.content)
        if 'url' in text.properties:
            return u"`%s`_" % ret
        if 'bold' in text.properties:
            return u"**%s**" % ret
        if 'italic' in text.properties:
            return u"*%s*" % ret
        if 'sub' in text.properties:
            return ur"\ :sub:`%s`\ " % ret
        if 'super' in text.properties:
            return ur"\ :sup:`%s`\ " % ret
        return ret
1102062
def split_sentences_regex(text):
    
    parts = regex.split(r'([a-zA-Z0-9][.?!])[\s$]', text)
    sentences = [''.join(s) for s in zip(parts[0::2], parts[1::2])]
    return sentences + [parts[-1]] if len(parts) % 2 else sentences
1367900
def classify(self, text):
        
        text_features = self.extract_features(text)
        return self.classifier.classify(text_features)
178349
def parse(self, text):
        
        gen = self.lexer.lex(text)
        next_val_fn = partial(next, *(gen,))
        token = next_val_fn()
        return self.rule_svg_path(next_val_fn, token)
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
890968
def extract(self, text):
        
        _extracted = []
        if text.strip() == "":
            return _extracted
        parsed_sentences = self._parse_text(text)
        for s in parsed_sentences:
            tokens = s.split()
            new_np = []
            for t in tokens:
                w, tag, phrase, role = t.split('/')
                # exclude some parser errors (e.g. VB within NP),
                # extend startswith tuple if necessary
                if 'NP' in phrase and not self._is_verb(w, tag):
                    if len(new_np) > 0 and w.lower() in START_NEW_NP:
                        _extracted.append(" ".join(new_np))
                        new_np = [w]
                    else:
                        # normalize capitalisation of sentence starters, except
                        # for nouns
                        new_np.append(w.lower() if tokens[0].startswith(w) and
                                      not tag.startswith('N') else w)
                else:
                    if len(new_np) > 0:
                        _extracted.append(" ".join(new_np))
                    new_np = []
        return self._filter_extracted(_extracted)
821806
def split_sentences(self, text):
        

        if self.model.has_pipe("sentence_boundary_detector"):
            self.model.remove_pipe(name="sentence_boundary_detector")

        if not self.model.has_pipe("sentencizer"):
            sentencizer = self.model.create_pipe("sentencizer")  # add sentencizer
            self.model.add_pipe(sentencizer)
        try:
            doc = self.model(text, disable=["parser", "tagger", "ner"])
        except ValueError:
            # temporary increase character limit of spacy
            # 'Probably save' according to spacy, as no parser or NER is used
            previous_max_length = self.model.max_length
            self.model.max_length = 100_000_000
            self.logger.warning(
                f"Temporarily increased spacy maximum "
                f"character limit to {self.model.max_length} to split sentences."
            )
            doc = self.model(text, disable=["parser", "tagger", "ner"])
            self.model.max_length = previous_max_length
            self.logger.warning(
                f"Spacy maximum "
                f"character limit set back to {self.model.max_length}."
            )

        doc.is_parsed = True
        position = 0
        for sent in doc.sents:
            parts = defaultdict(list)
            text = sent.text

            for i, token in enumerate(sent):
                parts["words"].append(str(token))
                parts["lemmas"].append(token.lemma_)
                parts["pos_tags"].append(token.pos_)
                parts["ner_tags"].append("")  # placeholder for later NLP parsing
                parts["char_offsets"].append(token.idx)
                parts["abs_char_offsets"].append(token.idx)
                parts["dep_parents"].append(0)  # placeholder for later NLP parsing
                parts["dep_labels"].append("")  # placeholder for later NLP parsing

            # make char_offsets relative to start of sentence
            parts["char_offsets"] = [
                p - parts["char_offsets"][0] for p in parts["char_offsets"]
            ]
            parts["position"] = position
            parts["text"] = text

            position += 1

            yield parts
1358946
def tokenize_words(self, text):
        """"""
        return [
            self.strip_punctuation(word) for word in text.split(' ')
            if self.strip_punctuation(word)
        ]
935282
def analyse_text(text):
        
        result = 0.0
        lines = text.split('\n')
        if len(lines) > 0:
            if JclLexer._JOB_HEADER_PATTERN.match(lines[0]):
                result = 1.0
        assert 0.0 <= result <= 1.0
        return result
1302431
def parse(self, text):
        
        #: Do not process empty strings (Issue #3)
        if text.strip() == "":
            return ""
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif text.strip() in PUNCTUATION:
            _sym = text.strip()
            if _sym in tuple('.?!'):
                _tag = "."
            else:
                _tag = _sym
            if self.lemmata:
                return "{0}/{1}/O/O/{0}".format(_sym, _tag)
            else:
                return "{0}/{1}/O/O".format(_sym, _tag)
        if self.tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(text))
        else:
            _tokenized = text

        _parsed = pattern_parse(_tokenized,
                                # text is tokenized before it is passed on to
                                # pattern.de.parse
                                tokenize=False,
                                tags=self.tags, chunks=self.chunks,
                                relations=self.relations, lemmata=self.lemmata,
                                encoding=self.encoding, tagset=self.tagset)
        if self.pprint:
            _parsed = pattern_pprint(_parsed)

        return _parsed
727170
function parse(text, language) {
  return text.split(newlines)
             .map(parseLine(language))
             .reduce(assimilateSimilarTokens, []) }
244400
public String process(String text){
		String ret = text;
		
		//Pipeline
		ret = convertor.convert(ret);
		ret = senSegment(ret);
		ret = senTokenize(ret);
		ret = wordSegment(ret);
		ret = postProcessing(ret);
		ret = posTagging(ret);
		return ret;
	}
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
996897
public List<AnalyzedSentence> analyzeText(String text) throws IOException {
    List<String> sentences = sentenceTokenize(text);
    return analyzeSentences(sentences);
  }
94257
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._parser(text)

        extractions = list()
        for sent in doc.sents:
            this_extraction = Extraction(value=sent.text,
                                         extractor_name=self.name,
                                         start_token=sent[0],
                                         end_token=sent[-1],
                                         start_char=sent.text[0],
                                         end_char=sent.text[-1])
            extractions.append(this_extraction)

        return extractions