Predicts the Language of a given text.

            :param text: Unicode text to be classified.
420051
def classify(self, text=u''):
        
        result = self.calculate(doc_terms=self.tokenize(text))
        #return (result['calc_id'], result)
        return (result['calc_id'], self.karbasa(result))
562069
def text(self, text):
        
        tc = self._tc
        tc.clear_content()
        p = tc.add_p()
        r = p.add_r()
        r.text = text
379169
def guess_lexer(_text, **options):
    """"""

    # try to get a vim modeline first
    ft = get_filetype_from_buffer(_text)

    if ft is not None:
        try:
            return get_lexer_by_name(ft, **options)
        except ClassNotFound:
            pass

    best_lexer = [0.0, None]
    for lexer in _iter_lexerclasses():
        rv = lexer.analyse_text(_text)
        if rv == 1.0:
            return lexer(**options)
        if rv > best_lexer[0]:
            best_lexer[:] = (rv, lexer)
    if not best_lexer[0] or best_lexer[1] is None:
        raise ClassNotFound('no lexer matching the text found')
    return best_lexer[1](**options)
1119544
def decode(self, text):
        
        # only decode byte strings into unicode if it hasn't already
        # been done by a subclass
        if isinstance(text, six.text_type):
            return text

        # empty text? nothing to decode
        if not text:
            return u''

        # use chardet to automatically detect the encoding text
        result = chardet.detect(text)
        return text.decode(result['encoding'])
348574
def lex(self, text):
        
        pos = 0
        while text:
            token_class, match = self.tokens.get_token(text)
            if token_class is not None:
                matched_text = text[match.start():match.end()]
                yield token_class(matched_text)
                text = text[match.end():]
                pos += match.end()
            elif text[0] in self.blank_chars:
                text = text[1:]
                pos += 1
            else:
                raise LexerError(
                        'Invalid character %s in %s' % (text[0], text),
                        position=pos)

        yield self.end_token()
545751
def scores(self, text: str) -> Dict[str, float]:
        
        values = extract(text)
        input_fn = _to_func(([values], []))
        prediction = self._classifier.predict_proba(input_fn=input_fn)
        probabilities = next(prediction).tolist()
        sorted_languages = sorted(self.languages)
        return dict(zip(sorted_languages, probabilities))
828377
def _prepare_text(self, text):
        
        # Remove characters that should be escaped for XML input (but
        # which cause problems when escaped, since they become
        # tokens).
        text = re.sub(r'[<>&]', '', text)
        pattern = r'({})'.format(self._tokenizer.pattern)
        return re.sub(pattern, self._base_token_markup, text)
1348385
def probable_languages(
            self,
            text: str,
            max_languages: int = 3) -> Tuple[str, ...]:
        
        scores = self.scores(text)

        # Sorted from the most probable language to the least probable
        sorted_scores = sorted(scores.items(), key=itemgetter(1), reverse=True)
        languages, probabilities = list(zip(*sorted_scores))

        # Find the most distant consecutive languages.
        # A logarithmic scale is used here because the probabilities
        # are most of the time really close to zero
        rescaled_probabilities = [log(proba) for proba in probabilities]
        distances = [
            rescaled_probabilities[pos] - rescaled_probabilities[pos+1]
            for pos in range(len(rescaled_probabilities)-1)]

        max_distance_pos = max(enumerate(distances, 1), key=itemgetter(1))[0]
        limit = min(max_distance_pos, max_languages)
        return languages[:limit]
41896
def tag(self, text):
        
        matches = self._match(text.text)
        matches = self._resolve_conflicts(matches)

        if self.return_layer:
            return matches
        else:
            text[self.layer_name] = matches
978181
def condition_from_text(text) -> Condition:
        
        try:
            condition = pypeg2.parse(text, output.Condition)
        except SyntaxError:
            # Invalid conditions are possible, see https://github.com/duniter/duniter/issues/1156
            # In such a case, they are store as empty PEG grammar object and considered unlockable
            condition = Condition(text)
        return condition
323479
def p_text(self, text):
        ''''''
        item = text[1]
        text[0] = item if item[0] != "\n" else u""
        if len(text) > 2:
            text[0] += "\n"
1017638
def is_text_type(text):
    
    if isinstance(text, six.text_type) or isinstance(text, six.string_types):
        return True

    return False
799804
def str_or_unicode(text):
    
    encoding = sys.stdout.encoding
    if sys.version_info > (3, 0):
        return text.encode(encoding).decode(encoding)
    return text.encode(encoding)
133749
def sentiment(
    text       = None,
    confidence = False
    ):
    
    try:
        words = text.split(" ")
        # Remove empty strings.
        words = [word for word in words if word]
        features = word_features(words)
        classification = classifier.classify(features)
        confidence_classification = classifier.prob_classify(features).prob(classification)
    except:
        classification = None
        confidence_classification = None
    if confidence:
        return (
            classification,
            confidence_classification
        )
    else:
        return classification
251802
@Override
    public String classify(String text) throws IllegalArgumentException, IllegalStateException
    {
        Map<String, Double> scoreMap = predict(text);

        return CollectionUtility.max(scoreMap);
    }
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
1362988
def learn(self, text):
        """"""
        if type(text) != types.UnicodeType:
            # Assume that non-Unicode text is encoded as utf-8, which
            # should be somewhat safe in the modern world.
            text = text.decode("utf-8", "ignore")

        tokens = self.tokenizer.split(text)
        trace("Brain.learn_input_token_count", len(tokens))

        self._learn_tokens(tokens)