Extracts features from a body of text.

        :rtype: dictionary of features
433019
def extract_features(self, text):
        
        # Feature extractor may take one or two arguments
        try:
            return self.feature_extractor(text, self.train_set)
        except (TypeError, AttributeError):
            return self.feature_extractor(text)
670427
def word_tokenize(self, text: str) -> List[str]:
        
        return word_tokenize(text, custom_dict=self.__trie_dict, engine=self.__engine)
426064
def analyze(self, text, tokenizer=str.split):
        
        if not self.tagger:
            self.tagger = Tagger(self.model,
                                 preprocessor=self.p,
                                 tokenizer=tokenizer)

        return self.tagger.analyze(text)
741783
def from_text(cls, text, mapping='mapping'):
        
        graphemes = Counter(grapheme_pattern.findall(text))
        specs = [
            OrderedDict([
                (cls.GRAPHEME_COL, grapheme),
                ('frequency', frequency),
                (mapping, grapheme)])
            for grapheme, frequency in graphemes.most_common()]
        return cls(*specs)
823349
def __extract_description(self):
        
        tmp = list()
        for line in self._comment:
            if len(line) >= 1 and line[0] == '@':
                break

            tmp.append(line)

        tmp = self.__remove_trailing_empty_lines(tmp)

        self._description = os.linesep.join(tmp)
533689
def _filter(self, text, context, encoding):
        """"""

        docstrings = []
        strings = []
        comments = []
        prev_token_type = tokenize.NEWLINE
        indent = ''
        name = None
        stack = [(context, 0, self.MODULE)]
        last_comment = False
        possible_fmt_str = None

        src = io.StringIO(text)

        for token in tokenizer(src.readline):
            token_type = token[0]
            value = token[1]
            line = str(token[2][0])
            line_num = token[2][0]

            value = token[1]
            line = str(token[2][0])

            # Track function and class ancestry
            if token_type == tokenize.NAME:
                possible_fmt_str = None
                if value in ('def', 'class'):
                    name = value
                elif name:
                    parent = stack[-1][2]
                    prefix = ''
                    if parent != self.MODULE:
                        prefix = '.' if parent == self.CLASS else ', '
                    if name == 'class':
                        stack.append(('%s%s' % (prefix, value), len(indent), self.CLASS))
                    elif name == 'def':
                        stack.append(('%s%s()' % (prefix, value), len(indent), self.FUNCTION))
                    name = None
                elif not F_SUPPORT and value in FMT_STR:
                    possible_fmt_str = (prev_token_type, value)
            elif token_type != tokenize.STRING:
                possible_fmt_str = None

            if token_type == tokenize.COMMENT and self.comments:
                # Capture comments
                if (
                    self.group_comments and
                    last_comment and
                    comments and (comments[-1][2] + 1) == line_num
                ):
                    # Group multiple consecutive comments
                    comments[-1][0] += '\n' + value[1:]
                    comments[-1][2] = line_num
                else:
                    if len(stack) > 1:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                    else:
                        loc = "%s(%s)" % (stack[0][0], line)
                    comments.append([value[1:], loc, line_num])
            if token_type == tokenize.STRING:
                # Capture docstrings.
                # If we captured an `INDENT` or `NEWLINE` previously we probably have a docstring.
                # `NL` means end of line, but not the end of the Python code line (line continuation).
                if (
                    (prev_token_type in PREV_DOC_TOKENS) or
                    (possible_fmt_str and possible_fmt_str[0] in PREV_DOC_TOKENS)
                ):
                    if self.docstrings:
                        value = value.strip()
                        if possible_fmt_str and value.startswith(("'", "\"")):
                            value = possible_fmt_str[1] + value
                        string, is_bytes = self.process_strings(value, docstrings=True)
                        if string:
                            loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                            docstrings.append(
                                filters.SourceText(string, loc, 'utf-8', 'py-docstring')
                            )
                elif self.strings:
                    value = value.strip()
                    if possible_fmt_str and value.startswith(("'", "\"")):
                        value = possible_fmt_str[1] + value
                    string, is_bytes = self.process_strings(value)
                    if string:
                        loc = "%s(%s): %s" % (stack[0][0], line, ''.join([crumb[0] for crumb in stack[1:]]))
                        strings.append(filters.SourceText(string, loc, 'utf-8', 'py-string'))

            if token_type == tokenize.INDENT:
                indent = value
            elif token_type == tokenize.DEDENT:
                indent = indent[:-4]
                if len(stack) > 1 and len(indent) <= stack[-1][1]:
                    stack.pop()

            # We purposefully avoid storing comments as comments can come before docstrings,
            # and that can mess up our logic. So if the token is a comment we won't track it,
            # and comments are always followed with `NL` so we ignore that as well.
            # We only care that docstrings are preceded by `NEWLINE`.
            if token_type != tokenize.COMMENT and not (last_comment and token_type == tokenize.NL):
                prev_token_type = token_type
                last_comment = False
            else:
                last_comment = True

        final_comments = []
        for comment in comments:
            final_comments.append(filters.SourceText(textwrap.dedent(comment[0]), comment[1], encoding, 'py-comment'))

        return docstrings + final_comments + strings
955589
def text_to_data(self, text, elt, ps):
        '''
        '''
        if self.strip: text = text.strip()
        if self.pyclass is not None:
            return self.pyclass(text.encode(UNICODE_ENCODING))
        return text.encode(UNICODE_ENCODING)
813314
def _parse_features(cls, feat_response):
        # type: (Text) -> Dict[Text, Text]
        
        features = {}
        if feat_response.split("-")[0] == "211":
            for line in feat_response.splitlines():
                if line.startswith(" "):
                    key, _, value = line[1:].partition(" ")
                    features[key] = value
        return features
522610
def extract(self):
        

        out = {}

        for key, group in self.data.items():
            out[key] = group

        return out
701972
def _parse(self, text):
        '''
        
        '''
        # ensure unicode first
        text = str(text).strip()

        if not text.startswith(DELIMITER):
            return {}, text

        try:
            _, fm, content = BOUNDARY.split(text, 2)
        except ValueError:
            # if we can't split, bail
            return {}, text
        # loader_class = self.loader_factory(self)
        metadata = yaml.load(fm, Loader=self.loader_class)
        metadata = metadata if (isinstance(metadata, dict)) else {}
        return metadata, content
1277217
def segment(text: str, custom_dict: Trie = None) -> List[str]:
    
    if not text or not isinstance(text, str):
        return []

    return list(_multicut(text, custom_dict=custom_dict))
1144798
def analyze(text, features = default_features)
      if text.nil? || features.nil?
        raise ArgumentError.new(NIL_ARGUMENT_ERROR)
      end

      response = self.class.post(
        "#{@url}/analyze?version=#{@version}",
        body: {
          text: text.to_s,
          features: features
        }.to_json,
        basic_auth: auth,
        headers: {
          "Content-Type" => CONTENT_TYPE
        }
      )

      response.parsed_response
    end
642581
def get_labels_and_texts(self, metadata=None):
		'''
		
		'''
		to_ret = {'categories': self._corpus.get_categories(), 'labels': [], 'texts': []}
		labels = self._corpus._y.astype(int)
		texts = self._get_texts_to_display()
		if self._use_non_text_features:
			to_ret['extra'] = []
			extrafeats = self._corpus.list_extra_features()
		if metadata is not None:
			to_ret['meta'] = []
		for label_i in range(len(self._corpus._category_idx_store)):
			label_indices = np.arange(0, len(labels))[labels == label_i]
			if self.max_per_category < len(label_indices):
				label_indices = np.random.choice(label_indices, self.max_per_category, replace=False)
				to_ret['labels'] += list([int(e) for e in labels[label_indices]])
				to_ret['texts'] += list(texts[label_indices])
				if metadata is not None:
					to_ret['meta'] += [metadata[i] for i in label_indices]
				if self._use_non_text_features:
					to_ret['extra'] += [extrafeats[i] for i in label_indices]

		return to_ret
767328
def parse(text):
    
    rv = {}
    m = META.match(text)

    while m:
        key = m.group(1)
        value = m.group(2)
        value = INDENTATION.sub('\n', value.strip())
        rv[key] = value
        text = text[len(m.group(0)):]
        m = META.match(text)

    return rv, text
94257
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._parser(text)

        extractions = list()
        for sent in doc.sents:
            this_extraction = Extraction(value=sent.text,
                                         extractor_name=self.name,
                                         start_token=sent[0],
                                         end_token=sent[-1],
                                         start_char=sent.text[0],
                                         end_char=sent.text[-1])
            extractions.append(this_extraction)

        return extractions
224655
def _extract_line_features(self):
        
        for line in self.line_iterator:
            m = COMPILED_AUTH_LOG_REGEX.match(line)

            data = {
                'raw': line
            }

            if m:
                data.update({
                    'timestamp': self._to_epoch(m.group(1)),
                    'hostname': m.group(2),
                    'program': m.group(3),
                    'processid': m.group(4),
                    'message': m.group(5),
                })

            self.parsed_lines.append(data)
586338
def extract_keywords_from_text(self, text):
        
        sentences = nltk.tokenize.sent_tokenize(text)
        self.extract_keywords_from_sentences(sentences)