Extract surface word and feature from analyzed lines.
        Extracted results are returned with list, whose elements are TokenizedResult class
        [TokenizedResult]
548495
def __postprocess_analyzed_result(self, string_mecab_parsed_result, is_feature, is_surface):
        # type: (text_type,bool,bool)->List[TokenizedResult]
        
        assert isinstance(string_mecab_parsed_result, str)
        check_tab_separated_line = lambda x: True if '\t' in x else False

        tokenized_objects = [
            self.__result_parser(analyzed_line=analyzed_line,
                                 is_feature=is_feature,
                                 is_surface=is_surface)
            for analyzed_line in string_mecab_parsed_result.split('\n')
            if not analyzed_line=='EOS' and check_tab_separated_line(analyzed_line)
        ]

        assert isinstance(tokenized_objects, list)
        return tokenized_objects
1262208
def extract_columns(self, result={}, selector='', table_headers=[], attr='', connector='', default='', verbosity=0, *args, **kwargs):
		
		result_list = []

		try:
			if type(selector) in [str, unicode]:
				selectors = [selector]
			elif type(selector) == list:
				selectors = selector[:]
			else:
				raise Exception("Use a list of selector expressions for the various columns")
			from itertools import izip, count
			pairs = izip(table_headers, selectors)
			columns = {}
			for head, selector in pairs:
				columns[head] = self.get_tree_tag(selector)
			try:
				for i in count(start=0):
					r = result.copy()
					for head in columns.keys():
						if verbosity > 1:
							print("\nExtracting", head, "attribute", sep=' ', end='')
						col = columns[head][i]
						if attr == "text":
							try:
								content = connector.join([make_ascii(x).strip() for x in col.itertext()])
							except Exception:
								content = default
							content = content.replace("\n", " ").strip()
						else:
							content = col.get(attr)
							if attr in ["href", "src"]:
								content = urljoin(self.url, content)
						r[head] = content
					result_list.append(r)
			except IndexError:
				pass
		except XPathError:
			raise Exception("Invalid %s selector - %s" % (self.__selector_type__, selector))
		except TypeError:
			raise Exception("Selector expression string to be provided. Got " + selector)

		return result_list
1214406
def tokenize(self, vector_list):
        '''
        
        '''
        if self.computable_distance is None:
            self.computable_distance = EuclidDistance()
        vector_arr = np.array(vector_list)
        distance_arr = np.empty_like(vector_arr)
        feature_arr = self.__dbm.get_feature_point(layer_number=0)
        key_arr = np.empty(vector_arr.shape[0], dtype=int)
        for i in range(vector_arr.shape[0]):
            distance_arr = self.computable_distance.compute(
                np.expand_dims(vector_arr[i], axis=0).repeat(feature_arr.shape[0], axis=0),
                feature_arr
            )
            key_arr[i] = distance_arr.argmin(axis=0)
        return self.token_arr[key_arr]
1309589
function(features) {
        var collection, geometry, isCollection;
        if (features.constructor == Array) {
            collection = features;
            isCollection = true;
        } else {
            collection = [features];
            isCollection = false;
        }
        var pieces = [];
        if (isCollection) {
            pieces.push('GEOMETRYCOLLECTION(');
        }
        for (var i=0, len=collection.length; i<len; ++i) {
            if (isCollection && i>0) {
                pieces.push(',');
            }
            geometry = collection[i].geometry;
            pieces.push(this.extractGeometry(geometry));
        }
        if (isCollection) {
            pieces.push(')');
        }
        return pieces.join('');
    }
505335
def print_featurelist(feature_list):
    
    input_features = sum(map(lambda n: n.get_dimension(), feature_list))
    print("## Features (%i)" % input_features)
    print("```")
    for algorithm in feature_list:
        print("* %s" % str(algorithm))
    print("```")
177381
def extract(self, item, article_candidate_list):
        
        list_text = []

        # The minimal number of words a text needs to have
        min_number_words = 15

        # The texts of the article candidates and the respective extractors are saved in a tuple in list_text.
        for article_candidate in article_candidate_list:
            if article_candidate.text != None:
                list_text.append((article_candidate.text, article_candidate.extractor))

        # Remove texts that are shorter than min_number_words.
        for text_tuple in list_text:
            if len(text_tuple[0].split()) < min_number_words:
                list_text.remove(text_tuple)

        # If there is no value in the list, return None.
        if len(list_text) == 0:
            return None

        # If there is only one solution, return it.
        if len(list_text) < 2:
            return list_text[0][0]
        else:

            # If there is more than one solution, do the following:

            # Create a list which holds triple of the score and the two extractors
            list_score = []

            # Compare every text with all other texts at least once
            for a, b, in itertools.combinations(list_text, 2):

                # Create sets from the texts
                set_a = set(a[0].split())
                set_b = set(b[0].split())
                symmetric_difference_a_b = set_a ^ set_b
                intersection_a_b = set_a & set_b

                # Replace 0 with -1 in order to elude division by zero
                if intersection_a_b == 0:
                    intersection_a_b = -1

                # Create the score. It divides the number of words which are not in both texts by the number of words which
                # are in both texts and subtracts the result from 1. The closer to 1 the more similiar they are.
                score = 1 - ((len(symmetric_difference_a_b)) / (2 * len(intersection_a_b)))
                list_score.append((score, a[1], b[1]))

            # Find out which is the highest score
            best_score = max(list_score, key=lambda item: item[0])

            # If one of the solutions is newspaper return it
            if "newspaper" in best_score:
                return (list(filter(lambda x: x[1] == "newspaper", list_text))[0][0])
            else:
                # If not, return the text that is longer

                # A list that holds the extracted texts and their extractors which were most similar
                top_candidates = []
                for tuple in list_text:
                    if tuple[1] == best_score[1] or tuple[1] == best_score[2]:
                        top_candidates.append(tuple)

                if len(top_candidates[0][0]) > len(top_candidates[1][0]):
                    return (top_candidates[0][0])
                else:
                    return (top_candidates[1][0])
137194
def _read_parsed(self, lines):
        
        self.log(u"Parsing fragments from parsed text format")
        pairs = []
        for line in lines:
            pieces = line.split(gc.PARSED_TEXT_SEPARATOR)
            if len(pieces) == 2:
                identifier = pieces[0].strip()
                text = pieces[1].strip()
                if len(identifier) > 0:
                    pairs.append((identifier, [text]))
        self._create_text_fragments(pairs)
1243795
def extract_objects_from_source(self, text, type_filter=None):
    '''
    '''
    objects = parse_vhdl(text)
    self._register_array_types(objects)

    if type_filter:
      objects = [o for o in objects if isinstance(o, type_filter)]

    return objects
558232
def extract_spellout_values(text):
    """"""
    values = []
    for item in r.REG_TXT.finditer(text):
        surface, span = clean_surface(item.group(0), item.span())
        if not surface or surface.lower() in r.SCALES:
            continue
        curr = result = 0.0
        for word in surface.split():
            try:
                scale, increment = 1, float(word.lower())
            except ValueError:
                scale, increment = r.NUMWORDS[word.lower()]
            curr = curr * scale + increment
            if scale > 100:
                result += curr
                curr = 0.0
        values.append({'old_surface': surface,
                       'old_span': span,
                       'new_surface': unicode(result + curr)})

    for item in re.finditer(r'\d+(,\d{3})+', text):
        values.append({'old_surface': item.group(0),
                       'old_span': item.span(),
                       'new_surface': unicode(item.group(0).replace(',', ''))})

    return sorted(values, key=lambda x: x['old_span'][0])
701036
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._tokenizer.tokenize_to_spacy_doc(text)
        self._load_matcher()

        matches = [x for x in self._matcher(doc) if x[1] != x[2]]
        pos_filtered_matches = []
        neg_filtered_matches = []
        for idx, start, end in matches:
            span_doc = self._tokenizer.tokenize_to_spacy_doc(doc[start:end].text)
            this_spacy_rule = self._matcher.get(idx)
            relations = self._find_relation(span_doc, this_spacy_rule)
            rule_id, _ = self._hash_map[idx]
            this_rule = self._rule_lst[rule_id]
            if self._filter_match(doc[start:end], relations, this_rule.patterns):
                value = self._form_output(doc[start:end], this_rule.output_format, relations, this_rule.patterns)
                if this_rule.polarity:
                    pos_filtered_matches.append((start, end, value, rule_id, relations))
                else:
                    neg_filtered_matches.append((start, end, value, rule_id, relations))

        return_lst = []
        if pos_filtered_matches:
            longest_lst_pos = self._get_longest(pos_filtered_matches)
            if neg_filtered_matches:
                longest_lst_neg = self._get_longest(neg_filtered_matches)
                return_lst = self._reject_neg(longest_lst_pos, longest_lst_neg)
            else:
                return_lst = longest_lst_pos

        extractions = []
        for (start, end, value, rule_id, relation) in return_lst:
            this_extraction = Extraction(value=value,
                                         extractor_name=self.name,
                                         start_token=start,
                                         end_token=end,
                                         start_char=doc[start].idx,
                                         end_char=doc[end-1].idx+len(doc[end-1]),
                                         rule_id=rule_id.split("rule_id##")[0],
                                         match_mapping=relation)
            extractions.append(this_extraction)

        return extractions
1241776
def find_lines_with(cls, lines, what):
        
        result = []
        for line in lines:
            if what in line:
                result = result + [line]
        return result
1124410
def feature_extraction(self, algorithms):
        """"""
        assert type(algorithms) is list
        features = []
        for algorithm in algorithms:
            new_features = algorithm(self)
            assert len(new_features) == algorithm.get_dimension(), \
                "Expected %i features from algorithm %s, got %i features" % \
                (algorithm.get_dimension(), str(algorithm), len(new_features))
            features += new_features
        return features
967257
def feature_detector(self, tokens, index, history):
        

        word = tokens[index]
        if index == 0: # At the beginning of the sentence
            prevword = prevprevword = None
            prevtag = prevprevtag = None
            #word = word.lower() # Lowercase at the beginning of sentence
        elif index == 1:
            prevword = tokens[index-1] # Note: no lowercase
            prevprevword = None
            prevtag = history[index-1]
            prevprevtag = None
        else:
            prevword = tokens[index-1]
            prevprevword = tokens[index-2]
            prevtag = history[index-1]
            prevprevtag = history[index-2]

        if re.match('[0-9]+([\.,][0-9]*)?|[0-9]*[\.,][0-9]+$', word):
            # Included "," as decimal point
            shape = 'number'
        elif re.compile('\W+$', re.UNICODE).match(word):
            # Included unicode flag
            shape = 'punct'
        elif re.match('([A-ZÄÖÜ]+[a-zäöüß]*-?)+$', word):
            # Included dash for dashed words and umlauts
            shape = 'upcase'
        elif re.match('[a-zäöüß]+', word):
            # Included umlauts
            shape = 'downcase'
        elif re.compile("\w+", re.UNICODE).match(word):
            # Included unicode flag
            shape = 'mixedcase'
        else:
            shape = 'other'

        features = {
            'prevtag': prevtag,
            'prevprevtag': prevprevtag,
            'word': word,
            'word.lower': word.lower(),
            'suffix3': word.lower()[-3:],
            #'suffix2': word.lower()[-2:],
            #'suffix1': word.lower()[-1:],
            'preffix1': word[:1], # included
            'prevprevword': prevprevword,
            'prevword': prevword,
            'prevtag+word': '%s+%s' % (prevtag, word),
            'prevprevtag+word': '%s+%s' % (prevprevtag, word),
            'prevword+word': '%s+%s' % (prevword, word),
            'shape': shape
            }
        return features
52313
def _extract_fields(self, line):
        
        acc = {}
        field_tokens = line.split("\t")
        for field_token in field_tokens:
            kv_tokens = field_token.split('::')
            if len(kv_tokens) == 2:
                (key, value) = kv_tokens
                acc[key] = value

        return acc
1001189
def analyze(self, text):
        
        try:
            self.process  # make sure things are loaded
            text = render_safe(text).replace('\n', ' ').lower()
            results = []
            for chunk in string_pieces(text):
                self.send_input((chunk + '\n').encode('utf-8'))
                while True:
                    out_line = self.receive_output_line().decode('utf-8')
                    if out_line == 'EOS\n':
                        break

                    word, info = out_line.strip('\n').split('\t')
                    record_parts = [word] + info.split(',')

                    # Pad the record out to have 10 parts if it doesn't
                    record_parts += [None] * (10 - len(record_parts))
                    record = MeCabRecord(*record_parts)

                    # special case for detecting nai -> n
                    if (record.surface == 'ん' and
                        record.conjugation == '不変化型'):
                        # rebuild the record so that record.root is 'nai'
                        record_parts[MeCabRecord._fields.index('root')] = 'ない'
                        record = MeCabRecord(*record_parts)

                    results.append(record)
            return results
        except ProcessError:
            self.restart_process()
            return self.analyze(text)
943803
def _extract_sel_info(sel):
    """"""
    from cssselect2.parser import (CombinedSelector, CompoundSelector,
                                   PseudoClassSelector,
                                   FunctionalPseudoClassSelector)
    steps = []
    extras = []
    if isinstance(sel, CombinedSelector):
        lstep, lextras = _extract_sel_info(sel.left)
        rstep, rextras = _extract_sel_info(sel.right)
        steps = lstep + rstep
        extras = lextras + rextras
    elif isinstance(sel, CompoundSelector):
        for ssel in sel.simple_selectors:
            s, e = _extract_sel_info(ssel)
            steps.extend(s)
            extras.extend(e)
    elif isinstance(sel, FunctionalPseudoClassSelector):
        if sel.name == 'pass':
            steps.append(serialize(sel.arguments).strip('"\''))
    elif isinstance(sel, PseudoClassSelector):
        if sel.name == 'deferred':
            extras.append('deferred')
    return (steps, extras)
1278836
function extractInline($) {
  var scripts = [];
  $('script:not([src])').each(function() {
    var el  = $(this);
    var src = el.text();
    var closestPolymerElement = el.closest('polymer-element');
    if(closestPolymerElement.length) {
      src = fixPolymerInvocation(src, $(closestPolymerElement).attr('name'))
    }
    scripts.push(src);
    el.remove();
  })
  return scripts;
}