Reduces a number of keyword sets to a bag-of-words.

    Input:  - list_of_keyword_sets: This is a python list of sets of strings.

    Output: - bag_of_words: This is the corresponding multi-set or bag-of-words, in the form of a python dictionary.
183623
def reduce_list_of_bags_of_words(list_of_keyword_sets):
    
    bag_of_words = dict()
    get_bag_of_words_keys = bag_of_words.keys
    for keyword_set in list_of_keyword_sets:
        for keyword in keyword_set:
            if keyword in get_bag_of_words_keys():
                bag_of_words[keyword] += 1
            else:
                bag_of_words[keyword] = 1

    return bag_of_words
685016
def filter_embeddings(embeddings, vocab, dim):
    
    if not isinstance(embeddings, dict):
        return
    _embeddings = np.zeros([len(vocab), dim])
    for word in vocab:
        if word in embeddings:
            word_idx = vocab[word]
            _embeddings[word_idx] = embeddings[word]

    return _embeddings
17585
def tokenize_words(self):
        
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        tok = self.__word_tokenizer
        text = self.text
        dicts = []
        for sentence in self[SENTENCES]:
            sent_start, sent_end = sentence[START], sentence[END]
            sent_text = text[sent_start:sent_end]
            spans = tok.span_tokenize(sent_text)
            for start, end in spans:
                dicts.append({START: start+sent_start, END: end+sent_start, TEXT: sent_text[start:end]})
        self[WORDS] = dicts
        return self
1197054
def _get_words_from_dataset(dataset):
    
    # Words may be either a string or a list of tokens. Return an iterator
    # of tokens accordingly
    def tokenize(words):
        if isinstance(words, basestring):
            return word_tokenize(words, include_punc=False)
        else:
            return words
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)
879817
def keywords2marc(self, key, values):
    
    result_695 = self.get('695', [])
    result_084 = self.get('084', [])
    result_6531 = self.get('6531', [])

    for value in values:
        schema = value.get('schema')
        source = value.get('source')
        keyword = value.get('value')

        if schema == 'PACS' or schema == 'PDG':
            result_084.append({
                '2': schema,
                '9': source,
                'a': keyword,
            })
        elif schema == 'JACOW':
            result_6531.append({
                '2': 'JACoW',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INSPIRE':
            result_695.append({
                '2': 'INSPIRE',
                '9': source,
                'a': keyword,
            })
        elif schema == 'INIS':
            result_695.append({
                '2': 'INIS',
                '9': source,
                'a': keyword,
            })
        elif source != 'magpie':
            result_6531.append({
                '9': source,
                'a': keyword,
            })

    self['6531'] = result_6531
    self['084'] = result_084
    return result_695
1240538
def create_order_keyword_list(keywords):
     
    result = []
    for keyword in keywords:
        result.append((keyword, ''))
        result.append(('-%s' % keyword, ''))
    return result
687181
def get_multi_word_keywords(features):
        
        keys = {
            'is not': Token(TokenTypes.NOT_EQUAL, 'is not'),
        }
        return OrderedDict(sorted(list(keys.items()), key=lambda t: len(t[0]), reverse=True))
841224
function keywords() {
        var words = [];
        for (var _i = 0; _i < arguments.length; _i++) {
            words[_i - 0] = arguments[_i];
        }
        return choice.apply(void 0, words.map(keyword));
    }
477953
def tuple_to_schema(tuple_):
    
    schema = []
    for element in tuple_:
        if isinstance(element, (tuple, list)):
            try:
                if isinstance(element[1], six.string_types):
                    schema.append((element[0],))
                else:
                    schema.append(tuple_to_schema(element))
            except IndexError:
                schema.append((element[0],))
        else:
            schema.append(element)
    return tuple(schema)
505623
def generate_skip_gram_data_set(self, token_list):
        '''
        
        '''
        n_gram_tuple_zip = self.generate_tuple_zip(token_list, 3)
        skip_gram_list = []
        for pre, point, post in n_gram_tuple_zip:
            skip_gram_list.append((point, pre))
            skip_gram_list.append((point, post))
        return zip(skip_gram_list)
1275724
def separateKeywords(kwArgsDict):
    
    standardKws = {}
    ourKws = {}
    for k in kwArgsDict:
        if k in STANDARD_KEYS:
            standardKws[k]=kwArgsDict[k]
        else:
            ourKws[k]=kwArgsDict[k]
    return (standardKws, ourKws)
1001920
def __get_keywords(self):
        
        txt = self.text
        for line in txt:
            for word in split_words(line):
                yield(word)
1051570
def get_keywords(lexer):
    
    if not hasattr(lexer, 'tokens'):
        return []
    if 'keywords' in lexer.tokens:
        try:
            return lexer.tokens['keywords'][0][0].words
        except:
            pass
    keywords = []
    for vals in lexer.tokens.values():
        for val in vals:
            try:
                if isinstance(val[0], words):
                    keywords.extend(val[0].words)
                else:
                    ini_val = val[0]
                    if ')\\b' in val[0] or ')(\\s+)' in val[0]:
                        val = re.sub(r'\\.', '', val[0])
                        val = re.sub(r'[^0-9a-zA-Z|]+', '', val)
                        if '|' in ini_val:
                            keywords.extend(val.split('|'))
                        else:
                            keywords.append(val)
            except Exception:
                continue
    return keywords
109424
def param_keywords(self):
        
        return dict((key, self[key]) for key in self if key not in self.extra_keywords())
534088
def term_count_buckets(self):

        

        buckets = {}
        for term, count in self.term_counts().items():
            if count in buckets: buckets[count].append(term)
            else: buckets[count] = [term]

        return buckets
517433
def _bag_of_words(x):
  
  def _bow(x):
    """Comptue BOW weights.

    As tf layer's sum combiner is used, the weights can be just ones. Tokens are
    not summed together here.
    """
    return tf.SparseTensor(
      indices=x.indices,
      values=tf.to_float(tf.ones_like(x.values)),
      dense_shape=x.dense_shape)

  return _bow(x)
538992
def find_keyword(word_list: List[str], min_len: int = 3) -> Dict[str, int]:
    
    word_list = rank(word_list, exclude_stopwords=True)

    return {k: v for k, v in word_list.items() if v >= min_len}