Create WordSet from raw dictionary data.
333749
def from_dict(raw_data):
        """"""
        try:
            set_id = raw_data['id']
            title = raw_data['title']
            terms = [Term.from_dict(term) for term in raw_data['terms']]
            return WordSet(set_id, title, terms)
        except KeyError:
            raise ValueError('Unexpected set json structure')
682028
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'collections' in _dict:
            args['collections'] = [
                Collection._from_dict(x) for x in (_dict.get('collections'))
            ]
        return cls(**args)
1214778
def from_dict(self, description):
        """"""
        assert(self.ident == description['ident'])
        self.partitions = description['partitions']
        self.indices    = description['indices']
971641
def get_en_words() -> Set[str]:
    

    pull_en_words()

    with open(config.EN_WORDS_PATH) as words_f:
        raw_words = words_f.readlines()
    en_words = set([word.strip().lower() for word in raw_words])
    NA_WORDS_IN_EN_DICT = set(["kore", "nani", "karri", "imi", "o", "yaw", "i",
                           "bi", "aye", "imi", "ane", "kubba", "kab", "a-",
                           "ad", "a", "mak", "selim", "ngai", "en", "yo",
                           "wud", "mani", "yak", "manu", "ka-", "mong",
                           "manga", "ka-", "mane", "kala", "name", "kayo",
                           "kare", "laik", "bale", "ni", "rey", "bu",
                           "re", "iman", "bom", "wam",
                           "alu", "nan", "kure", "kuri", "wam", "ka", "ng",
                           "yi", "na", "m", "arri", "e", "kele", "arri", "nga",
                           "kakan", "ai", "ning", "mala", "ti", "wolk",
                           "bo", "andi", "ken", "ba", "aa", "kun", "bini",
                           "wo", "bim", "man", "bord", "al", "mah", "won",
                           "ku", "ay", "belen", "wen", "yah", "muni",
                           "bah", "di", "mm", "anu", "nane", "ma", "kum",
                           "birri", "ray", "h", "kane", "mumu", "bi", "ah",
                           "i-", "n", "mi", "bedman", "rud", "le", "babu",
                           "da", "kakkak", "yun", "ande", "naw", "kam", "bolk",
                           "woy", "u", "bi-",
                           ])
    EN_WORDS_NOT_IN_EN_DICT = set(["screenprinting"])
    en_words = en_words.difference(NA_WORDS_IN_EN_DICT)
    en_words = en_words | EN_WORDS_NOT_IN_EN_DICT
    return en_words
1153611
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'tokens' in _dict:
            args['tokens'] = [
                TokenResult._from_dict(x) for x in (_dict.get('tokens'))
            ]
        if 'sentences' in _dict:
            args['sentences'] = [
                SentenceResult._from_dict(x) for x in (_dict.get('sentences'))
            ]
        return cls(**args)
1062129
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'text' in _dict:
            args['text'] = _dict.get('text')
        if 'entities' in _dict:
            args['entities'] = [
                SemanticRolesEntity._from_dict(x)
                for x in (_dict.get('entities'))
            ]
        if 'keywords' in _dict:
            args['keywords'] = [
                SemanticRolesKeyword._from_dict(x)
                for x in (_dict.get('keywords'))
            ]
        return cls(**args)
306280
def word(ctx, text, number, by_spaces=False):
    
    return word_slice(ctx, text, number, conversions.to_integer(number, ctx) + 1, by_spaces)
173162
def words(ctx, input, output):
    """"""
    log.info('chemdataextractor.read.elements')
    log.info('Reading %s' % input.name)
    doc = Document.from_file(input)
    for element in doc.elements:
        if isinstance(element, Text):
            for sentence in element.sentences:
                output.write(u' '.join(sentence.raw_tokens))
                output.write(u'\n')
429778
def load_or_create_vocab(data: str, vocab_path: Optional[str], num_words: int, word_min_count: int,
                         pad_to_multiple_of: Optional[int] = None) -> Vocab:
    
    if vocab_path is None:
        return build_from_paths(paths=[data], num_words=num_words, min_count=word_min_count,
                                pad_to_multiple_of=pad_to_multiple_of)
    else:
        return vocab_from_json(vocab_path)
527460
def from_dict(cls, d):
        """"""
        return cls(**{k: v for k, v in d.items() if k in cls.ENTRIES})
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval
1147046
def _create_dataset(self, *data):
        """"""
        # Make sure data is a tuple of dense tensors
        data = [self._to_torch(x, dtype=torch.FloatTensor) for x in data]
        return TensorDataset(*data)
914929
def from_vocabfile(cls, filename):
    
    words = [x.strip() for x in _open(filename, 'r').read().splitlines()]
    return cls(words=words)
989082
def _make_data(data) -> Tuple[List[Dict], List[Dict]]:
        """"""
        jsdata = []
        for idx, row in data.iterrows():
            row.index = row.index.astype(str)
            rdict = row.to_dict()
            rdict.update(dict(key=str(idx)))
            jsdata.append(rdict)

        return jsdata, Table._make_columns(data.columns)
188176
def synsets(self):
        """"""
        synsets = []
        for wn_annots in self.wordnet_annotations:
            word_synsets = []
            for wn_annot in wn_annots:
                for synset in wn_annot.get(SYNSETS, []):
                    word_synsets.append(deepcopy(synset))
            synsets.append(word_synsets)
        return synsets
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
345554
def bysize(words):
    
    ret = {}
    for w in words:
        if len(w) not in ret:
            ret[len(w)] = set()
        ret[len(w)].add(w)
    return ret