The list of annotated synsets of ``words`` layer.
188176
def synsets(self):
        """"""
        synsets = []
        for wn_annots in self.wordnet_annotations:
            word_synsets = []
            for wn_annot in wn_annots:
                for synset in wn_annot.get(SYNSETS, []):
                    word_synsets.append(deepcopy(synset))
            synsets.append(word_synsets)
        return synsets
1048611
def words(fpth):
    
    from collections import Counter
    words__ = Counter()
    with open(fpth) as fp:
        for line in fp:
            words__.update(line.strip().split(' '))
    return words__
701970
def wordnet_annotations(self):
        """"""
        if not self.is_tagged(WORDNET):
            self.tag_wordnet()
        return [[a[WORDNET] for a in analysis] for analysis in self.analysis]
241246
def set_embeddings(self, embeddings):
        '''
        
        '''
        if self.embeddings_ is not None:
            raise Exception("You have already set embeddings by running set_embeddings or set_embeddings_model.")
        assert embeddings.shape[1] == self.corpus_.get_num_terms()
        self.embeddings_ = embeddings.T
        self.vocab_ = self.corpus_.get_terms()
        return self
1367810
def allFeatures(self):
        
        for dataset in self.getDatasets():
            for featureSet in dataset.getFeatureSets():
                for feature in featureSet.getFeatures():
                    yield feature
482352
def p_word_list(p):
    ''''''
    parserobj = p.context
    if len(p) == 2:
        p[0] = [_expandword(parserobj, p.slice[1])]
    else:
        p[0] = p[1]
        p[0].append(_expandword(parserobj, p.slice[2]))
637197
def full_keywords(soup):
    ""
    if not raw_parser.author_keywords(soup):
        return []
    return list(map(node_contents_str, raw_parser.author_keywords(soup)))
662251
public List<String> phraseDel(String[] words){
		List<String> list = new ArrayList<String>(); 
		String s;
		int length= words.length;
		for(int i = 0; i < length; i++){
			s = words[i];
			if(!isStopWord(s))
				list.add(s);
		}
		return list;
	}
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
1197054
def _get_words_from_dataset(dataset):
    
    # Words may be either a string or a list of tokens. Return an iterator
    # of tokens accordingly
    def tokenize(words):
        if isinstance(words, basestring):
            return word_tokenize(words, include_punc=False)
        else:
            return words
    all_words = chain.from_iterable(tokenize(words) for words, _ in dataset)
    return set(all_words)
385208
def words(self, fileids=None) -> Generator[str, str, None]:
        
        for sentence in self.sents(fileids):
            words = self._word_tokenizer.tokenize(sentence)
            for word in words:
                yield word
393026
def annotations(self, qname=True):
        
        if qname:
            return sorted([(uri2niceString(x, self.namespaces)
                            ), (uri2niceString(y, self.namespaces)), z]
                          for x, y, z in self.triples)
        else:
            return sorted(self.triples)
796432
def parse_wordnet(self,debug=False):
        '''
        '''
        synList = []
        self.milestone = 0 # to start from beginning of file
        while self.milestone < os.path.getsize(self.fileName) - 5:
            if debug:
                print ('self.milestone', self.milestone)
            a = self.parse_synset(offset=self.milestone)
            synList.append(a)
            self.milestone = self.file.tell()
        return synList
397658
def parse_synset(self, offset=None, debug=False):
        
        if False:
            pass
        else:
            # WORD_INSTANCE
            def _word_instance():
                _synset(True)

            # WORD_MEANING
            def _synset(pn=False):
                if not pn:
                    self.synset = Synset()
                    self.pn = False
                else:
                    self.synset = WordInstance()
                    self.pn = True
                if self.DRN:
                    self.synset.number = self.DRN
                self.targetType = None

            def _variants():
                self.synset.variants = Variants()

            def _literal():
                a = Variant()
                self.synset.variants.append(a)
                self.synset.variants[-1].literal = self.fieldValue

            def _target_literal():
                    self.target_synset.variants.append(Variant())
                    self.target_synset.variants[-1].literal = self.fieldValue

            def _sense():
                self.synset.variants[-1].sense = self.fieldValue

            def _status():
                self.noQuotes = True
                try:
                    self.synset.variants[-1].status = as_unicode(self.fieldValue)
                except:
                    self.synset.variants[-1].status = as_unicode(str(self.fieldValue))
                self.noQuotes = False

            def _target_sense():
                self.target_synset.variants[-1].sense = self.fieldValue
                if self.targetType == 'internal':
                    self.synset.internalLinks[
                        -1].target_concept = self.target_synset
                elif self.targetType == 'ili':
                    self.synset.eqLinks[-1].target_concept = self.target_synset
                elif self.targetType == 'pv':
                    self.synset.propertyValues[-1].value = self.target_synset
                else:
                    print ('BOOOOOOOOO!!') # Error TODO

            def _gloss():
                self.synset.variants[-1].gloss = self.fieldValue
                self.synset.definition = self.fieldValue                            # ADDED BY KOM

            def _translations():
                self.synset.variants[-1].translations = Translations()

            def _translation():
                self.synset.variants[-1].translations.append(
                    Translation(
                        language=self.fieldValue.split(':')[0],
                        translation_value = self.fieldValue.split(':')[1])
                    )

            def _examples():
                self.synset.variants[-1].examples = Examples()
            def _usage_labels():
                self.synset.variants[-1].usage_labels = Usage_Labels()

            def _external_info():
                self.synset.variants[-1].externalInfo = External_Info()

            def _example():
                self.synset.variants[-1].examples.append(
                    Example(self.fieldValue)
                    )

            def _usage_label():
                self.synset.variants[
                    -1].usage_labels.append(
                    Usage_Label(name=self.fieldValue)
                    )

            def _usage_label_value():
                self.synset.variants[
                    -1].usage_labels[-1].usage_label_value = self.fieldValue

            def _source_id():
                if self.targetType == 'internal':
                    self.synset.internalLinks[-1].source_id = self.fieldValue
                    # self.synset.internalLinks[-1].source_ids.append(
                    #     Relation_Source_Id(number=self.fieldValue))
                elif self.targetType == 'ili':
                    self.synset.eqLinks[-1].source_id = self.fieldValue
                    # self.synset.eqLinks[-1].source_ids.append(
                    #     Relation_Source_Id(number=self.fieldValue))

                else:
                    if self.synset.variants[-1].external_info:
                        self.synset.variants[
                            -1].external_info.source_ids.append(
                            Source_Id(number=self.fieldValue)
                            )
                    else:
                        self.synset.variants[-1].external_info = External_Info()
                        self.synset.variants[
                            -1].external_info.source_ids.append(
                            Source_Id(number=self.fieldValue)
                            )

            def _corpus_id():
                if self.targetType == 'internal': # not needed
                    self.synset.internalLinks[-1].corpus_ids.append(
                        Relation_Corpus_Id(number=self.fieldValue))
                else:
                    if self.synset.variants[-1].external_info:
                        self.synset.variants[
                            -1].external_info.corpus_ids.append(
                            Corpus_Id(number=self.fieldValue)
                            )
                    else:
                        self.synset.variants[-1].external_info = External_Info()
                        self.synset.variants[
                            -1].external_info.corpus_ids.append(
                            Corpus_Id(number=self.fieldValue)
                            )

            def _frequency():
                self.synset.variants[
                    -1].external_info.corpus_ids[-1].frequency = self.fieldValue

            def _text_key():
                self.synset.variants[
                    -1].external_info.source_ids[-1].text_key = self.fieldValue

            def _number_key():
                self.synset.variants[
                    -1].external_info.source_ids[
                    -1].number_key = self.fieldValue
            def _pos():
                self.synset.pos = self.fieldValue

            # INTERNAL_LINKS
            def _target_concept():
                self.target_synset = Synset()
                self.target_synset.variants = Variants()
                if self.levelNumber == 3: # and self.fieldValue:
                    self.target_synset.number = int(self.fieldValue or 0)

            def _target_pos():
                self.target_synset.pos = self.fieldValue

            def _internal_links():
                self.synset.internalLinks = InternalLinks()
                self.targetType = 'internal'

            def _relation():
                if self.targetType == 'internal':
                    self.synset.internalLinks.append(Relation())
                    self.synset.internalLinks[-1].name = self.fieldValue
                elif self.targetType == 'ili':
                    self.synset.eqLinks.append(EqLink())
                    self.synset.eqLinks[-1].name = self.fieldValue
                else:
                    print ('BOOOOOOOOO!!') # Error TODO

            def _features():
                if self.targetType == 'internal':
                    self.synset.internalLinks[-1].features = Features()
                else:
                    self.synset.variants[-1].features = Features()
                    self.synset.variants[-1].features.append(Feature())

            def _feature():
                self.synset.variants[-1].features[-1].name = self.fieldValue

            def _feature_value():
                self.synset.variants[
                    -1].features[-1].featureValue = self.fieldValue

            def _reversed():
                self.synset.internalLinks[-1].features.append(Feature())
                self.synset.internalLinks[-1].features[-1].name = self.fieldTag
                self.synset.internalLinks[-1].features[-1].featureValue = True

            def _variant_to_variant():
                self.synset.internalLinks[-1].features.append(Feature())
                self.synset.internalLinks[-1].features[-1].name = self.fieldTag

            def _source_variant():
                self.variant_to_variant_source = self.fieldValue

            def _target_variant():
                self.variant_to_variant_target = self.fieldValue
                self.synset.internalLinks[
                    -1].features[-1].featureValue = (
                    self.variant_to_variant_source,
                    self.variant_to_variant_target)

            # EQ_LINKS
            def _eq_links():
                self.synset.eqLinks = EqLinks()
                self.targetType = 'ili'

            def _wn_offset():
                self.target_synset.wordnet_offset = self.fieldValue
                self.synset.eqLinks[-1].target_concept = self.target_synset

            def _add_on_id():
                self.target_synset.add_on_id = self.fieldValue
                self.synset.eqLinks[-1].target_concept = self.target_synset

            # PROPERTIES
            def _properties():
                self.synset.properties = Properties()

            def _name():
                if self.pn:
                    self.synset.propertyValues.append(
                        PropertyValue(name=self.fieldValue))
                else:
                    self.synset.properties.append(Property(self.fieldValue))

            # PROPERTY_VALUES
            def _property_values():
                self.synset.propertyValues = PropertyValues()

            def _property_value():
                self.synset.propertyValues[-1].value = self.fieldValue
                self.targetType = 'pv'

            def _property_wm():
                pass

            rulez = {
                (0,'WORD_MEANING'): _synset,
                (0,'WORD_INSTANCE'): _word_instance,
                (1,'PART_OF_SPEECH'): _pos,
                (1,'VARIANTS'): _variants,
                (2,'LITERAL'): _literal,
                (3,'SENSE'): _sense,
                (3,'STATUS'): _status,
                (3,'DEFINITION'): _gloss,
                (3,'EXAMPLES'): _examples,
                (3,'USAGE_LABELS'): _usage_labels,
                (4,'USAGE_LABEL'): _usage_label,
                (5,'USAGE_LABEL_VALUE'): _usage_label_value,
                (4,'EXAMPLE'): _example,
                (3,'TRANSLATIONS'): _translations,
                (4,'TRANSLATION'): _translation,
                (3,'EXTERNAL_INFO'): _external_info,
                (4,'SOURCE_ID'): _source_id,
                (4,'CORPUS_ID'): _corpus_id,
                (5,'FREQUENCY'): _frequency,
                (5,'TEXT_KEY'): _text_key,
                (5,'NUMBER_KEY'): _number_key,
                (1,'INTERNAL_LINKS'): _internal_links,
                (2,'RELATION'): _relation,
                (3,'TARGET_CONCEPT'): _target_concept,
                (4,'PART_OF_SPEECH'): _target_pos,
                (4,'LITERAL'): _target_literal,
                (5,'SENSE'): _target_sense,
                (3,'FEATURES'): _features,
                (4,'FEATURE'): _feature,
                (5,'FEATURE_VALUE'): _feature_value,
                (4,'REVERSED'): _reversed,
                (4,'VARIANT_TO_VARIANT'): _variant_to_variant,
                (5,'SOURCE_VARIANT'): _source_variant,
                (5,'TARGET_VARIANT'): _target_variant,
                (3,'SOURCE_ID'): _source_id,
                (1,'EQ_LINKS'): _eq_links,
                (2,'EQ_RELATION'): _relation,
                (3,'TARGET_ILI'): _target_concept,
                (4,'WORDNET_OFFSET'): _wn_offset,
                (4,'ADD_ON_ID'): _add_on_id,
                (1,'PROPERTIES'): _properties,
                (1,'PROPERTY_VALUES'): _property_values,
                (2,'NAME'): _name,
                (3,'VALUE'): _property_value,
                (3,'VALUE_AS_TEXT'): _property_value,
                (3,'VALUE_AS_WORD_MEANING'): _target_concept,
                }

            if not offset:
                offset = self.milestone
            else:
                self.milestone=offset
            if self.file:
                self.file.seek(offset,0)

            line = 'X'
            ili = False
            var = False
            while line.strip():

                offset = self.file.tell()
                self.file.seek(offset,0)

                line = as_unicode(self.file.readline(), self.encoding).strip()
                if debug:
                    print (line.encode('utf-8'))


                self.parse_line(line)
                self.noQuotes = None

                select = (self.levelNumber,self.fieldTag)

                if select in rulez.keys():
                    rulez[select]()
                else:
                    if line:
                        print (self.synset.polarisText)
                        raise ParseError("No parsing rule for '%s'" % line)
        return self.synset
51647
def synsets(self, lemma, pos = None):
        '''
        
        '''
        return sorted(set(lemma_obj.synset
                          for lemma_obj in self.lemmas(lemma, pos)))
1002506
def defined_annotation_keywords(self) -> Set[str]:
        """"""
        return (
            set(self.annotation_pattern) |
            set(self.annotation_url) |
            set(self.annotation_list)
        )
691556
def tag(self, corpus, tokenize=True):
        ''''''
        # Assume untokenized corpus has \n between sentences and ' ' between words
        s_split = SentenceTokenizer().tokenize if tokenize else lambda t: t.split('\n')
        w_split = WordTokenizer().tokenize if tokenize else lambda s: s.split()
        def split_sents(corpus):
            for s in s_split(corpus):
                yield w_split(s)

        prev, prev2 = self.START
        tokens = []
        for words in split_sents(corpus):
            context = self.START + [self._normalize(w) for w in words] + self.END
            for i, word in enumerate(words):
                tag = self.tagdict.get(word)
                if not tag:
                    features = self._get_features(i, word, context, prev, prev2)
                    tag = self.model.predict(features)
                tokens.append((word, tag))
                prev2 = prev
                prev = tag
        return tokens