Vocabulary of the sentences.
436187
public Set<String> getVocab() {
        Set<String> vocab = new HashSet<String>();
        for (Sentence sent : this) {
            for (String label : sent) {
                vocab.add(label);
            }
        }
        return vocab;
    }
1083426
def _build_vocab(self, sentences, markers=[]):
        
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict
833370
function(sentences, numberOfTopics, numberOfTermsPerTopic, languages, alphaValue, betaValue, randomSeed) {
    // The result will consist of topics and their included terms [[{"term":"word1", "probability":0.065}, {"term":"word2", "probability":0.047}, ... ], [{"term":"word1", "probability":0.085}, {"term":"word2", "probability":0.024}, ... ]].
    var result = [];
    // Index-encoded array of sentences, with each row containing the indices of the words in the vocabulary.
    var documents = new Array();
    // Hash of vocabulary words and the count of how many times each word has been seen.
    var f = {};
    // Vocabulary of unique words (porter stemmed).
    var vocab=new Array();
    // Vocabulary of unique words in their original form.
    var vocabOrig = {};
    // Array of stop words
    languages = languages || Array('en');

    if (sentences && sentences.length > 0) {
      var stopwords = new Array();

      languages.forEach(function(value) {
          var stopwordsLang = require('./stopwords_' + value + ".js");
          stopwords = stopwords.concat(stopwordsLang.stop_words);
      });

      for(var i=0;i<sentences.length;i++) {
          if (sentences[i]=="") continue;
          documents[i] = new Array();

          var words = sentences[i] ? sentences[i].split(/[\s,\"]+/) : null;

          if(!words) continue;
          for(var wc=0;wc<words.length;wc++) {
              var w=words[wc].toLowerCase().replace(/[^a-z\'A-Z0-9\u00C0-\u00ff ]+/g, '');
              var wStemmed = stem(w);
              if (w=="" || !wStemmed || w.length==1 || stopwords.indexOf(w.replace("'", "")) > -1 || stopwords.indexOf(wStemmed) > -1 || w.indexOf("http")==0) continue;
              if (f[wStemmed]) { 
                  f[wStemmed]=f[wStemmed]+1;
              } 
              else if(wStemmed) { 
                  f[wStemmed]=1; 
                  vocab.push(wStemmed);
                  vocabOrig[wStemmed] = w;
              };
              
              documents[i].push(vocab.indexOf(wStemmed));
          }
      }

      var V = vocab.length;
      var M = documents.length;
      var K = parseInt(numberOfTopics);
      var alpha = alphaValue || 0.1;  // per-document distributions over topics
      var beta = betaValue || .01;  // per-topic distributions over words
      documents = documents.filter((doc) => { return doc.length }); // filter empty documents

      lda.configure(documents,V,10000, 2000, 100, 10, randomSeed);
      lda.gibbs(K, alpha, beta);

      var theta = lda.getTheta();
      var phi = lda.getPhi();

      var text = '';

      //topics
      var topTerms=numberOfTermsPerTopic;
      for (var k = 0; k < phi.length; k++) {
          var things = new Array();
          for (var w = 0; w < phi[k].length; w++) {
               things.push(""+phi[k][w].toPrecision(2)+"_"+vocab[w] + "_" + vocabOrig[vocab[w]]);
          }
          things.sort().reverse();
          //console.log(things);
          if(topTerms>vocab.length) topTerms=vocab.length;

          //console.log('Topic ' + (k + 1));
          var row = [];
          
          for (var t = 0; t < topTerms; t++) {
              var topicTerm=things[t].split("_")[2];
              var prob=parseInt(things[t].split("_")[0]*100);
              if (prob<2) continue;
              
              //console.log('Top Term: ' + topicTerm + ' (' + prob + '%)');
              
              var term = {};
              term.term = topicTerm;
              term.probability = parseFloat(things[t].split("_")[0]);
              row.push(term);
          }

          result.push(row);
      }
    }
    
    return result;
}
696581
def sentence(options={})
    word = options[:first_word] || self.first_word
    num_words_option = options[:words] || (3..15)
    count = Util.rand_count(num_words_option,randomizer)
    punctuation = options[:punctuation] || self.punctuation

    words = count.times.collect do
      word.tap {word = model.next_word(word,randomizer)}
    end.compact

    words = extend_trailing_preposition(Util.max(num_words_option), words)

    Util.capitalize words.compact.join(" ") + punctuation
  end
865662
def _readSentence(self):
        
        reply_word, words = self.protocol.readSentence()
        words = dict(parseWord(word) for word in words)
        return reply_word, words
960948
def related(self):
        
        return [ch for ch in self.sentence.chunks 
                    if ch != self and intersects(unzip(0, ch.relations), unzip(0, self.relations))]
963268
def p_system_sentence(self, t):
        """"""

        if len(t) == 3:
            t[0] = system(t[2], reference=True, line=t.lineno(1))
        else:
            t[0] = system(t[2], t[4], line=t.lineno(1))
22100
def tag_verb_chains(self):
        
        if not self.is_tagged(CLAUSES):
            self.tag_clauses()
        if self.__verbchain_detector is None:
            self.__verbchain_detector = load_default_verbchain_detector()
        sentences = self.divide()
        verbchains = []
        for sentence in sentences:
            chains = self.__verbchain_detector.detectVerbChainsFromSent( sentence )
            for chain in chains:
                # 1) Get spans for all words of the phrase
                word_spans = [ ( sentence[idx][START], sentence[idx][END] ) \
                                 for idx in sorted( chain[PHRASE] ) ]
                # 2) Assign to the chain
                chain[START] = [ span[0] for span in word_spans ]
                chain[END]   = [ span[1] for span in word_spans ]
            verbchains.extend(chains)
        self[VERB_CHAINS] = verbchains
        return self
250181
public List<VocabularyWord> words() {
        List<VocabularyWord> vocab = new ArrayList<>(vocabulary.values());
        Collections.sort(vocab, new Comparator<VocabularyWord>() {
            @Override
            public int compare(VocabularyWord o1, VocabularyWord o2) {
                return Integer.compare(o2.getCount(), o1.getCount());
            }
        });

        return vocab;
    }
959811
def _create_sentence_objects(self):
    '''
    '''
    sentence_objects = []
    sent_tokenizer = SentenceTokenizer(locale=self.language.code)
    seq = Sequence(self.raw)
    seq = sent_tokenizer.transform(seq)
    for start_index, end_index in zip(seq.idx[:-1], seq.idx[1:]):
      # Sentences share the same models as their parent blob
      sent = seq.text[start_index: end_index].strip()
      if not sent: continue
      s = Sentence(sent, start_index=start_index, end_index=end_index)
      s.detected_languages = self.detected_languages
      sentence_objects.append(s)
    return sentence_objects
594583
def _sentence_to_features(self, sentence: list):
        # type: (List[Tuple[Text, Text, Text, Text]]) -> List[Dict[Text, Any]]
        """"""

        sentence_features = []
        prefixes = ('-1', '0', '+1')

        for word_idx in range(len(sentence)):
            # word before(-1), current word(0), next word(+1)
            word_features = {}
            for i in range(3):
                if word_idx == len(sentence) - 1 and i == 2:
                    word_features['EOS'] = True
                    # End Of Sentence
                elif word_idx == 0 and i == 0:
                    word_features['BOS'] = True
                    # Beginning Of Sentence
                else:
                    word = sentence[word_idx - 1 + i]
                    prefix = prefixes[i]
                    features = self.crf_features[i]
                    for feature in features:
                        # append each feature to a feature vector
                        value = self.function_dict[feature](word)
                        word_features[prefix + ":" + feature] = value
            sentence_features.append(word_features)
        return sentence_features
1071961
def disambiguate(self, words):
        
        words = vm.SentenceAnalysis([as_wordanalysis(w) for w in words])
        disambiguated = self._morf.disambiguate(words)
        return [postprocess_result(mr, False, True) for mr in disambiguated]
333356
def load_words(self, words):
        
        self._dictionary.update([word.lower() for word in words])
        self._update_dictionary()
413457
def train(self, sentences, nr_iter=5):
        
        self._make_tagdict(sentences)
        self.perceptron.classes = self.classes
        for iter_ in range(nr_iter):
            c = 0
            n = 0
            for sentence in sentences:
                prev, prev2 = self.START
                context = [t[0] for t in sentence]
                for i, (token, tag) in enumerate(sentence):
                    guess = self.tagdict.get(token)
                    if not guess:
                        feats = self._get_features(i, context, prev, prev2)
                        guess = self.perceptron.predict(feats)
                        self.perceptron.update(tag, guess, feats)
                    prev2 = prev
                    prev = guess
                    c += guess == tag
                    n += 1
            random.shuffle(sentences)
            log.debug('Iter %s: %s/%s=%s' % (iter_, c, n, (float(c) / n) * 100))
        self.perceptron.average_weights()
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
1287688
public List<Token> findNouns(Sentence sentence) {
    List<Token> nouns = new ArrayList<Token>();

    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("PIV") || tag.equals("ACC") || tag.equals("SC")) {

        for (Token token : syntChunks.get(i).getTokens()) {

          if (token.getPOSTag().equals("n") || token.getPOSTag().equals("pron-pers") || token.getPOSTag().equals("prop")) {
              nouns.add(token);
            }
          }
        }
      }

    return nouns;
  }
576780
def sentences(ctx, input, output):
    """"""
    log.info('chemdataextractor.read.elements')
    log.info('Reading %s' % input.name)
    doc = Document.from_file(input)
    for element in doc.elements:
        if isinstance(element, Text):
            for raw_sentence in element.raw_sentences:
                output.write(raw_sentence.strip())
                output.write(u'\n')