Let's fetch a random sentence that we then need to substitute bits of...
    @
    :param counts:
    :param sentence_id:
488430
def __get_sentence(counts, sentence_id=None):
    

    # First of all we need a cursor and a query to retrieve our ID's
    cursor = CONN.cursor()
    check_query = "select sen_id from sursentences"

    # Now we fetch the result of the query and save it into check_result
    cursor.execute(check_query)
    check_result = cursor.fetchall()

    # declare an empty list to be populated below
    id_list = []
    id_to_fetch = None

    # Populate the id_list variable with all of the ID's we retrieved from the database query.
    for row in check_result:
        id_list.append(row[0])

    if sentence_id is not None:
        if type(sentence_id) is int:
            id_to_fetch = sentence_id
    else:
        id_to_fetch = random.randint(1, counts['max_sen'])

        while id_to_fetch not in id_list:
            id_to_fetch = random.randint(1, counts['max_sen'])

    query = ("select * from sursentences where sen_id = {0}".format(id_to_fetch))
    cursor.execute(query)
    result = cursor.fetchone()
    # cursor.close()

    return result
1118599
def p_for_sentence(p):
    
    p[0] = p[1]
    if is_null(p[0]):
        return
    p[1].appendChild(make_block(p[2], p[3]))
    gl.LOOPS.pop()
60571
def p_statement_randomize(p):
    
    p[0] = make_sentence('RANDOMIZE', make_number(0, lineno=p.lineno(1), type_=TYPE.ulong))
989923
def _make_tagdict(self, sentences):
        ''''''
        counts = defaultdict(lambda: defaultdict(int))
        for words, tags in sentences:
            for word, tag in zip(words, tags):
                counts[word][tag] += 1
                self.classes.add(tag)
        freq_thresh = 20
        ambiguity_thresh = 0.97
        for word, tag_freqs in counts.items():
            tag, mode = max(tag_freqs.items(), key=lambda item: item[1])
            n = sum(tag_freqs.values())
            # Don't add rare words to the tag dictionary
            # Only add quite unambiguous words
            if n >= freq_thresh and (float(mode) / n) >= ambiguity_thresh:
                self.tagdict[word] = tag
409168
def listup_sentence(self, data, counter=0):
        '''
        

        '''
        delimiter = self.delimiter_list[counter]
        sentence_list = []
        [sentence_list.append(sentence + delimiter) for sentence in data.split(delimiter) if sentence != ""]
        if counter + 1 < len(self.delimiter_list):
            sentence_list_r = []
            [sentence_list_r.extend(self.listup_sentence(sentence, counter+1)) for sentence in sentence_list]
            sentence_list = sentence_list_r

        return sentence_list
958906
def update(self, new: 'Sentence', flags: Flags):
        

        items = [i for i in self.items if i.flags != flags]
        items.extend(new.items)
        self.items = items
977759
def count_token_occurrences(cls, words):
        
        counts = {}
        for word in words:
            if word in counts:
                counts[word] += 1
            else:
                counts[word] = 1
        return counts
1266524
def get_unique_sentence_indices( text ):
    '''  '''
    # Add sentence annotation (if missing)
    if not text.is_tagged( SENTENCES ):
        text.tokenize_sentences()
    # Collect (unique) sent indices over the whole text
    sent_indices = []
    sent_id = 0
    for sub_text in text.split_by( SENTENCES ):
        for word in sub_text.words:
            sent_indices.append( sent_id )
        sent_id += 1
    assert len(sent_indices) == len(text.words), '(!) Number of sent indices should match nr of words!'
    return sent_indices
1077098
def add(self, sentence_text, **kwargs):
        '''  '''
        sent = MeCabSent.parse(sentence_text, **kwargs)
        self.sents.append(sent)
        return sent
833370
function(sentences, numberOfTopics, numberOfTermsPerTopic, languages, alphaValue, betaValue, randomSeed) {
    // The result will consist of topics and their included terms [[{"term":"word1", "probability":0.065}, {"term":"word2", "probability":0.047}, ... ], [{"term":"word1", "probability":0.085}, {"term":"word2", "probability":0.024}, ... ]].
    var result = [];
    // Index-encoded array of sentences, with each row containing the indices of the words in the vocabulary.
    var documents = new Array();
    // Hash of vocabulary words and the count of how many times each word has been seen.
    var f = {};
    // Vocabulary of unique words (porter stemmed).
    var vocab=new Array();
    // Vocabulary of unique words in their original form.
    var vocabOrig = {};
    // Array of stop words
    languages = languages || Array('en');

    if (sentences && sentences.length > 0) {
      var stopwords = new Array();

      languages.forEach(function(value) {
          var stopwordsLang = require('./stopwords_' + value + ".js");
          stopwords = stopwords.concat(stopwordsLang.stop_words);
      });

      for(var i=0;i<sentences.length;i++) {
          if (sentences[i]=="") continue;
          documents[i] = new Array();

          var words = sentences[i] ? sentences[i].split(/[\s,\"]+/) : null;

          if(!words) continue;
          for(var wc=0;wc<words.length;wc++) {
              var w=words[wc].toLowerCase().replace(/[^a-z\'A-Z0-9\u00C0-\u00ff ]+/g, '');
              var wStemmed = stem(w);
              if (w=="" || !wStemmed || w.length==1 || stopwords.indexOf(w.replace("'", "")) > -1 || stopwords.indexOf(wStemmed) > -1 || w.indexOf("http")==0) continue;
              if (f[wStemmed]) { 
                  f[wStemmed]=f[wStemmed]+1;
              } 
              else if(wStemmed) { 
                  f[wStemmed]=1; 
                  vocab.push(wStemmed);
                  vocabOrig[wStemmed] = w;
              };
              
              documents[i].push(vocab.indexOf(wStemmed));
          }
      }

      var V = vocab.length;
      var M = documents.length;
      var K = parseInt(numberOfTopics);
      var alpha = alphaValue || 0.1;  // per-document distributions over topics
      var beta = betaValue || .01;  // per-topic distributions over words
      documents = documents.filter((doc) => { return doc.length }); // filter empty documents

      lda.configure(documents,V,10000, 2000, 100, 10, randomSeed);
      lda.gibbs(K, alpha, beta);

      var theta = lda.getTheta();
      var phi = lda.getPhi();

      var text = '';

      //topics
      var topTerms=numberOfTermsPerTopic;
      for (var k = 0; k < phi.length; k++) {
          var things = new Array();
          for (var w = 0; w < phi[k].length; w++) {
               things.push(""+phi[k][w].toPrecision(2)+"_"+vocab[w] + "_" + vocabOrig[vocab[w]]);
          }
          things.sort().reverse();
          //console.log(things);
          if(topTerms>vocab.length) topTerms=vocab.length;

          //console.log('Topic ' + (k + 1));
          var row = [];
          
          for (var t = 0; t < topTerms; t++) {
              var topicTerm=things[t].split("_")[2];
              var prob=parseInt(things[t].split("_")[0]*100);
              if (prob<2) continue;
              
              //console.log('Top Term: ' + topicTerm + ' (' + prob + '%)');
              
              var term = {};
              term.term = topicTerm;
              term.probability = parseFloat(things[t].split("_")[0]);
              row.push(term);
          }

          result.push(row);
      }
    }
    
    return result;
}
998527
def paragraphs(total, opts)
      list = []
      total.times do
        sentence_count = rand(sentence_count_range_in_a_paragraph)
        start_sep = opts[:start_sep].to_s
        end_sep = opts[:end_sep].to_s
        list << start_sep + self.sentences(sentence_count) + end_sep
      end
      return list.join()
    end
846248
def get_word_counts(word)
      # fetch all (class,count) pairs for a given word
      row = @cassandra.get(:classes, word.to_s)
      return row.to_hash if row.empty?
      row.inject({}){|counts, col| counts[col.first.to_sym] = [col.last.to_f,0].max; counts}
    end
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
214226
def sentence_bytes(self, sentence):
        
        result = [TOKENS[sentence[0]]]

        for i in sentence[1:]:  # Remaining bytes
            if isinstance(i, str):
                result.extend(self.literal(i))
            elif isinstance(i, float) or isinstance(i, int):  # A number?
                result.extend(self.number(i))
            else:
                result.extend(i)  # Must be another thing

        return result
1166880
def chunked(sentence):
    
    # For example, to construct a training vector with the head of previous chunks as a feature.
    # Doing this with Sentence.chunks would discard the punctuation marks and conjunctions
    # (Sentence.chunks only yields Chunk objects), which amy be useful features.
    chunks = []
    for word in sentence:
        if word.chunk is not None:
            if len(chunks) == 0 or chunks[-1] != word.chunk:
                chunks.append(word.chunk)
        else:
            ch = Chink(sentence)
            ch.append(word.copy(ch))
            chunks.append(ch)
    return chunks
1018114
def sentences(self, index = None):
        
        if index is None:
            return self.select(Sentence,None,True,default_ignore_structure)
        else:
            if index < 0:
                index = self.count(Sentence,None,True,default_ignore_structure) + index
            for i,e in enumerate(self.select(Sentence,None,True,default_ignore_structure)):
                if i == index:
                    return e
            raise IndexError
934450
def __replace_repeat(sentence):
    

    ######### USE SENTENCE_ID 47 for testing!

    repeat_dict = {}

    if sentence is not None:

        while sentence.find('#DEFINE_REPEAT') != -1:
            begin_index = sentence.find('#DEFINE_REPEAT')
            start_index = begin_index + 15
            end_index = sentence.find(']')

            if sentence.find('#DEFINE_REPEAT') is not None:
                sub_list = sentence[start_index:end_index].split(',')
                choice = sub_list[0]
                repeat_text = sub_list[1]
                repeat_dict[choice] = repeat_text
                sentence = sentence.replace(sentence[begin_index:end_index + 1], '', 1)

        while sentence.find('#REPEAT') != -1:
            if sentence.find('#REPEAT') is not None:
                repeat_begin_index = sentence.find('#REPEAT')
                repeat_start_index = repeat_begin_index + 8
                # by searching from repeat_index below we don't encounter dodgy bracket-matching errors.
                repeat_end_index = sentence.find(']', repeat_start_index)
                repeat_index = sentence[repeat_start_index:repeat_end_index]

                if repeat_index in repeat_dict:
                    sentence = sentence.replace(sentence[repeat_begin_index:repeat_end_index + 1],
                                                  str(repeat_dict[repeat_index]))

        if sentence.find('#REPEAT') == -1:
            return sentence
        return sentence
    else:
        return sentence