Reduce a list of [sentence [word [syllable]]] to [sentence [syllable]].
        :param syllables_words: Elided text
        :return: Text tokenized only at the sentence and syllable level
        :rtype : list
557307
def _syllable_condenser(self, words_syllables):
        

        sentences_syllables = []
        for sentence in words_syllables:
            syllables_sentence = []
            for word in sentence:
                syllables_sentence += word
            sentences_syllables.append(syllables_sentence)
        return sentences_syllables
16843
List<List<IWord>> sentenceTokenize(List<Sentence> sentence) throws IOException
    {
        List<List<IWord>> senWords = new ArrayList<List<IWord>>();
        for ( Sentence sen : sentence ) {
            List<IWord> words = new ArrayList<IWord>();
            wordSeg.reset(new StringReader(sen.getValue()));
            IWord word = null;
            while ( (word = wordSeg.next()) != null ) {
                words.add(word);
            }
            
            senWords.add(words);
        }
        
        return senWords;
    }
1204851
def words(self):
        
        return WordList(
            word_tokenize(self.raw, self.tokenizer, include_punc=False))
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
1245027
def pad_sentences(sentences, padding_word="</s>"):
    
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    for i, sentence in enumerate(sentences):
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences
688546
function (s) {
            var c = clean( s );
            var all = _.words( c, wordDelimiter );
            var words = _(all).chain()
                    // remove stop words
                    .filter( function (w) {
                        return (stopWords.indexOf( w ) === -1) ;
                    })
                    // apply stemmer
                    .map( function (w) {
                        return stemmer( w );
                    })
                    // collect word frequencies
                    .reduce( function (collect, w) {
                        collect[w] = collect[w] ? collect[w] + 1 : 1 ;
                        return collect;
                    }, {}).value();
            // remove a word from this sentence to reduce redundancy in results
            var remove = function (w) {
                return delete words[w];
            };
            return {
                orig: s,
                words: words,
                remove: remove
            };
        }
143024
def text_to_phonemes(self, word: str) -> list:
        
        phonemes = []
        is_repeted = False
        if len(word) >= 2:
            for index in range(len(word) - 1):
                if is_repeted:
                    is_repeted = False
                    continue
                if word[index:index + 2] in self.diphthongs_ipa:  # diphthongs
                    phonemes.append(self.diphthongs_ipa_class[word[index] + word[index + 1]])
                    is_repeted = True
                elif word[index] == word[index + 1]:
                    phonemes.append(self.ipa_class[word[index]].lengthen())
                    is_repeted = True
                else:
                    phonemes.append(self.ipa_class[word[index]])
            if not is_repeted:
                phonemes.append(self.ipa_class[word[len(word) - 1]])
        else:
            phonemes.append(self.ipa_class[word[0]])
        return phonemes
351967
def to_text(sentence):
    
    text = ""
    for i, tok in enumerate(sentence.token):
        if i != 0:
            text += tok.before
        text += tok.word
    return text
1018915
def sentence_texts(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.texts(SENTENCES)
890968
def extract(self, text):
        
        _extracted = []
        if text.strip() == "":
            return _extracted
        parsed_sentences = self._parse_text(text)
        for s in parsed_sentences:
            tokens = s.split()
            new_np = []
            for t in tokens:
                w, tag, phrase, role = t.split('/')
                # exclude some parser errors (e.g. VB within NP),
                # extend startswith tuple if necessary
                if 'NP' in phrase and not self._is_verb(w, tag):
                    if len(new_np) > 0 and w.lower() in START_NEW_NP:
                        _extracted.append(" ".join(new_np))
                        new_np = [w]
                    else:
                        # normalize capitalisation of sentence starters, except
                        # for nouns
                        new_np.append(w.lower() if tokens[0].startswith(w) and
                                      not tag.startswith('N') else w)
                else:
                    if len(new_np) > 0:
                        _extracted.append(" ".join(new_np))
                    new_np = []
        return self._filter_extracted(_extracted)
57692
def _normalize(sentences):
    ''''''

    tfidf = TfidfVectorizer(tokenizer=_tokenize,
                            stop_words='english',
                            decode_error='ignore',
                            ngram_range=(1,2))
    return tfidf.fit_transform(sentences)
775621
def split_sentences(s, pad=0):
    
    sentences = []
    for index, sentence in enumerate(s.split('. ')):
        padding = ''
        if index > 0:
            padding = ' ' * (pad + 1)
        if sentence.endswith('.'):
            sentence = sentence[:-1]
        sentences.append('%s %s.' % (padding, sentence.strip()))
    return "\n".join(sentences)
272456
def _bpe_to_words(sentence, delimiter='@@'):
    """"""
    words = []
    word = ''
    delimiter_len = len(delimiter)
    for subwords in sentence:
        if len(subwords) >= delimiter_len and subwords[-delimiter_len:] == delimiter:
            word += subwords[:-delimiter_len]
        else:
            word += subwords
            words.append(word)
            word = ''
    return words
1295847
def scan_text(self, input_string):
        
        syllables = self._make_syllables(input_string)
        sentence_syllables = self._syllable_condenser(syllables)
        meter = self._scansion(sentence_syllables)
        return meter
670427
def word_tokenize(self, text: str) -> List[str]:
        
        return word_tokenize(text, custom_dict=self.__trie_dict, engine=self.__engine)
1018114
def sentences(self, index = None):
        
        if index is None:
            return self.select(Sentence,None,True,default_ignore_structure)
        else:
            if index < 0:
                index = self.count(Sentence,None,True,default_ignore_structure) + index
            for i,e in enumerate(self.select(Sentence,None,True,default_ignore_structure)):
                if i == index:
                    return e
            raise IndexError