Replace long and short values for each input syllable.
        :param sentence_syllables: A list of strings
        :return : '˘' and '¯' to represent short and long syllables,
        respectively
        :rtype : list
320344
def _scansion(self, sentence_syllables):
        

        scanned_text = []
        for sentence in sentence_syllables:
            scanned_sent = []
            for syllable in sentence:
                if self._long_by_position(syllable, sentence) or \
                   self._long_by_nature(syllable):
                    scanned_sent.append('¯')
                else:
                    scanned_sent.append('˘')
            if len(scanned_sent) > 1:
                del scanned_sent[-1]
                scanned_sent.append('x')
            scanned_text.append(''.join(scanned_sent))
        return scanned_text
453727
def sentence_similarity(sentence1, sentence2):
    
    # Tokenize and tag
    sentence1 = pos_tag(word_tokenize(sentence1))
    sentence2 = pos_tag(word_tokenize(sentence2))

    # Get the synsets for the tagged words
    synsets1 = [tagged_to_synset(*tagged_word) for tagged_word in sentence1]
    synsets2 = [tagged_to_synset(*tagged_word) for tagged_word in sentence2]

    # Filter out the Nones
    synsets1 = [ss for ss in synsets1 if ss]
    synsets2 = [ss for ss in synsets2 if ss]
    #print(synsets1)
    #print(synsets2)
    score, count = 0.0, 0.0

    # For each word in the first sentence
    for synset in synsets1:
        # Get the similarity value of the most similar word in the other sentence

        best_score=[synset.path_similarity(ss) for ss in synsets2 if synset.path_similarity(ss)]

        # Check that the similarity could have been computed
        if best_score:
            score += max(best_score)
            count += 1

    # Average the values
    if count > 0:
        score /= count
    else:
        score = 0

    return score
619827
def __replace_verbs(sentence, counts):
    

    if sentence is not None:
        while sentence.find('#VERB') != -1:
            sentence = sentence.replace('#VERB', str(__get_verb(counts)), 1)

            if sentence.find('#VERB') == -1:
                return sentence
        return sentence
    else:
        return sentence
1309766
def __replace_nouns(sentence, counts):
    

    if sentence is not None:
        while sentence.find('#NOUN') != -1:
            sentence = sentence.replace('#NOUN', str(__get_noun(counts)), 1)

            if sentence.find('#NOUN') == -1:
                return sentence

        return sentence
    else:
        return sentence
787313
def gen_sentences(self, tokens, aliases=None):
        
        if aliases is None:
            aliases = {}
        for sentence in self._gen_sentences(tokens):
            try:
                alias = aliases[str(sentence[0])]
            except KeyError:
                # do nothing if no alias is found
                pass
            except IndexError:
                pass
            else:
                sentence[0:1] = list(Program(alias).gen_tokens())
            yield transform(Sentence(sentence), self.transforms)
856161
def train_sentence_tokenizer(self: object, text: str):
        
        language_punkt_vars = PunktLanguageVars

        # Set punctuation
        if self.punctuation:
            if self.strict:
                language_punkt_vars.sent_end_chars = self.punctuation + self.strict_punctuation
            else:
                language_punkt_vars.sent_end_chars = self.punctuation

        # Set abbreviations
        trainer = PunktTrainer(text, language_punkt_vars)
        trainer.INCLUDE_ALL_COLLOCS = True
        trainer.INCLUDE_ABBREV_COLLOCS = True

        tokenizer = PunktSentenceTokenizer(trainer.get_params())

        if self.abbreviations:
            for abbreviation in self.abbreviations:
                tokenizer._params.abbrev_types.add(abbreviation)

        return tokenizer
726734
def batch_split_sentences(self, texts: List[str]) -> List[List[str]]:
        
        return [self.split_sentences(text) for text in texts]
1139547
def get_sentences_list(self, sentences=1):
        
        if sentences < 1:
            raise ValueError('Param "sentences" must be greater than 0.')

        sentences_list = []

        while sentences:
            num_rand_words = random.randint(self.MIN_WORDS, self.MAX_WORDS)

            random_sentence = self.make_sentence(
                random.sample(self.words, num_rand_words))

            sentences_list.append(random_sentence)
            sentences -= 1

        return sentences_list
1363492
def remove_blank_spaces(syllables: List[str]) -> List[str]:
    
    cleaned = []
    for syl in syllables:
        if syl == " " or syl == '':
            pass
        else:
            cleaned.append(syl)
    return cleaned
1267807
def __replace_random(sentence):
    

    sub_list = None
    choice = None

    if sentence is not None:

        while sentence.find('#RANDOM') != -1:

            random_index = sentence.find('#RANDOM')
            start_index = sentence.find('#RANDOM') + 8
            end_index = sentence.find(']')

            if sentence.find('#RANDOM') is not None:
                sub_list = sentence[start_index:end_index].split(',')

                choice = random.randint(1, int(sub_list[0]))
                # _sub_list[_choice]

            to_be_replaced = sentence[random_index:end_index + 1]
            sentence = sentence.replace(to_be_replaced, sub_list[choice], 1)

            if sentence.find('#RANDOM') == -1:
                return sentence

        return sentence
    else:
        return sentence
1018915
def sentence_texts(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.texts(SENTENCES)
531704
def clean_text_by_sentences(text, language="english", additional_stopwords=None):
    
    init_textcleanner(language, additional_stopwords)
    original_sentences = split_sentences(text)
    filtered_sentences = filter_words(original_sentences)

    return merge_syntactic_units(original_sentences, filtered_sentences)
477573
def generate(topic, add_punctuation, character_count=None):
    
    corpus_cursor = db.markovify.find({'topic': topic})
    if(corpus_cursor):
        corpus = ''
        for text in corpus_cursor:
            corpus = punctuate(corpus, text['text'], add_punctuation)
        text_model = markovify.Text(corpus)
        sentence = text_model.make_short_sentence(character_count) \
            if character_count else text_model.make_sentence()
        if not sentence:
            raise Exception('There is not enough in the corpus to generate a sentence.')
        return sentence
    raise Exception('No text found for topic: ' + topic)
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
1097695
def mark_sentence_boundaries(sequences, drop=0.0):  # pragma: no cover
    """"""
    for sequence in sequences:
        sequence.insert(0, "-EOL-")
        sequence.insert(0, "-EOL-")
        sequence.append("-EOL-")
        sequence.append("-EOL-")
    return sequences, None
207298
function tokenize_sentence(sentence) {
  var tokenized = tokenizer.tokenize(sentence);
  logger.info("tokenize_sentence: " + tokenized);
  return(tokenized);
}