Lets find and replace all instances of #ADJECTIVE_MAYBE
    :param _sentence:
    :param counts:
555293
def ___replace_adjective_maybe(sentence, counts):
    

    random_decision = random.randint(0, 1)

    if sentence is not None:

        while sentence.find('#ADJECTIVE_MAYBE') != -1:

            if random_decision % 2 == 0:
                sentence = sentence.replace('#ADJECTIVE_MAYBE',
                                              ' ' + str(__get_adjective(counts)), 1)
            elif random_decision % 2 != 0:
                sentence = sentence.replace('#ADJECTIVE_MAYBE', '', 1)

            if sentence.find('#ADJECTIVE_MAYBE') == -1:
                return sentence
        return sentence
    else:
        return sentence
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features
1118599
def p_for_sentence(p):
    
    p[0] = p[1]
    if is_null(p[0]):
        return
    p[1].appendChild(make_block(p[2], p[3]))
    gl.LOOPS.pop()
726648
def apply(self, doc):
        
        if not isinstance(doc, Document):
            raise TypeError(
                "Input Contexts to MentionSentences.apply() must be of type Document"
            )

        for sentence in doc.sentences:
            yield TemporarySpanMention(
                char_start=0, char_end=len(sentence.text) - 1, sentence=sentence
            )
409168
def listup_sentence(self, data, counter=0):
        '''
        

        '''
        delimiter = self.delimiter_list[counter]
        sentence_list = []
        [sentence_list.append(sentence + delimiter) for sentence in data.split(delimiter) if sentence != ""]
        if counter + 1 < len(self.delimiter_list):
            sentence_list_r = []
            [sentence_list_r.extend(self.listup_sentence(sentence, counter+1)) for sentence in sentence_list]
            sentence_list = sentence_list_r

        return sentence_list
1277360
def inflect(word: str,
            count: int):
    
    if word in ['time', 'sentence']:
        return word if count == 1 else word + 's'
    elif word == 'was':
        return 'was' if count == 1 else 'were'
    else:
        return word + '(s)'
585359
public final AnalyzedSentence replace(AnalyzedSentence sentence) throws IOException {
    DisambiguationPatternRuleReplacer replacer = new DisambiguationPatternRuleReplacer(this);
    return replacer.replace(sentence);
  }
1287688
public List<Token> findNouns(Sentence sentence) {
    List<Token> nouns = new ArrayList<Token>();

    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("PIV") || tag.equals("ACC") || tag.equals("SC")) {

        for (Token token : syntChunks.get(i).getTokens()) {

          if (token.getPOSTag().equals("n") || token.getPOSTag().equals("pron-pers") || token.getPOSTag().equals("prop")) {
              nouns.add(token);
            }
          }
        }
      }

    return nouns;
  }
1181840
def _tokenize(cls, sentence):
        
        while True:
            match = cls._regex_tag.search(sentence)
            if not match:
                yield from cls._split(sentence)
                return
            chunk = sentence[:match.start()]
            yield from cls._split(chunk)
            tag = match.group(0)
            yield tag
            sentence = sentence[(len(chunk) + len(tag)):]
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
305893
def update(self, group: 'SentenceGroup', flags: Flags) -> None:
        

        to_append = []

        for old, new in zip_longest(self.sentences, group.sentences):
            if old is None:
                old = Sentence()
                to_append.append(old)

            if new is None:
                new = Sentence()

            old.update(new, flags)

        self.sentences.extend(to_append)
495548
def search(self, sentence):
        
        if sentence.__class__.__name__ == "Sentence":
            pass
        elif isinstance(sentence, list) or sentence.__class__.__name__ == "Text":
            a=[]; [a.extend(self.search(s)) for s in sentence]; return a
        elif isinstance(sentence, basestring):
            sentence = Sentence(sentence)
        elif isinstance(sentence, Match) and len(sentence) > 0:
            sentence = sentence[0].sentence.slice(sentence[0].index, sentence[-1].index + 1)
        a = []
        v = self._variations()
        u = {}
        m = self.match(sentence, _v=v)
        while m:
            a.append(m)
            m = self.match(sentence, start=m.words[-1].index+1, _v=v, _u=u)
        return a
367816
def _add_new_labels(self, sentences):
        '''
        

        '''
        sentence_no = -1
        total_words = 0
        vocab = self.model.vocab
        model_sentence_n = len([l for l in vocab if l.startswith("DOC_")])
        n_sentences = 0
        for sentence_no, sentence in enumerate(sentences):
            sentence_length = len(sentence.words)
            for label in sentence.labels:
                total_words += 1
                if label in vocab:
                    vocab[label].count += sentence_length
                else:
                    vocab[label] = gensim.models.word2vec.Vocab(
                        count=sentence_length)

                    vocab[label].index = len(self.model.vocab) - 1
                    vocab[label].code = [0]
                    vocab[label].sample_probability = 1.
                    self.model.index2word.append(label)
                    n_sentences += 1
                    
        return n_sentences
1267807
def __replace_random(sentence):
    

    sub_list = None
    choice = None

    if sentence is not None:

        while sentence.find('#RANDOM') != -1:

            random_index = sentence.find('#RANDOM')
            start_index = sentence.find('#RANDOM') + 8
            end_index = sentence.find(']')

            if sentence.find('#RANDOM') is not None:
                sub_list = sentence[start_index:end_index].split(',')

                choice = random.randint(1, int(sub_list[0]))
                # _sub_list[_choice]

            to_be_replaced = sentence[random_index:end_index + 1]
            sentence = sentence.replace(to_be_replaced, sub_list[choice], 1)

            if sentence.find('#RANDOM') == -1:
                return sentence

        return sentence
    else:
        return sentence
1237650
def get(self, sent_id, **kwargs):
        ''' '''
        if sent_id is not None and not isinstance(sent_id, int):
            sent_id = int(sent_id)
        if sent_id is None or not self.has_id(sent_id):
            if 'default' in kwargs:
                return kwargs['default']
            else:
                raise KeyError("Invalid sentence ID ({})".format(sent_id))
        return self.__sent_map[sent_id]
415064
public Token findVerb(Sentence sentence) {
    List<SyntacticChunk> syntChunks = sentence.getSyntacticChunks();

    for (int i = 0; i < syntChunks.size(); i++) {
      String tag = syntChunks.get(i).getTag();

      if (tag.equals("P") || tag.equals("MV") || tag.equals("PMV")
          || tag.equals("AUX") || tag.equals("PAUX"))
        return syntChunks.get(i).getTokens().get(0);
    }

    return null;
  }