One sentence per line, formatted like

        The###DET dog###NN ate###V the###DET apple###NN

    Returns a list of pairs (tokenized_sentence, tags)
187560
def load_data(file_path: str) -> Tuple[List[str], List[str]]:
    
    data = []

    with open(file_path) as f:
        for line in f:
            pairs = line.strip().split()
            sentence, tags = zip(*(pair.split("###") for pair in pairs))
            data.append((sentence, tags))

    return data
351967
def to_text(sentence):
    
    text = ""
    for i, tok in enumerate(sentence.token):
        if i != 0:
            text += tok.before
        text += tok.word
    return text
1102062
def split_sentences_regex(text):
    
    parts = regex.split(r'([a-zA-Z0-9][.?!])[\s$]', text)
    sentences = [''.join(s) for s in zip(parts[0::2], parts[1::2])]
    return sentences + [parts[-1]] if len(parts) % 2 else sentences
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
752568
def word_texts(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return [word[TEXT] for word in self[WORDS]]
691556
def tag(self, corpus, tokenize=True):
        ''''''
        # Assume untokenized corpus has \n between sentences and ' ' between words
        s_split = SentenceTokenizer().tokenize if tokenize else lambda t: t.split('\n')
        w_split = WordTokenizer().tokenize if tokenize else lambda s: s.split()
        def split_sents(corpus):
            for s in s_split(corpus):
                yield w_split(s)

        prev, prev2 = self.START
        tokens = []
        for words in split_sents(corpus):
            context = self.START + [self._normalize(w) for w in words] + self.END
            for i, word in enumerate(words):
                tag = self.tagdict.get(word)
                if not tag:
                    features = self._get_features(i, word, context, prev, prev2)
                    tag = self.model.predict(features)
                tokens.append((word, tag))
                prev2 = prev
                prev = tag
        return tokens
1064469
def features(sentence, i):
    

    word = sentence[i]

    yield "word:{}" + word.lower()

    if word[0].isupper():
        yield "CAP"

    if i > 0:
        yield "word-1:{}" + sentence[i - 1].lower()
        if i > 1:
            yield "word-2:{}" + sentence[i - 2].lower()
    if i + 1 < len(sentence):
        yield "word+1:{}" + sentence[i + 1].lower()
        if i + 2 < len(sentence):
            yield "word+2:{}" + sentence[i + 2].lower()
326635
def get_texts(self):
        
        with self.getstream() as text_stream:
            for i, line in enumerate(text_stream):
                line = to_unicode(line)
                line = (TweetCorpus.case_normalizer or passthrough)(line)
                # line = self.case_normalizer(line)
                if self.mask is not None and not self.mask[i]:
                    continue
                ngrams = []
                for ng in tokens2ngrams((TweetCorpus.tokenizer or str.split)(line), n=self.num_grams):
                    if self.ignore_matcher(ng):
                        continue
                    ngrams += [ng]
                if not (i % 1000):
                    print(line)
                    print(ngrams)
                yield ngrams
292250
def tokenize(text=""):
    
    if text == '':
        return []
    else:
        # split tokens
        mylist = TOKEN_PATTERN.split(text)
        # don't remove newline \n
        mylist = [TOKEN_REPLACE.sub('', x) for x in mylist if x]
        # remove empty substring
        mylist = [x for x in mylist if x]
        return mylist
409168
def listup_sentence(self, data, counter=0):
        '''
        

        '''
        delimiter = self.delimiter_list[counter]
        sentence_list = []
        [sentence_list.append(sentence + delimiter) for sentence in data.split(delimiter) if sentence != ""]
        if counter + 1 < len(self.delimiter_list):
            sentence_list_r = []
            [sentence_list_r.extend(self.listup_sentence(sentence, counter+1)) for sentence in sentence_list]
            sentence_list = sentence_list_r

        return sentence_list
93652
def parse_frog(lines):
    
    sid = 0
    for i, line in enumerate(lines):
        if not line:
            # end of sentence marker
            sid += 1
        else:
            parts = line.split("\t")
            tid, token, lemma, morph, pos, conf, ne, _, parent, rel = parts
            if rel:
                rel = (rel, int(parent) - 1)
            word = u' '.join(token.split(u'_'))
            result = dict(id=i, sentence=sid, word=word, lemma=lemma, pos=pos,
                          pos_confidence=float(conf), rel=rel)
            if ne != 'O':
                # NER label from BIO tags
                result["ne"] = ne.split('_', 1)[0][2:]
            yield result
1001660
function tag_sentence_wordnet(tagged_sentence, callback) {
  var wordnet_results = {};
  var nr_tokens = tagged_sentence.length;

  tagged_sentence.forEach(function(tagged_word) {
    logger.debug("tag_sentence: processing " + tagged_word);
    wordnet.lookup(tagged_word[0], function(results) {
      results.forEach(function(result) {
        if (tagged_word.lastIndexOf(result.pos) <= 0) {
          tagged_word.push(result.pos);
          logger.debug("Lexical category of " + tagged_word[0] + " is: " + result.pos);
        }
      });

      nr_tokens--;
      if (nr_tokens === 0) {
        logger.info("Exit tag_sentence_wordnet: " + JSON.stringify(tagged_sentence));
        callback(tagged_sentence);
      }
    });
  });
}
300364
def _parse(self, lines, grammar, ignore_comments=False):
        
        results = []
        for c in lines:
            if c != '' and not (ignore_comments and c[0] == '#'):
                try:
                    results.append(grammar.parseString(c))
                except pyparsing.ParseException as e:
                    raise ValueError('Invalid syntax. Verify line {} is '
                        'correct.\n{}\n\n{}'.format(e.lineno, c, e))
        return results
504626
def split_long_sentence(sentence, words_per_line):
    
    words = sentence.split(' ')
    split_sentence = ''
    for i in range(len(words)):
        split_sentence = split_sentence + words[i]
        if (i+1) % words_per_line == 0:
            split_sentence = split_sentence + '\n'
        elif i != len(words) - 1:
            split_sentence = split_sentence + " "
    return split_sentence
1167378
def line(self, sentences, line_number=None):
        
        if line_number is None:
            line_number = self.current_line + 10
        self.current_line = line_number

        sep = []
        result = []
        for sentence in sentences:
            result.extend(sep)
            result.extend(self.sentence_bytes(sentence))
            sep = [ord(':')]

        result.extend([ENTER])
        result = self.line_number(line_number) + self.numberLH(len(result)) + result

        return result
1139547
def get_sentences_list(self, sentences=1):
        
        if sentences < 1:
            raise ValueError('Param "sentences" must be greater than 0.')

        sentences_list = []

        while sentences:
            num_rand_words = random.randint(self.MIN_WORDS, self.MAX_WORDS)

            random_sentence = self.make_sentence(
                random.sample(self.words, num_rand_words))

            sentences_list.append(random_sentence)
            sentences -= 1

        return sentences_list
774248
def _getPhrase( self, i, sentence, NPlabels ):
        ''' 
        ''' 
        phrase  = []
        indices = []
        if 0 <= i and i < len(sentence) and NPlabels[i] == 'B':
            phrase  = [ sentence[i] ]
            indices = [ i ]
            j = i + 1
            while ( j < len(sentence) ):
                if NPlabels[j] in ['B', '']:
                    break
                else:
                    phrase.append( sentence[j] )
                    indices.append( j )
                j += 1
        return phrase, indices