Fixes out-of-the-sentence links in the given sentence.
        The sentence is a sublist of *alignments*, starting from 
        *sent_start* and ending one token before *sent_end*;
962317
def _fix_out_of_sentence_links( alignments, sent_start, sent_end ):
    ''' 
    ''' 
    sent_len = sent_end - sent_start
    j = sent_start
    while j < sent_start + sent_len:
        for rel_id, rel in enumerate( alignments[j][PARSER_OUT] ):
            if int( rel[1] ) >= sent_len:
                # If the link points out-of-the-sentence, fix
                # the link so that it points inside the sentence
                # boundaries:
                wid = j - sent_start
                if sent_len == 1:
                    # a single word becomes a root
                    rel[1] = -1
                elif wid-1 > -1:
                    # word at the middle/end is linked to the previous
                    rel[1] = wid - 1
                elif wid-1 == -1:
                    # word at the beginning is linked to the next
                    rel[1] = wid + 1
                alignments[j][PARSER_OUT][rel_id] = rel
        j += 1
687798
def sentence_starts(self):
        """"""
        if not self.is_tagged(SENTENCES):
            self.tokenize_sentences()
        return self.starts(SENTENCES)
1344251
function fix(text) {
    if (!text) return;
    return text.split('\n').map(function(line) {
      if (line.match(/^- /))
        line = '  ' + line;
      return line;
    }).join('\n');
  }
395814
def split_infinitive_warning(sentence_str):
    """"""
    sent_doc = textacy.Doc(sentence_str, lang='en_core_web_lg')
    inf_pattern = r'<PART><ADV><VERB>' # To aux/auxpass* csubj
    infinitives = textacy.extract.pos_regex_matches(sent_doc, inf_pattern)
    for inf in infinitives:
        if inf[0].text.lower() != 'to':
            continue
        if inf[-1].tag_ != 'VB':
           continue 
        return 'SplitInfinitiveWarning'
448850
def match(self, sentence, start=0, _v=None, _u=None):
        
        if sentence.__class__.__name__ == "Sentence":
            pass
        elif isinstance(sentence, list) or sentence.__class__.__name__ == "Text":
            return find(lambda m,s: m is not None, ((self.match(s, start, _v), s) for s in sentence))[0]
        elif isinstance(sentence, basestring):
            sentence = Sentence(sentence)
        elif isinstance(sentence, Match) and len(sentence) > 0:
            sentence = sentence[0].sentence.slice(sentence[0].index, sentence[-1].index + 1)
        # Variations (_v) further down the list may match words more to the front.
        # We need to check all of them. Unmatched variations are blacklisted (_u).
        # Pattern.search() calls Pattern.match() with a persistent blacklist (1.5x faster).
        a = []
        for sequence in (_v is not None and _v or self._variations()):
            if _u is not None and id(sequence) in _u:
                continue
            m = self._match(sequence, sentence, start)
            if m is not None:
                a.append((m.words[0].index, len(m.words), m))
            if m is not None and m.words[0].index == start:
                return m
            if m is None and _u is not None:
                _u[id(sequence)] = False
        # Return the leftmost-longest.
        if len(a) > 0:
            return sorted(a)[0][-1]
16752
def span_tokenize_sents(self, strings):
        
        raise NotImplementedError("span_tokenizer and span_tokenzie_sents not yet implemented. ;)")
        for s in strings:
            yield list(self.span_tokenize(s))
453299
private void improveGermanSentences(JCas jcas) {
		/* 
		 * these POS tag sequences will decide whether we want to merge two sentences
		 * that have (supposedly wrongfully) been split.
		 */
		HashSet<String[]> posRules = new HashSet<String[]>();
		posRules.add(new String[] {"CARD", "\\$.", "NN"});
		posRules.add(new String[] {"CARD", "\\$.", "NE"});
		
		FSIterator sentIter = jcas.getAnnotationIndex(Sentence.type).iterator();
		
		// compare two sentences at a time in order to have access to all POS tags
		HashSet<HashSet<Sentence>> toMerge = new HashSet<HashSet<Sentence>>();
		Sentence prevSent = null, thisSent = null;
		while(sentIter.hasNext()) {
			if(thisSent == null) {
				thisSent = (Sentence) sentIter.next();
				continue;
			}
			
			prevSent = thisSent;
			thisSent = (Sentence) sentIter.next();
			/* 
			 * select the last two tokens within the previous sentence as well as the
			 * first of the current one and check for matches.
			 */
			Token penultimateToken = null, ultimateToken = null, firstToken = null;
			FSIterator tokIter = jcas.getAnnotationIndex(Token.type).subiterator(thisSent);
			if(tokIter.hasNext()) {
				firstToken = (Token) tokIter.next();
			}
			
			tokIter = jcas.getAnnotationIndex(Token.type).subiterator(prevSent);
			while(tokIter.hasNext()) {
				if(ultimateToken == null) {
					ultimateToken = (Token) tokIter.next();
					continue;
				}
				penultimateToken = ultimateToken;
				ultimateToken = (Token) tokIter.next();
			}
			
			// check that all tokens for further analysis are present. if not: skip
			if(penultimateToken == null || ultimateToken == null || firstToken == null) {
				continue;
			}
			
			// check rules, memorize sentences to be merged
			for(String[] posRule : posRules) {
				/* 
				 * either one of the pre-defined POS rules fit, or the first token's 
				 * covered text begins with lower case characters.
				 */
				if((penultimateToken.getPos() != null && penultimateToken.getPos().matches(posRule[0]) &&
						ultimateToken.getPos() != null && ultimateToken.getPos().matches(posRule[1]) &&
						firstToken.getPos() != null && firstToken.getPos().matches(posRule[2]))
						||
						(firstToken.getCoveredText().matches("^[a-z/].*"))) {
					/* 
					 * check whether one of the previous candidate pairs already 
					 * contains one of our sentences.
					 */
					Boolean candidateExisted = false;
					for(HashSet<Sentence> mergeCandidate : toMerge) {
						if(mergeCandidate.contains(thisSent) || mergeCandidate.contains(prevSent)) {
							// we add both here because sets ignore duplicates
							mergeCandidate.add(prevSent);
							mergeCandidate.add(thisSent);
							
							candidateExisted = true;
							break;
						}
					}
					
					/* 
					 * if one of the sentences was not already to be merged with another,
					 * create a new merge candidate set
					 */
					if(!candidateExisted) {
						HashSet<Sentence> newCandidate = new HashSet<Sentence>();
						newCandidate.add(prevSent);
						newCandidate.add(thisSent);
						
						toMerge.add(newCandidate);
					}
					
					break; // don't need to do the next rules; already merging.
				}
			}
		}
		
		// iterate over the previously collected merge candidates
		
		for(HashSet<Sentence> mergeCandidate : toMerge) {
			// find the earliest beginning and latest end for the set of sentences
			Integer beginIndex = Integer.MAX_VALUE, endIndex = Integer.MIN_VALUE;

			Sentence mergedSent = new Sentence(jcas);
			for(Sentence s : mergeCandidate) {
				if(s.getBegin() < beginIndex) {
					beginIndex = s.getBegin();
				}
				
				if(s.getEnd() > endIndex) {
					endIndex = s.getEnd();
				}
				
				s.removeFromIndexes();
			}
			
			// set values, add to jcas
			mergedSent.setBegin(beginIndex);
			mergedSent.setEnd(endIndex);
			mergedSent.addToIndexes();
		}
	}
403481
private void fixActiveEdgeAfterSuffixLink(Suffix<T, S> suffix) {
        while (activeEdge != null && activeLength > activeEdge.getLength()) {
            activeLength = activeLength - activeEdge.getLength();
            activeNode = activeEdge.getTerminal();
            Object item = suffix.getItemXFromEnd(activeLength + 1);
            activeEdge = activeNode.getEdgeStarting(item);
        }
        resetActivePointToTerminal();
    }
1266524
def get_unique_sentence_indices( text ):
    '''  '''
    # Add sentence annotation (if missing)
    if not text.is_tagged( SENTENCES ):
        text.tokenize_sentences()
    # Collect (unique) sent indices over the whole text
    sent_indices = []
    sent_id = 0
    for sub_text in text.split_by( SENTENCES ):
        for word in sub_text.words:
            sent_indices.append( sent_id )
        sent_id += 1
    assert len(sent_indices) == len(text.words), '(!) Number of sent indices should match nr of words!'
    return sent_indices
1074119
public String tokenizeToCoNLLOffsets() {

    final StringBuilder sb = new StringBuilder();
    final String[] sentences = segmenter.segmentSentence();
    final List<List<Token>> tokens = tokenizer.tokenize(sentences);
    for (final List<Token> tokSentence : tokens) {
      for (final Token token : tokSentence) {
        String tokenValue = token.getTokenValue();
        if (tokenValue.equals(RuleBasedSegmenter.PARAGRAPH)) {
          tokenValue = DEFAULT_TOKEN_VALUE;
        }
        sb.append(tokenValue.trim()).append(DELIMITER).append(token.startOffset())
            .append(DELIMITER).append(token.tokenLength()).append(LINE_BREAK);
      }
      sb.append(LINE_BREAK);
    }
    return sb.toString();
  }
959128
private String fillSentenceWithSpaces(List<String> sentence) {
        sentenceWithSpaces.clear();

        //We don't need to do this process if the sentence received is a single word.
        if (sentence.size() > 1) {
            //We fill with normal spaces first, we can do this with confidence because "fitsInSentence"
            //already takes these spaces into account.
            for (String word : sentence) {
                sentenceWithSpaces.add(word);
                sentenceWithSpaces.add(NORMAL_SPACE);
            }

            //Filling sentence with thin spaces.
            while (fitsInSentence(HAIR_SPACE, sentenceWithSpaces, false)) {
                //We remove 2 from the sentence size because we need to make sure we are not adding
                //spaces to the end of the line.
                sentenceWithSpaces.add(getRandomNumber(sentenceWithSpaces.size() - 2), HAIR_SPACE);
            }
        }

        return getSentenceFromList(sentenceWithSpaces, false);
    }
1077098
def add(self, sentence_text, **kwargs):
        '''  '''
        sent = MeCabSent.parse(sentence_text, **kwargs)
        self.sents.append(sent)
        return sent
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
775621
def split_sentences(s, pad=0):
    
    sentences = []
    for index, sentence in enumerate(s.split('. ')):
        padding = ''
        if index > 0:
            padding = ' ' * (pad + 1)
        if sentence.endswith('.'):
            sentence = sentence[:-1]
        sentences.append('%s %s.' % (padding, sentence.strip()))
    return "\n".join(sentences)
398095
public Sentence filter(Sentence sent)
	{
		//logger.debug("\"" + Array.toString(sent) + "\"");

		int start = 0, end = sent.length() - 1;
		boolean b = false;
		for (int i=0;i<sent.length();i++)
		{
			if (sent.wordAt(i).getRole().equals(Word.AGENT_LABEL))
			{
					start = i - getSize();
					b = true;
					break;
			}
		} // end for i

		if (!b)
			return null;

		end = start + 2 * getSize() + 1;

		if (start < 0)
			start = 0;

		if (end > sent.length())
			end = sent.length();

		return sent.fragment(start, end);

	}
769757
@Override
    protected List<Term> segSentence(char[] sentence)
    {
        if (sentence.length == 0) return Collections.emptyList();
        List<Term> termList = roughSegSentence(sentence);
        if (!(config.ner || config.useCustomDictionary || config.speechTagging))
            return termList;
        List<Vertex> vertexList = toVertexList(termList, true);
        if (config.speechTagging)
        {
            Viterbi.compute(vertexList, CoreDictionaryTransformMatrixDictionary.transformMatrixDictionary);
            int i = 0;
            for (Term term : termList)
            {
                if (term.nature != null) term.nature = vertexList.get(i + 1).guessNature();
                ++i;
            }
        }
        if (config.useCustomDictionary)
        {
            combineByCustomDictionary(vertexList);
            termList = convert(vertexList, config.offset);
        }
        return termList;
    }
774071
def _mangle_sentences_from_file(input_file):
    """"""
    try:
        with open(input_file, 'r') as f:
            # final sentence may not be a complete sentence, save and prepend to next chunk
            leftovers = ''
            sentence_no = 0
            for chunk in read_in_chunks(f): # lazy way of reading our file in case it's large
                # prepend leftovers to chunk
                chunk = chunk.decode('utf8')
                chunk = leftovers + chunk
                chunk = chunk.replace(';', '.') # replace semi colons with periods 
                doc = nlp(chunk)

                # last sentence may not be sentence, move to next chunk
                sents = [sent.string.strip() for sent in doc.sents]
                if len(sents) > 1:
                    leftovers = sents[-1] + chunk.rpartition(sents[-1])[-1]
                    sents = sents[:-1]

                for sent in sents:
                    sent = sent.replace('\n', ' ')
                    sent = sent.replace('\r', ' ')
                    sent = re.sub( '\s+', ' ', sent ).strip()
                    if len(sent) < 5:
                        continue # skip tiny sentences
                    # add original sentence to database
                    cursor.execute('''insert into orignal_sentences (sentence,
                    flesch_reading_ease, flesch_kincaid_grade_level) values (?,
                    ?, ?)''', (sent, textstat.flesch_reading_ease(sent),
                        textstat.flesch_kincaid_grade(sent)))
                    og_sent_id = cursor.lastrowid
                    for mangled_sentence in mangle_agreement(sent):
                        # add mangled sentence to database
                        cursor.execute('''insert into mangled_sentences
                        (original_sentence_id, sentence) values (?, ?)''',
                        (og_sent_id, mangled_sentence))
                conn.commit() # commit all sentences in this chunk

    except Exception as e:
        print('error on {}'.format(input_file))
        print(e)