Annotates the spans of {@code sentence} with feature vectors
created by applying {@code featureGenerator}.

@param sentence
@param featureGenerator
@return
428195
public static SpanFeatureAnnotation annotate(AnnotatedSentence sentence,
      FeatureVectorGenerator<StringContext> featureGenerator) {
    int numTerminals = sentence.size();
    Tensor[] featureVectors = new Tensor[numTerminals * (numTerminals + 1) / 2];
    for (int i = 0; i < numTerminals; i++) {
      for (int j = i; j < numTerminals; j++) {
        int spanIndex = getSpanIndex(i, j, numTerminals);
        featureVectors[spanIndex] = featureGenerator.apply(
            new StringContext(i, j, sentence));
      }
    }
    return new SpanFeatureAnnotation(featureVectors, numTerminals);
  }
1168418
public static String sentences(int sentenceCount, boolean supplemental) {
        List<String> sentenceList = new ArrayList<String>();
        for (int i = 0; i < sentenceCount; i++) {
            sentenceList.add(sentence(3, supplemental, 6));
        }

        String sentenceString = StringUtils.join(sentenceList, " ");
        return sentenceString;

    }
408801
private static void splitInSentences(HashSet<String> sentencesSet, String text) {
		
		if (inputType == inputTypes.PARAGRAPH) { // split in multiple sentences
			LinkedList<LinkedList<Word>> sentences = Splitting.tokenizeSentences(LVMorphologyReaderAndWriter.getAnalyzer(), text);
			for (LinkedList<Word> sentence : sentences) 
			{
				if(sentence != null)
				{
					String madeSentence = "";
					for(int w = 0; w < sentence.size(); w++)
					{
						madeSentence = madeSentence + " " + sentence.get(w);
					}
					madeSentence = madeSentence.trim();
					//pārbaudam, vai teikuma atšķiribas nav punkts teikuma beigas
					if(madeSentence.length() > 0 && madeSentence.lastIndexOf(".") >= 0) madeSentence = madeSentence.substring(0, madeSentence.length() - 1);
					sentencesSet.add(madeSentence.trim());
				}
			}
		} 
	}
1001660
function tag_sentence_wordnet(tagged_sentence, callback) {
  var wordnet_results = {};
  var nr_tokens = tagged_sentence.length;

  tagged_sentence.forEach(function(tagged_word) {
    logger.debug("tag_sentence: processing " + tagged_word);
    wordnet.lookup(tagged_word[0], function(results) {
      results.forEach(function(result) {
        if (tagged_word.lastIndexOf(result.pos) <= 0) {
          tagged_word.push(result.pos);
          logger.debug("Lexical category of " + tagged_word[0] + " is: " + result.pos);
        }
      });

      nr_tokens--;
      if (nr_tokens === 0) {
        logger.info("Exit tag_sentence_wordnet: " + JSON.stringify(tagged_sentence));
        callback(tagged_sentence);
      }
    });
  });
}
925596
def _add_sentence(self, sentence, ignore_traces=True):
        
        self.sentences.append(self._node_id)
        # add edge from document root to sentence root
        self.add_edge(self.root, self._node_id, edge_type=dg.EdgeTypes.dominance_relation)
        self._parse_sentencetree(sentence, ignore_traces=ignore_traces)
        self._node_id += 1
955032
function transformer(cst) {
    var concatenate = concatenateFactory();

    visit(cst, 'SentenceNode', any);
    visit(cst, concatenate);

    concatenate.done();
}
585359
public final AnalyzedSentence replace(AnalyzedSentence sentence) throws IOException {
    DisambiguationPatternRuleReplacer replacer = new DisambiguationPatternRuleReplacer(this);
    return replacer.replace(sentence);
  }
141888
@Override
    public Tokenizer create(String toTokenize) {
        if (toTokenize.isEmpty()) {
            throw new IllegalArgumentException("Unable to proceed; no sentence to tokenize");
        }

        Tokenizer t = new JapaneseTokenizer(kuromoji, toTokenize, useBaseForm);

        if (preProcessor != null) {
            t.setTokenPreProcessor(preProcessor);
        }

        return t;
    }
262274
public static HMMPOSTagger learn(String[][] sentences, PennTreebankPOS[][] labels) {
        int index = 1;
        int suffixIndex = 0;
        Map<String, Integer> symbol = new HashMap<>();
        Map<String, Integer> suffix = new HashMap<>();
        for (int i = 0; i < sentences.length; i++) {
            for (int j = 0; j < sentences[i].length; j++) {
                Integer sym = symbol.get(sentences[i][j]);
                if (sym == null) {
                    symbol.put(sentences[i][j], index++);
                }
                
                if (sentences[i][j].length() > 2) {
                    String s = sentences[i][j].substring(sentences[i][j].length() - 2);
                    sym = suffix.get(s);
                    if (sym == null) {
                        suffix.put(s, suffixIndex++);
                    }
                }
            }
        }

        int numStates = PennTreebankPOS.values().length;

        double[] pi = new double[numStates];
        double[][] a = new double[numStates][numStates];
        double[][] b = new double[numStates][symbol.size() + 1];
        double[][] c = new double[numStates][suffix.size()];
        
        PennTreebankPOS[] tags = PennTreebankPOS.values();
        for (int i = 0; i < numStates; i++) {
            if (tags[i].open) {
                b[i][0] = 1.0;
            }
        }

        for (int i = 0; i < sentences.length; i++) {
            int[] tag = translate(labels[i]);
            int[][] obs = translate(symbol, suffix, sentences[i]);

            pi[tag[0]]++;
            b[tag[0]][obs[0][0]]++;            
            if (obs[0][1] >= 0) {
                c[tag[0]][obs[0][1]]++;
            }
            
            for (int j = 1; j < obs.length; j++) {
                a[tag[j-1]][tag[j]]++;
                b[tag[j]][obs[j][0]]++;
                if (obs[j][1] >= 0) {
                    c[tag[j]][obs[j][1]]++;
                }
            }                
        }

        Math.unitize1(pi);
        for (int i = 0; i < numStates; i++) {
            Math.unitize1(a[i]);
            Math.unitize1(b[i]);
            Math.unitize1(c[i]);
        }
        
        return new HMMPOSTagger(symbol, suffix, pi, a, b, c);
    }
1205482
public synchronized List<Sentence> sentences(String text) {
        requireValidHandle();
        Libvoikko lib = getLib();
        List<Sentence> result = new ArrayList<Sentence>();
        if (!isValidInput(text)) {
            result.add(new Sentence(text, SentenceStartType.NONE));
            return result;
        }
        byte[] textBytes = s2n(text);
        int textLen = textBytes.length - 1;
        SizeTByReference sentenceLenByRef = new SizeTByReference();
        while (textLen > 0) {
            int sentenceTypeInt = lib.voikkoNextSentenceStartCstr(handle, textBytes, new SizeT(textLen), sentenceLenByRef);
            int sentenceLen = sentenceLenByRef.getValue().intValue();
            SentenceStartType sentenceType = SentenceStartType.values()[sentenceTypeInt];
            String tokenText = text.substring(0, sentenceLen);
            result.add(new Sentence(tokenText, sentenceType));
            text = text.substring(sentenceLen);
            textBytes = s2n(text);
            textLen = textBytes.length - 1;
        }
        return result;
    }
905350
public void setSentenceIterator(@NonNull SentenceIterator iterator) {
        //if (tokenizerFactory == null) throw new IllegalStateException("Please call setTokenizerFactory() prior to setSentenceIter() call.");

        if (tokenizerFactory != null) {
            SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(iterator)
                            .tokenizerFactory(tokenizerFactory)
                            .allowMultithreading(configuration == null || configuration.isAllowParallelTokenization())
                            .build();
            this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
        } else
            log.error("Please call setTokenizerFactory() prior to setSentenceIter() call.");
    }
329471
String[] tokenize(String sentence) {
    TokenizerME tokenizer = new TokenizerME(tokenModel);
    String cleanString = sentence.replace('’', '\'');  // this is the type of apostrophe that OpenNLP expects
    return tokenizer.tokenize(cleanString);
  }
1266395
public List<String> generateFeats(String[] tokens) {
        tokens = Arrays.stream(tokens).filter(Objects::nonNull).filter(x -> !x.trim().isEmpty())
                .map(String::toLowerCase).toArray(String[]::new);

        if (tokens.length == 0) {
            return new ArrayList<>();
        }

        List<String> feats = new ArrayList<>();
        feats.add("word_count_" + String.valueOf(tokens.length));

        if (tokens[0].toLowerCase().startsWith("wh")) {
            feats.add("wh_start");
        }

        // Verify, Ask, Say - imperative
        boolean isImperativeStart = IMPERATIVE_KEYWORDS.contains(tokens[0]);
        String lastToken = tokens[tokens.length - 1];
        if (isImperativeStart) {
            feats.add("imperative_start");
            if (!Pattern.matches("\\p{Punct}+", lastToken)) {
                String[] newTokens = new String[tokens.length + 1];
                System.arraycopy(tokens, 0, newTokens, 0, tokens.length);
                tokens = newTokens;
            }
            tokens[tokens.length - 1] = "!";
        }

        // Start with question words.
        String sentence = Arrays.stream(tokens).collect(Collectors.joining(StringUtils.SPACE));
        long questionPrefixCount = QUESTION_PREFIX.stream().filter(sentence::startsWith).count();
        if (questionPrefixCount > 0) {
            feats.add("question_prefix");
            if (!Pattern.matches("\\p{Punct}+", lastToken)) {
                String[] newTokens = new String[tokens.length + 1];
                System.arraycopy(tokens, 0, newTokens, 0, tokens.length);
                tokens = newTokens;
            }
            tokens[tokens.length - 1] = "?";
        }

        feats.add("first_word_" + tokens[0]);

        if (tokens.length > 1) {
            feats.add("sec_word_" + tokens[1]);
        }

        lastToken = tokens[tokens.length - 1];
        if (Pattern.matches("\\p{Punct}+", lastToken)) {
            feats.add("punct_" + lastToken);
        }

        // whether
        long whetherKeyWord = Arrays.stream(tokens).filter("whether"::equals).count();
        if (whetherKeyWord > 0) {
            feats.add("has_whether");
        }

        List<String> biWord = new ArrayList<>();
        List<String> triWord = new ArrayList<>();

        for (int i = 0; i < tokens.length && i < 6; i++) { // 6 words maximum
            // Bigram
            if (i + 1 < tokens.length) {
                biWord.add(tokens[i] + "_" + tokens[i + 1]);
            }

            // Trigram
            if (i + 2 < tokens.length) {
                triWord.add(tokens[i] + "_" + tokens[i + 1] + "_" + tokens[i + 2]);
            }
        }

        feats.addAll(biWord);
        feats.addAll(triWord);

//        LOG.debug("feats: " + feats);
        return feats;
    }
285176
@Override
  public List<List<Token>> tokenize(final String[] sentences) {
    final long start = System.nanoTime();
    int noTokens = 0;
    int prevIndex = 0;
    int curIndex = 0;
    final String language = this.lang;
    final List<List<Token>> result = new ArrayList<List<Token>>();
    // TODO improve this
    final String offsetText = this.originalText;
    for (final String sentence : sentences) {
      if (DEBUG) {
        System.err.println("-> Segmented:" + sentence);
      }
      final List<Token> tokens = new ArrayList<Token>();
      final String[] curTokens = getTokens(sentence);
      for (final String arrayToken : curTokens) {
        curIndex = offsetText.indexOf(arrayToken, prevIndex);
        // crap rule for corrected URLs
        if (curIndex == -1) {
          curIndex = prevIndex + 1;
        }
        final Token curToken = this.tokenFactory.createToken(arrayToken,
            curIndex, arrayToken.length());
        // exceptions to WFs
        addTokens(curToken, tokens);
        if (DEBUG) {
          System.err.println("-> Token:" + arrayToken + " curIndex: " + curIndex
              + " prev: " + prevIndex);
        }
        prevIndex = curIndex + curToken.tokenLength();
      }
      result.add(tokens);
      noTokens = noTokens + curTokens.length;
    }
    normalizeTokens(result, language);
    final long duration = System.nanoTime() - start;
    final double toksPerSecond = noTokens / (duration / 1000000000.0);
    if (DEBUG) {
      System.err.printf(
          "ixa-pipe-tok tokenized %d tokens at %.2f tokens per second.%n",
          noTokens, toksPerSecond);
    }
    return result;
  }
52858
private List<Map.Entry<Sentence, Map.Entry<Integer, Integer>>> getRelevantSentences(List<String> inputs, String token, int maxSentences) throws IOException {
    List<Map.Entry<Sentence, Map.Entry<Integer, Integer>>> sentences = new ArrayList<>();
    for (String input : inputs) {
      if (new File(input).isDirectory()) {
        File file = new File(input, token + ".txt");
        if (!file.exists()) {
          throw new RuntimeException("File with example sentences not found: " + file);
        }
        try (FileInputStream fis = new FileInputStream(file)) {
          SentenceSource sentenceSource = new PlainTextSentenceSource(fis, language);
          sentences = getSentencesFromSource(inputs, token, maxSentences, sentenceSource);
        }
      } else {
        SentenceSource sentenceSource = MixingSentenceSource.create(inputs, language);
        sentences = getSentencesFromSource(inputs, token, maxSentences, sentenceSource);
      }
    }
    return sentences;
  }
65088
public static Window windowForWordInPosition(int windowSize, int wordPos, List<String> sentence) {
        List<String> window = new ArrayList<>();
        List<String> onlyTokens = new ArrayList<>();
        int contextSize = (int) Math.floor((windowSize - 1) / 2);

        for (int i = wordPos - contextSize; i <= wordPos + contextSize; i++) {
            if (i < 0)
                window.add("<s>");
            else if (i >= sentence.size())
                window.add("</s>");
            else {
                onlyTokens.add(sentence.get(i));
                window.add(sentence.get(i));

            }
        }

        String wholeSentence = StringUtils.join(sentence);
        String window2 = StringUtils.join(onlyTokens);
        int begin = wholeSentence.indexOf(window2);
        int end = begin + window2.length();
        return new Window(window, begin, end);

    }
423952
@Override
    public void typeSystemInit(TypeSystem typeSystem) throws AnalysisEngineProcessException {

        // sentence type
        this.sentenceType = AnnotatorUtil.getRequiredTypeParameter(this.context, typeSystem,
                        UimaUtil.SENTENCE_TYPE_PARAMETER);

        // token type
        this.tokenType = AnnotatorUtil.getRequiredTypeParameter(this.context, typeSystem,
                        UimaUtil.TOKEN_TYPE_PARAMETER);

        // pos feature
        this.posFeature = AnnotatorUtil.getRequiredFeatureParameter(this.context, this.tokenType,
                        UimaUtil.POS_FEATURE_PARAMETER, CAS.TYPE_NAME_STRING);

        this.probabilityFeature = AnnotatorUtil.getOptionalFeatureParameter(this.context, this.tokenType,
                        UimaUtil.PROBABILITY_FEATURE_PARAMETER, CAS.TYPE_NAME_DOUBLE);
    }