The main check method. Tokenizes the text into sentences and matches these
sentences against all currently active rules.
@since 3.7
326182
public List<RuleMatch> check(AnnotatedText annotatedText, boolean tokenizeText, ParagraphHandling paraMode, RuleMatchListener listener) throws IOException {
    return check(annotatedText, tokenizeText, paraMode, listener, Mode.ALL);
  }
1180384
public void validate() {
        check("firstLine", this.firstLine);
        check("beforeEachLine", this.beforeEachLine);
        check("endLine", this.endLine);
        check("firstLineDetectionPattern", this.firstLineDetectionPattern);
        check("lastLineDetectionPattern", this.lastLineDetectionPattern);
        check("isMultiline", this.isMultiline);
        check("allowBlankLines", this.allowBlankLines);
        // skip line can be null
    }
267111
boolean matchComment(
            final boolean blockMatching,
            final int markupLevel, final int markupBlockIndex) {

        checkMarkupLevel(markupLevel);

        if (this.markupSelectorItem.anyLevel() || markupLevel == 0 || (this.prev != null && this.prev.matchedMarkupLevels[markupLevel - 1])) {
            // This text has not matched yet, but might match, so we should check

            this.matchesThisLevel = this.markupSelectorItem.matchesComment(markupBlockIndex, this.markupBlockMatchingCounter);

            if (matchesPreviousOrCurrentLevel(markupLevel)) {
                // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

                if (this.next != null) {
                    return this.next.matchComment(blockMatching, markupLevel, markupBlockIndex);
                }
                return (blockMatching? true : this.matchesThisLevel);

            } else if (this.matchesThisLevel) {
                // This filter was not matched before. So the fact that it matches now means we need to consume it,
                // therefore not delegating.

                return (this.next == null);

            }

        } else if (matchesPreviousOrCurrentLevel(markupLevel)) {
            // This filter was already matched by a previous level (through an "open" event), so just delegate to next.

            if (this.next != null) {
                return this.next.matchComment(blockMatching, markupLevel, markupBlockIndex);
            }
            return blockMatching;

        }

        return false;

    }
62615
public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.err.println("Usage: Parse -fun head_rules < train_parses");
      System.err.println("Reads training parses (one-sentence-per-line) and displays parse structure.");
      System.exit(1);
    }
    int ai=0;
    if (args[0].equals("-fun")) {
      Parse.useFunctionTags(true);
      ai++;
    }
    HeadRules rules = new ch.epfl.bbp.shaded.opennlp.tools.lang.english.HeadRules(args[ai]);
    java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(System.in));
    for (String line = in.readLine(); line != null; line = in.readLine()) {
      Parse p = Parse.parseParse(line);
      p.updateHeads(rules);
      p.show();
      //p.showCodeTree();
    }
  }
1342709
private boolean match(SyntacticChunk chunk, Element element, int baseTokenIndex, Sentence sentence) {
		boolean match;
		boolean negated;
		// Sees if the mask must or not match.
		// Negated is optional, so it can be null, true or false.
		// If null, consider as false.
		 if (element.isNegated() == null) {
			match = false;
			negated = false;
		} else {
			match = element.isNegated().booleanValue();
			negated = element.isNegated().booleanValue();
		}
		for (Mask mask : element.getMask()) {
			// If the token must match the mask.
			if (!negated) {
				// If not negated, match starts as false and just one match is needed to make it true.
				if (mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(chunk.toPlainText())) {
					match = true;
				} else if (mask.getTagMask() != null && chunk.getMorphologicalTag() != null) {
					match = match | (chunk.getMorphologicalTag().matchExact(mask.getTagMask(), false) && chunk.getSyntacticTag().match(mask.getTagMask()));
				} else if (mask.getPrimitiveMask() != null /*&& chunk.getTokens().size() > 0*/ && matchLemma(chunk.getChildChunks().get(0).getMainToken(), mask.getPrimitiveMask())) {
					match = true;
				} else if (mask.getTagReference() != null && chunk.getMorphologicalTag() != null) {
					TagMask t = RuleUtils.createTagMaskFromReferenceSyntatic(mask.getTagReference(), sentence, baseTokenIndex);
					match = match | (chunk.getMorphologicalTag().match(t, false) && (t.getSyntacticFunction() == null || chunk.getSyntacticTag().match(t)));
				} else if (mask.getOutOfBounds() != null
				     && chunk.getTokens().get(0).getLexeme().equals(OUT_OF_BOUNDS)) {
                    match = true;
                }
			} else { // The token must NOT match the mask.
				// If negated, match starts as true and just one match is needed to make it false.
				if ( mask.getLexemeMask() != null && mask.getLexemeMask().equalsIgnoreCase(chunk.toPlainText())) {
					match = false;
				} else if (mask.getTagMask() != null) {
					match = match & !(chunk.getMorphologicalTag().matchExact(mask.getTagMask(), false) && (mask.getTagMask().getSyntacticFunction() == null || chunk.getSyntacticTag().match(mask.getTagMask())));
				} else if (mask.getPrimitiveMask() != null /*&& chunk.getTokens().size() > 0*/ && matchLemma(chunk.getChildChunks().get(0).getMainToken(), mask.getPrimitiveMask())) {
                    match = false;
              }  else if (mask.getTagReference() != null) {
					TagMask t = RuleUtils.createTagMaskFromReferenceSyntatic(mask.getTagReference(), sentence, baseTokenIndex);
					match = match & !(chunk.getMorphologicalTag().match(t,false) && (t.getSyntacticFunction() == null || chunk.getSyntacticTag().match(t)));
			  } else if (mask.getOutOfBounds() != null && 
			        sentence.getSyntacticChunks().indexOf(chunk) == sentence.getSyntacticChunks().size() - 1) {
                    match = false;
              }
			}
		}
		return match;
	}
1223306
public int matches(Replaceable text,
                       int[] offset,
                       int limit,
                       boolean incremental) {
        int start = offset[0];
        int count = 0;
        while (count < maxCount) {
            int pos = offset[0];
            int m = matcher.matches(text, offset, limit, incremental);
            if (m == U_MATCH) {
                ++count;
                if (pos == offset[0]) {
                    // If offset has not moved we have a zero-width match.
                    // Don't keep matching it infinitely.
                    break;
                }
            } else if (incremental && m == U_PARTIAL_MATCH) {
                return U_PARTIAL_MATCH;
            } else {
                break;
            }
        }
        if (incremental && offset[0] == limit) {
            return U_PARTIAL_MATCH;
        }
        if (count >= minCount) {
            return U_MATCH;
        }
        offset[0] = start;
        return U_MISMATCH;
    }
1021771
public String tokenizeToText() {

    final StringBuilder sb = new StringBuilder();
    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token tok : tokSentence) {
          String tokenValue = tok.getTokenValue();
          sb.append(tokenValue.trim()).append(DELIMITER);
        }
        sb.append(LINE_BREAK);
      }
    } else {
      final String[] sentences = segmenter.segmentSentence();
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token token : tokSentence) {
          String tokenValue = token.getTokenValue();
          if (tokenValue.equals(RuleBasedSegmenter.PARAGRAPH)) {
            sb.append(DEFAULT_TOKEN_VALUE).append(LINE_BREAK);
          } else {
            sb.append(tokenValue.trim()).append(DELIMITER);
          }
        }
        sb.append(LINE_BREAK);
      }
    }
    return sb.toString().trim();
  }
237275
public Match find(String text) {

		if (regexp == null)
			return null;

		Matcher m = regexp.matcher(text);
		if (m.find()) {
			Match match = new Match();
			match.setText(text);
			match.match = m;
			match.start = m.start(0);
			match.end = m.end(0);
			match.line = text;
			return match;
		}
		else {
			return null;
		}
	}
459944
def get_token(self, text, start=0):
        
        best_class = best_match = None

        for token_class, match in self.matching_tokens(text):
            if best_match and best_match.end() >= match.end():
                continue
            best_match = match
            best_class = token_class

        return best_class, best_match
285176
@Override
  public List<List<Token>> tokenize(final String[] sentences) {
    final long start = System.nanoTime();
    int noTokens = 0;
    int prevIndex = 0;
    int curIndex = 0;
    final String language = this.lang;
    final List<List<Token>> result = new ArrayList<List<Token>>();
    // TODO improve this
    final String offsetText = this.originalText;
    for (final String sentence : sentences) {
      if (DEBUG) {
        System.err.println("-> Segmented:" + sentence);
      }
      final List<Token> tokens = new ArrayList<Token>();
      final String[] curTokens = getTokens(sentence);
      for (final String arrayToken : curTokens) {
        curIndex = offsetText.indexOf(arrayToken, prevIndex);
        // crap rule for corrected URLs
        if (curIndex == -1) {
          curIndex = prevIndex + 1;
        }
        final Token curToken = this.tokenFactory.createToken(arrayToken,
            curIndex, arrayToken.length());
        // exceptions to WFs
        addTokens(curToken, tokens);
        if (DEBUG) {
          System.err.println("-> Token:" + arrayToken + " curIndex: " + curIndex
              + " prev: " + prevIndex);
        }
        prevIndex = curIndex + curToken.tokenLength();
      }
      result.add(tokens);
      noTokens = noTokens + curTokens.length;
    }
    normalizeTokens(result, language);
    final long duration = System.nanoTime() - start;
    final double toksPerSecond = noTokens / (duration / 1000000000.0);
    if (DEBUG) {
      System.err.printf(
          "ixa-pipe-tok tokenized %d tokens at %.2f tokens per second.%n",
          noTokens, toksPerSecond);
    }
    return result;
  }
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
264035
private void matchEndLine() {
		// First, parse all whitespace characters except for new lines
		index = skipLineSpace(index);

		// Second, check whether we've reached the end-of-file (as signaled by
		// running out of tokens), or we've encountered some token which not a
		// newline.
		if (index >= tokens.size()) {
			return; // EOF
		} else if (tokens.get(index).kind != NewLine) {
			syntaxError("expected end-of-line", tokens.get(index));
		} else {
			index = index + 1;
		}
	}
531704
def clean_text_by_sentences(text, language="english", additional_stopwords=None):
    
    init_textcleanner(language, additional_stopwords)
    original_sentences = split_sentences(text)
    filtered_sentences = filter_words(original_sentences)

    return merge_syntactic_units(original_sentences, filtered_sentences)
964116
def contains_text(expected_text)
      assert_exists

      case expected_text
      when Regexp
        value() =~ expected_text
      when String
        value().index(expected_text)
      else
        raise TypeError, "expected String or Regexp, got #{expected_text.inspect}:#{expected_text.class}"
      end
    end
459347
function check($text)
    {
        $this->init();

        if (!$this->plink) {
            return array();
        }

        // tokenize
        $text = preg_split($this->separator, $text, NULL, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_OFFSET_CAPTURE);

        $diff       = 0;
        $matches    = array();

        foreach ($text as $w) {
            $word = trim($w[0]);
            $pos  = $w[1] - $diff;
            $len  = mb_strlen($word);

            // skip exceptions
            if ($this->dictionary->is_exception($word)) {
            }
            else if (!pspell_check($this->plink, $word)) {
                $suggestions = pspell_suggest($this->plink, $word);

                if (count($suggestions) > self::MAX_SUGGESTIONS) {
                    $suggestions = array_slice($suggestions, 0, self::MAX_SUGGESTIONS);
                }

                $matches[] = array($word, $pos, $len, null, $suggestions);
            }

            $diff += (strlen($word) - $len);
        }

        $this->matches = $matches;
        return $matches;
    }
1082043
public void tokenizeToKAF(final KAFDocument kaf) throws IOException {

    int noSents = 0;
    int noParas = 1;
    
    List<List<Token>> tokens;

    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      tokens = tokenizer.tokenize(sentences);
    } else {
      final String[] sentences = segmenter.segmentSentence();
      tokens = tokenizer.tokenize(sentences);
    }
    for (final List<Token> tokenizedSentence : tokens) {
      noSents = noSents + 1;
      for (final Token token : tokenizedSentence) {
        if (token.getTokenValue().equals(RuleBasedSegmenter.PARAGRAPH)) {
          ++noParas;
          // TODO debug this
          if (noSents < noParas) {
            ++noSents;
          }
        } else {
          final WF wf = kaf.newWF(token.startOffset(), token.getTokenValue(),
              noSents);
          wf.setLength(token.tokenLength());
          wf.setPara(noParas);
        }
      }
    }
  }
1191016
public static void main(String[] args) {
        Scanner scan = new Scanner(System.in);

        RegularExpression<String> regex = RegularExpressionParsers.word.parse(args[0]);
        System.out.println("regex: " + regex);
        System.out.println();

        while (scan.hasNextLine()) {
            String line = scan.nextLine();

            System.out.println("contains: " + regex.apply(Arrays.asList(line.split("\\s+"))));
            System.out.println("matches:  " + regex.matches(Arrays.asList(line.split("\\s+"))));
            System.out.println();
        }

        scan.close();
    }