Normalizes non-ambiguous quotes according to language and corpus.

@param sentence
the list of tokens
@param lang
the language
971731
public static void normalizeQuotes(final List<Token> sentence,
      final String lang) {

    for (final Token token : sentence) {
      if (lang.equalsIgnoreCase("en")) {
        token.setTokenValue(
            leftSingleQuote.matcher(token.getTokenValue()).replaceAll("`"));
        token.setTokenValue(
            rightSingleQuote.matcher(token.getTokenValue()).replaceAll("'"));
        token.setTokenValue(
            leftDoubleQuote.matcher(token.getTokenValue()).replaceAll("``"));
        token.setTokenValue(
            rightDoubleQuote.matcher(token.getTokenValue()).replaceAll("''"));
      } else if (lang.equalsIgnoreCase("ca") || lang.equalsIgnoreCase("de")
          || lang.equalsIgnoreCase("es") || lang.equalsIgnoreCase("eu")
          || lang.equalsIgnoreCase("fr") || lang.equalsIgnoreCase("gl")
          || lang.equalsIgnoreCase("it") || lang.equalsIgnoreCase("nl")
          || lang.equalsIgnoreCase("pt") || lang.equalsIgnoreCase("ru")) {
        token.setTokenValue(
            toAsciiSingleQuote.matcher(token.getTokenValue()).replaceAll("'"));
        token.setTokenValue(
            toAsciiDoubleQuote.matcher(token.getTokenValue()).replaceAll("\""));
      }
    }
  }
719263
@Override
  public final RuleMatch[] match(AnalyzedSentence sentence) {
    List<RuleMatch> ruleMatches = new ArrayList<>();
    AnalyzedTokenReadings[] tokens = sentence.getTokensWithoutWhitespace();
    boolean repetition = false;
    Set<String> inflectedWords = new TreeSet<>();
    String prevLemma;
    int curToken = 0;
    // start from real token, 0 = SENT_START
    for (int i = 1; i < tokens.length; i++) {
      String token = tokens[i].getToken();
      // avoid "..." etc. to be matched:
      boolean isWord = true;
      boolean hasLemma = true;

      if (token.length() < 2) {
        isWord = false;
      }

      for (AnalyzedToken analyzedToken : tokens[i]) {
        String posTag = analyzedToken.getPOSTag();
        if (posTag != null) {
          if (StringTools.isEmpty(posTag)) {
            isWord = false;
            break;
          }
          String lemma = analyzedToken.getLemma();
          if (lemma == null) {
            hasLemma = false;
            break;
          }

          if (getExcludedWordsPattern().contains(lemma)) {
            isWord = false;
            break;
          }

          Matcher m2 = getExcludedPos().matcher(posTag);
          if (m2.matches()) {
            isWord = false;
            break;
          }
        } else {
          hasLemma = false;
        }

      }

      Matcher m1 = getExcludedNonWordsPattern().matcher(tokens[i].getToken());
      if (isWord && m1.matches()) {
        isWord = false;
      }

      prevLemma = "";
      if (isWord) {
        boolean notSentEnd = false;
        for (AnalyzedToken analyzedToken : tokens[i]) {
          String pos = analyzedToken.getPOSTag();
          if (pos != null) {
            notSentEnd |= JLanguageTool.SENTENCE_END_TAGNAME.equals(pos);
          }
          if (hasLemma) {
            String curLemma = analyzedToken.getLemma();
            if (!prevLemma.equals(curLemma) && !notSentEnd) {
              if (inflectedWords.contains(curLemma) && curToken != i) {
                repetition = true;
              } else {
                inflectedWords.add(analyzedToken.getLemma());
                curToken = i;
              }
            }
            prevLemma = curLemma;
          } else {
            if (inflectedWords.contains(tokens[i].getToken()) && !notSentEnd) {
              repetition = true;
            } else {
              inflectedWords.add(tokens[i].getToken());
            }
          }
        }
      }

      if (repetition) {
        int pos = tokens[i].getStartPos();
        RuleMatch ruleMatch = new RuleMatch(this, sentence, pos, pos
            + token.length(), getMessage(), getShortMessage());
        ruleMatches.add(ruleMatch);
        repetition = false;
      }

    }
    return toRuleMatchArray(ruleMatches);
  }
1244580
public void setJavaLanguageLevel(final String languageLevel) {
        if ( Arrays.binarySearch( LANGUAGE_LEVELS,
                                  languageLevel ) < 0 ) {
            throw new RuntimeException( "value '" + languageLevel + "' is not a valid language level" );
        }
        this.languageLevel = languageLevel;
    }
344274
public void setTokenizerFactory(@NonNull TokenizerFactory tokenizerFactory) {
        this.tokenizerFactory = tokenizerFactory;

        if (sentenceIter != null) {
            SentenceTransformer transformer = new SentenceTransformer.Builder().iterator(sentenceIter)
                            .tokenizerFactory(this.tokenizerFactory).build();
            this.iterator = new AbstractSequenceIterator.Builder<>(transformer).build();
        }
    }
366215
def normalize_eols(text, eol='\n'):
    """"""
    for eol_char, _ in EOL_CHARS:
        if eol_char != eol:
            text = text.replace(eol_char, eol)
    return text
585359
public final AnalyzedSentence replace(AnalyzedSentence sentence) throws IOException {
    DisambiguationPatternRuleReplacer replacer = new DisambiguationPatternRuleReplacer(this);
    return replacer.replace(sentence);
  }
1327385
public boolean learn(Sentence sentence)
    {
        CharTable.normalize(sentence);
        if (!getPerceptronSegmenter().learn(sentence)) return false;
        if (posTagger != null && !getPerceptronPOSTagger().learn(sentence)) return false;
        if (neRecognizer != null && !getPerceptionNERecognizer().learn(sentence)) return false;
        return true;
    }
1102847
private void pushPiece(String sentence, String normalized, int start, int end, byte preType, List<String> wordList)
    {
        if (preType == CharType.CT_CHINESE)
        {
            segmenter.segment(sentence.substring(start, end), normalized.substring(start, end), wordList);
        }
        else
        {
            wordList.add(sentence.substring(start, end));
        }
    }
210247
@Override
  public List<String> tokenize(final String text) {
    final List<String> l = new ArrayList<>();
    final StringTokenizer st = new StringTokenizer(text, nlTokenizingChars, true);
    while (st.hasMoreElements()) {
      String token = st.nextToken();
      String origToken = token;
      if (token.length() > 1) {
        if (startsWithQuote(token) && endsWithQuote(token) && token.length() > 2) {
          l.add(token.substring(0, 1));
          l.add(token.substring(1, token.length()-1));
          l.add(token.substring(token.length()-1, token.length()));
        } else if (endsWithQuote(token)) {
          int cnt = 0;
          while (endsWithQuote(token)) {
            token = token.substring(0, token.length() - 1);
            cnt++;
          }
          l.add(token);
          for (int i = origToken.length() - cnt; i < origToken.length(); i++) {
            l.add(origToken.substring(i, i + 1));
          }
        } else if (startsWithQuote(token)) {
          while (startsWithQuote(token)) {
            l.add(token.substring(0, 1));
            token = token.substring(1, token.length());
          }
          l.add(token);
        } else {
          l.add(token);
        }
      } else {
        l.add(token);
      }
    }
    return joinEMailsAndUrls(l);
  }
399293
public void setText(Phrase phrase) {
        bidiLine = null;
        composite = false;
        compositeColumn = null;
        compositeElements = null;
        listIdx = 0;
        splittedRow = false;
        waitPhrase = phrase;
    }
358615
static List<GoogleToken> getGoogleTokens(String sentence, boolean addStartToken, Tokenizer wordTokenizer) {
    List<GoogleToken> result = new ArrayList<>();
    if (addStartToken) {
      result.add(new GoogleToken(LanguageModel.GOOGLE_SENTENCE_START, 0, 0));
    }
    List<String> tokens = wordTokenizer.tokenize(sentence);
    int startPos = 0;
    for (String token : tokens) {
      if (!StringTools.isWhitespace(token)) {
        result.add(new GoogleToken(token, startPos, startPos+token.length()));
      }
      startPos += token.length();
    }
    return result;
  }
882612
def enrich_sentences_with_NLP(self, all_sentences):
        
        if not self.has_NLP_support():
            raise NotImplementedError(
                f"Language {self.lang} not available in spacy beyond tokenization"
            )

        if len(all_sentences) == 0:
            return  # Nothing to parse

        if self.model.has_pipe("sentencizer"):
            self.model.remove_pipe("sentencizer")
            self.logger.debug(
                f"Removed sentencizer ('sentencizer') from model. "
                f"Now in pipeline: {self.model.pipe_names}"
            )

        if self.model.has_pipe("sentence_boundary_detector"):
            self.model.remove_pipe(name="sentence_boundary_detector")
        self.model.add_pipe(
            set_custom_boundary, before="parser", name="sentence_boundary_detector"
        )

        sentence_batches = self._split_sentences_by_char_limit(
            all_sentences, self.model.max_length
        )

        # TODO: We could do this in parallel. Test speedup in the future
        for sentence_batch in sentence_batches:
            custom_tokenizer = TokenPreservingTokenizer(self.model.vocab)
            # we circumvent redundant tokenization by using a custom
            # tokenizer that directly uses the already separated words
            # of each sentence as tokens
            doc = custom_tokenizer(sentence_batch)
            doc.user_data = sentence_batch
            for name, proc in self.model.pipeline:  # iterate over components in order
                doc = proc(doc)

            try:
                assert doc.is_parsed
            except Exception:
                self.logger.exception(f"{doc} was not parsed")

            for sent, current_sentence_obj in zip(doc.sents, sentence_batch):
                parts = defaultdict(list)

                for i, token in enumerate(sent):
                    parts["lemmas"].append(token.lemma_)
                    parts["pos_tags"].append(token.tag_)
                    parts["ner_tags"].append(
                        token.ent_type_ if token.ent_type_ else "O"
                    )
                    head_idx = (
                        0 if token.head is token else token.head.i - sent[0].i + 1
                    )
                    parts["dep_parents"].append(head_idx)
                    parts["dep_labels"].append(token.dep_)
                current_sentence_obj.pos_tags = parts["pos_tags"]
                current_sentence_obj.lemmas = parts["lemmas"]
                current_sentence_obj.ner_tags = parts["ner_tags"]
                current_sentence_obj.dep_parents = parts["dep_parents"]
                current_sentence_obj.dep_labels = parts["dep_labels"]
                yield current_sentence_obj
209856
private static Locale createConstant(String lang, String country) {
        BaseLocale base = BaseLocale.createInstance(lang, country);
        return getInstance(base, null);
    }
273510
public static Pair<String, MultiDimensionalMap<Integer, Integer, String>> stringWithLabels(String sentence,
                                                       TokenizerFactory tokenizerFactory) {
        MultiDimensionalMap<Integer, Integer, String> map = MultiDimensionalMap.newHashBackedMap();
        Tokenizer t = tokenizerFactory.create(sentence);
        List<String> currTokens = new ArrayList<>();
        String currLabel = null;
        String endLabel = null;
        List<Pair<String, List<String>>> tokensWithSameLabel = new ArrayList<>();
        while (t.hasMoreTokens()) {
            String token = t.nextToken();
            if (token.matches(BEGIN_LABEL)) {
                currLabel = token;

                //no labels; add these as NONE and begin the new label
                if (!currTokens.isEmpty()) {
                    tokensWithSameLabel.add(new Pair<>("NONE", (List<String>) new ArrayList<>(currTokens)));
                    currTokens.clear();

                }

            } else if (token.matches(END_LABEL)) {
                if (currLabel == null)
                    throw new IllegalStateException("Found an ending label with no matching begin label");
                endLabel = token;
            } else
                currTokens.add(token);

            if (currLabel != null && endLabel != null) {
                currLabel = currLabel.replaceAll("[<>/]", "");
                endLabel = endLabel.replaceAll("[<>/]", "");
                Preconditions.checkState(!currLabel.isEmpty(), "Current label is empty!");
                Preconditions.checkState(!endLabel.isEmpty(), "End label is empty!");
                Preconditions.checkState(currLabel.equals(endLabel), "Current label begin and end did not match for the parse. Was: %s ending with %s",
                        currLabel, endLabel);

                tokensWithSameLabel.add(new Pair<>(currLabel, (List<String>) new ArrayList<>(currTokens)));
                currTokens.clear();


                //clear out the tokens
                currLabel = null;
                endLabel = null;
            }


        }

        //no labels; add these as NONE and begin the new label
        if (!currTokens.isEmpty()) {
            tokensWithSameLabel.add(new Pair<>("none", (List<String>) new ArrayList<>(currTokens)));
            currTokens.clear();

        }

        //now join the output
        StringBuilder strippedSentence = new StringBuilder();
        for (Pair<String, List<String>> tokensWithLabel : tokensWithSameLabel) {
            String joinedSentence = StringUtils.join(tokensWithLabel.getSecond(), " ");
            //spaces between separate parts of the sentence
            if (!(strippedSentence.length() < 1))
                strippedSentence.append(" ");
            strippedSentence.append(joinedSentence);
            int begin = strippedSentence.toString().indexOf(joinedSentence);
            int end = begin + joinedSentence.length();
            map.put(begin, end, tokensWithLabel.getFirst());
        }


        return new Pair<>(strippedSentence.toString(), map);
    }
371182
public void setLang(java.lang.String lang) {
        getStateHelper().put(PropertyKeys.lang, lang);
        handleAttribute("lang", lang);
    }
23146
private void addEvents(String sentence) {
    String[] parts = sentence.split(" ");
    String outcome = NameFinderME.OTHER;
    List toks = new ArrayList();
    List outcomes = new ArrayList();
    for (int pi = 0, pl = parts.length; pi < pl; pi++) {
      if (parts[pi].equals("<START>")) {
        outcome = NameFinderME.START;
      }
      else if (parts[pi].equals("<END>")) {
        outcome = NameFinderME.OTHER;
      }
      else { //regular token
        toks.add(parts[pi]);
        outcomes.add(outcome);
        if (outcome.equals(NameFinderME.START)) {
          outcome = NameFinderME.CONTINUE;
        }
      }
    }
    events = new Event[toks.size()];
    for (int ti = 0, tl = toks.size(); ti < tl; ti++) {
      events[ti] = new Event((String) outcomes.get(ti), cg.getContext(ti, toks, outcomes, prevTags));
    }
    for (int ti=0,tl=toks.size();ti<tl;ti++) {
      prevTags.put(toks.get(ti),outcomes.get(ti));
    }
  }
1116852
def tokenize(self,
                 sentence,
                 normalize=True,
                 is_feature=False,
                 is_surface=False,
                 return_list=False,
                 func_normalizer=text_preprocess.normalize_text):
        # type: (text_preprocess, bool, bool, bool, bool, Callable[[str], text_type])->Union[List[text_type], TokenizedSenetence]
        
        assert isinstance(normalize, bool)
        assert isinstance(sentence, text_type)
        normalized_sentence = func_normalizer(sentence)
        result = self.call_juman_interface(normalized_sentence)

        token_objects = [
            self.__extract_morphological_information(
                mrph_object=morph_object,
                is_surface=is_surface,
                is_feature=is_feature
            )
            for morph_object in result]

        if return_list:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects
            )
            return tokenized_objects.convert_list_object()
        else:
            tokenized_objects = TokenizedSenetence(
                sentence=sentence,
                tokenized_objects=token_objects)

            return tokenized_objects