分词
@param text 文本
@return 分词结果
326220
public static List<Term> segment(String text)
    {
        List<Term> termList = new LinkedList<Term>();
        Matcher matcher = WEB_URL.matcher(text);
        int begin = 0;
        int end;
        while (matcher.find())
        {
            end = matcher.start();
            termList.addAll(SEGMENT.seg(text.substring(begin, end)));
            termList.add(new Term(matcher.group(), Nature.xu));
            begin = matcher.end();
        }
        if (begin < text.length()) termList.addAll(SEGMENT.seg(text.substring(begin)));

        return termList;
    }
1107991
public static List<Word> getWords(String text) {
        List<Word> result = new ArrayList<>();
        List<Word> words = WordParser.parse(text);
        for (Word word : words) {
            result.add(word);
        }
        return result;
    }
668373
public static byte[] getBarsPostnet(String text) {
        int total = 0;
        for (int k = text.length() - 1; k >= 0; --k) {
            int n = text.charAt(k) - '0';
            total += n;
        }
        text += (char)(((10 - (total % 10)) % 10) + '0');
        byte bars[] = new byte[text.length() * 5 + 2];
        bars[0] = 1;
        bars[bars.length - 1] = 1;
        for (int k = 0; k < text.length(); ++k) {
            int c = text.charAt(k) - '0';
            System.arraycopy(BARS[c], 0, bars, k * 5 + 1, 5);
        }
        return bars;
    }
420996
protected static List<Token> baseTokenize(String text) {
    String[] words = text.split(" ");
    List<Token> tokens = new LinkedList<Token>();
    for (String word : words) {
      tokens.add(new Token(word));
    }
    return tokens;
  }
336134
public static boolean hasText(final String text) {
		if (text != null && !"".equals(text.trim())) {
			return true;
		}
		return false;
	}
1345583
@SuppressWarnings("MethodMayBeStatic")
    public NerResultSet process(final String _text) {
        // tokenization
        final List<List<Token>> tokens = TOKENIZER.process(_text);
        // extract name phrase:
        final List<List<Phrase>> phrases = new ArrayList<>();

        for (final List<Token> tokenList : tokens) {
            // for each sentence
            final List<Phrase> sentencePhrase = new ArrayList<>();
            int wordPtr = 0;
            int phraseStubStartPtr = 0;
            while (wordPtr < tokenList.size()) {

                // iterate through every words in the sentence
                final List<Token> currentPhrase = new ArrayList<>();

                // get the phrase stub
                while (true) {
                    // we merge the names combined with the word 'and'
                    // if the second phrase is much shorter than the first phrase
                    // example: Canberra Institution of Science and Technology
                    boolean mergeAndFlag = false;
                    if ("and".equalsIgnoreCase(tokenList.get(wordPtr).text)
                        && !currentPhrase.isEmpty()
                        && wordPtr + 1 < tokenList.size()
                        && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1)) {
                        if (wordPtr + currentPhrase.size() - 1 >= tokenList.size()) {
                            mergeAndFlag = true;
                        }
                        else {
                            for (int i = wordPtr + 1; i <= wordPtr + currentPhrase.size() - 1; i++) {
                                if (!detectNameWordInSentenceByPosition(tokenList, i)) {
                                    mergeAndFlag = true;
                                    break;
                                }
                            }
                        }
                    }

                    if (detectNameWordInSentenceByPosition(tokenList, wordPtr)
                        || (equalsIgnoreCase(tokenList.get(wordPtr).text, "of", "the", "'s")
                            && !currentPhrase.isEmpty()
                            && wordPtr + 1 < tokenList.size()
                            && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1))
                        || mergeAndFlag) {
                        if (currentPhrase.isEmpty()) phraseStubStartPtr = wordPtr; // record the index of the first word
                        currentPhrase.add(tokenList.get(wordPtr));
                        wordPtr++;
                        if (wordPtr == tokenList.size()) break;
                    }
                    else break;
                }

                //get the attached 'the' or 'a'
                int phrasePos = phraseStubStartPtr;
                int phraseLen = currentPhrase.size();

                // if phrasePos is zero, then it is the first word, there won't be any 'the' or 'a' attached to this word.
                if (phrasePos > 0) {
                    final Token attachedArticle = tokenList.get(phrasePos - 1);
                    if ("the".equalsIgnoreCase(attachedArticle.text) || "a".equalsIgnoreCase(attachedArticle.text)) {
                        phrasePos--;
                        phraseLen++;
                    }
                }

                //handling non-name single words such as: Monday, January, etc.
                if (currentPhrase.size() == 1 && phraseStubStartPtr == phrasePos) {
                    if (NON_NAME_WORDS.contains(currentPhrase.get(0).text)) {
                        currentPhrase.clear();
                        wordPtr--;
                    }
                }

                if (currentPhrase.isEmpty()) {
                    // if the current pointed word is not a name phrase
                    // ESTIMATE IF IT IS A TIME/DATE
                    final List<Token> currentDatePhrase = new ArrayList<>();
                    final DatePhraseModel dpm = new DatePhraseModel();
                    int tempPtr = wordPtr;
                    while (true) {
                        final Token word = tokenList.get(tempPtr);
                        final DateType type = getDateType(word.text);
                        if (type != NONE
                            || equalsIgnoreCase(word.text, "of", ",", "the")
                               && !currentDatePhrase.isEmpty()
                               && tempPtr + 1 != tokenList.size()
                               && getDateType(tokenList.get(tempPtr + 1).text) != NONE
                               && getDateType(tokenList.get(tempPtr - 1).text) != NONE) {
                            // is date word:
                            currentDatePhrase.add(word);
                            dpm.addType(type);
                            tempPtr++;
                            if (tempPtr == tokenList.size()) break;
                        }
                        else break;
                    }
                    if (dpm.isDate()) {
                        // add the phrase to phrase array
                        sentencePhrase.add(new Date(currentDatePhrase, wordPtr, currentDatePhrase.size(), wordPtr));
                        wordPtr = tempPtr;
                    }
                }
                else if (!(currentPhrase.size() == 1
                           && (isPastTense(currentPhrase.get(0).text) || isPlural(currentPhrase.get(0).text)))) {
                    sentencePhrase.add(new Name(currentPhrase, phrasePos, phraseLen, phraseStubStartPtr));
                }

                wordPtr++;
            }

            phrases.add(sentencePhrase);
        }

        // find attached words(preps):
        for (int si = 0; si < tokens.size(); si++) {
            final List<Token> sentenceToken = tokens.get(si);
            final List<Phrase> sentencePhrase = phrases.get(si);
            for (int pi = 0; pi < sentencePhrase.size(); pi++) {
                getAttachedPrep(sentenceToken, sentencePhrase, pi);
            }
        }

        return new NerResultSet(phrases, tokens);
    }
1369765
public CAS process(String text) {
        CAS cas = retrieve();

        cas.setDocumentText(text);
        try {
            analysisEngine.process(cas);
        } catch (AnalysisEngineProcessException e) {
            if (text != null && !text.isEmpty())
                return process(text);
            throw new RuntimeException(e);
        }

        return cas;


    }
519288
@Override
    public List<Queue> parse(String text) throws ParseException {
        final List<Queue> queues;
        if (StringUtils.isNotBlank(text)) {
            queues = new LinkedList<Queue>();
            String separator = "\n";
            if (text.indexOf("\r\n") > 0) {
                separator = "\r\n";
            }
            final String[] lines = text.split(separator);
            Queue queue = null;
            for (final String line : lines) {
                Matcher matcher = PATTERN_QUEUE.matcher(line);
                if (matcher.matches()) {
                    if (queue != null) {
                        queues.add(queue);
                    }
                    queue = new Queue();
                    final String name = matcher.group(1);
                    queue.setName(name);
                } else if (StringUtils.isNotBlank(line)) {
                    String[] temp = Utils.splitFirst(line, CHAR_EQUALS);
                    if (temp.length == 2) {
                        final String key = temp[0].trim().toLowerCase();
                        final String value = temp[1].trim();
                        if ("queue_type".equals(key)) {
                            queue.setQueueType(value);
                        } else if ("priority".equals(key)) {
                            try {
                                queue.setPriority(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing queue priority: " + nfe.getMessage(), nfe);
                                queue.setPriority(-1);
                            }
                        } else if ("total_jobs".equals(key)) {
                            try {
                                queue.setTotalJobs(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing queue total jobs: " + nfe.getMessage(), nfe);
                                queue.setPriority(-1);
                            }
                        } else if ("state_count".equals(key)) {
                            queue.setStateCount(value);
                        } else if ("mtime".equals(key)) {
                            queue.setMtime(value);
                        } else if ("max_user_run".equals(key)) {
                            try {
                                queue.setMaxUserRun(Integer.parseInt(value));
                            } catch (NumberFormatException nfe) {
                                LOGGER.log(Level.WARNING, "Failed parsing queue max user run: " + nfe.getMessage(),
                                        nfe);
                                queue.setPriority(-1);
                            }
                        } else if ("enabled".equals(key)) {
                            queue.setEnabled(Boolean.parseBoolean(value));
                        } else if ("started".equals(key)) {
                            queue.setStarted(Boolean.parseBoolean(value));
                        } else if (key.startsWith("resources_max.")) {
                            queue.getResourcesMax().put(key, value);
                        } else if (key.startsWith("resources_min.")) {
                            queue.getResourcesMin().put(key, value);
                        } else if (key.startsWith("resources_assigned.")) {
                            queue.getResourcesAssigned().put(key, value);
                        } else if (key.startsWith("resources_default.")) {
                            queue.getResourcesDefault().put(key, value);
                        } else {
                            LOGGER.info("Unmmaped key, value: " + key + ", " + value);
                        }
                    }
                }
            }
            if (queue != null) {
                queues.add(queue);
            }
            return queues;
        } else {
            return Collections.emptyList();
        }
    }
1011297
public static String[] simpleTokenize(String text) {
		if (GeneralUtils.isEmpty(text))
			return null;
		return text.split("[ ,.;:><^`�\\]\\[}{|��+_)(*&�%$#@!=\\-?�\\/\"]+", -1);
	}
808045
static String[] splitOnTokens(String text)
    {
        // used by wildcardMatch
        // package level so a unit test may run on this

        if (text.indexOf('?') == -1 && text.indexOf('*') == -1)
        {
            return new String[]
                    {
                        text
                    };
        }

        char[] array = text.toCharArray();
        ArrayList<String> list = new ArrayList<String>();
        StringBuilder buffer = new StringBuilder();
        for (int i = 0; i < array.length; i++)
        {
            if (array[i] == '?' || array[i] == '*')
            {
                if (buffer.length() != 0)
                {
                    list.add(buffer.toString());
                    buffer.setLength(0);
                }
                if (array[i] == '?')
                {
                    list.add("?");
                }
                else if (list.isEmpty()
                        || i > 0 && list.get(list.size() - 1).equals("*") == false)
                {
                    list.add("*");
                }
            }
            else
            {
                buffer.append(array[i]);
            }
        }
        if (buffer.length() != 0)
        {
            list.add(buffer.toString());
        }

        return list.toArray(new String[list.size()]);
    }
352587
public String senTokenize(String text){
		String ret = text;
		
		if (isTokenization){
			ret = PennTokenizer.tokenize(text);
		}
		
		return ret.trim();
	}
194499
public static int getLineCount(final String text) {
    final Matcher matcher = LINE_FINDER.matcher(text);
    int i = 0;

    while (matcher.find()) {
      i++;
    }

    return i;
  }
566517
public TextView getText(String text)
	{
		if(config.commandLogging){
			Log.d(config.commandLoggingTag, "getText(\""+text+"\")");
		}
		
		return getter.getView(TextView.class, text, false);
	}
1224981
public @Nonnull String text() {
        return new NodeListSpliterator(node.getChildNodes()).stream()
                        .filter(it -> it instanceof Text)
                        .map(it -> ((Text) it).getNodeValue())
                        .collect(joining());
    }
1183484
public static String[] extractKeywords(String text)
    {
        List<Term> termList = NotionalTokenizer.segment(text);
        String[] wordArray = new String[termList.size()];
        Iterator<Term> iterator = termList.iterator();
        for (int i = 0; i < wordArray.length; i++)
        {
            wordArray[i] = iterator.next().word;
        }
        return wordArray;
    }
477347
public long text(int group, String text) {
		return send(new Filter(false, String.valueOf(group)), null, "text", text);
	}
602327
public String convert(String text){
		String ret = text;
		
		if (cpsUni2Uni == null) return ret;
		
		Iterator<String> it = cpsUni2Uni.keySet().iterator();
		while(it.hasNext()){
			String cpsChar = it.next();
			ret = ret.replaceAll(cpsChar, cpsUni2Uni.get(cpsChar));
		}
		
		return ret;
		
	}