Convert the given string into tokens.

@param text the text
@param tokenize whether to tokenize
@return the token[]
498052
public static Token[] makeTokens(String text, boolean tokenize) {
        if (text == null) {
            return null;
        }
        Token[] tokens;
        if (tokenize) {
            tokens = parse(text);
        } else {
            tokens = new Token[1];
            tokens[0] = new Token(text);
        }
        return tokens;
    }
47183
@Override
    public INDArray transform(String text) {
        Tokenizer tokenizer = tokenizerFactory.create(text);
        List<String> tokens = tokenizer.getTokens();

        // build document words count
        return transform(tokens);
    }
670427
def word_tokenize(self, text: str) -> List[str]:
        
        return word_tokenize(text, custom_dict=self.__trie_dict, engine=self.__engine)
821706
def _whitespace_tokenize(self, text):
        """"""
        text = text.strip()
        tokens = text.split()
        return tokens
269682
def get_tokens(self, format_string):
        
        return [t for t in self.tokenize_pattern.split(format_string) if t]
1345583
@SuppressWarnings("MethodMayBeStatic")
    public NerResultSet process(final String _text) {
        // tokenization
        final List<List<Token>> tokens = TOKENIZER.process(_text);
        // extract name phrase:
        final List<List<Phrase>> phrases = new ArrayList<>();

        for (final List<Token> tokenList : tokens) {
            // for each sentence
            final List<Phrase> sentencePhrase = new ArrayList<>();
            int wordPtr = 0;
            int phraseStubStartPtr = 0;
            while (wordPtr < tokenList.size()) {

                // iterate through every words in the sentence
                final List<Token> currentPhrase = new ArrayList<>();

                // get the phrase stub
                while (true) {
                    // we merge the names combined with the word 'and'
                    // if the second phrase is much shorter than the first phrase
                    // example: Canberra Institution of Science and Technology
                    boolean mergeAndFlag = false;
                    if ("and".equalsIgnoreCase(tokenList.get(wordPtr).text)
                        && !currentPhrase.isEmpty()
                        && wordPtr + 1 < tokenList.size()
                        && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1)) {
                        if (wordPtr + currentPhrase.size() - 1 >= tokenList.size()) {
                            mergeAndFlag = true;
                        }
                        else {
                            for (int i = wordPtr + 1; i <= wordPtr + currentPhrase.size() - 1; i++) {
                                if (!detectNameWordInSentenceByPosition(tokenList, i)) {
                                    mergeAndFlag = true;
                                    break;
                                }
                            }
                        }
                    }

                    if (detectNameWordInSentenceByPosition(tokenList, wordPtr)
                        || (equalsIgnoreCase(tokenList.get(wordPtr).text, "of", "the", "'s")
                            && !currentPhrase.isEmpty()
                            && wordPtr + 1 < tokenList.size()
                            && detectNameWordInSentenceByPosition(tokenList, wordPtr + 1))
                        || mergeAndFlag) {
                        if (currentPhrase.isEmpty()) phraseStubStartPtr = wordPtr; // record the index of the first word
                        currentPhrase.add(tokenList.get(wordPtr));
                        wordPtr++;
                        if (wordPtr == tokenList.size()) break;
                    }
                    else break;
                }

                //get the attached 'the' or 'a'
                int phrasePos = phraseStubStartPtr;
                int phraseLen = currentPhrase.size();

                // if phrasePos is zero, then it is the first word, there won't be any 'the' or 'a' attached to this word.
                if (phrasePos > 0) {
                    final Token attachedArticle = tokenList.get(phrasePos - 1);
                    if ("the".equalsIgnoreCase(attachedArticle.text) || "a".equalsIgnoreCase(attachedArticle.text)) {
                        phrasePos--;
                        phraseLen++;
                    }
                }

                //handling non-name single words such as: Monday, January, etc.
                if (currentPhrase.size() == 1 && phraseStubStartPtr == phrasePos) {
                    if (NON_NAME_WORDS.contains(currentPhrase.get(0).text)) {
                        currentPhrase.clear();
                        wordPtr--;
                    }
                }

                if (currentPhrase.isEmpty()) {
                    // if the current pointed word is not a name phrase
                    // ESTIMATE IF IT IS A TIME/DATE
                    final List<Token> currentDatePhrase = new ArrayList<>();
                    final DatePhraseModel dpm = new DatePhraseModel();
                    int tempPtr = wordPtr;
                    while (true) {
                        final Token word = tokenList.get(tempPtr);
                        final DateType type = getDateType(word.text);
                        if (type != NONE
                            || equalsIgnoreCase(word.text, "of", ",", "the")
                               && !currentDatePhrase.isEmpty()
                               && tempPtr + 1 != tokenList.size()
                               && getDateType(tokenList.get(tempPtr + 1).text) != NONE
                               && getDateType(tokenList.get(tempPtr - 1).text) != NONE) {
                            // is date word:
                            currentDatePhrase.add(word);
                            dpm.addType(type);
                            tempPtr++;
                            if (tempPtr == tokenList.size()) break;
                        }
                        else break;
                    }
                    if (dpm.isDate()) {
                        // add the phrase to phrase array
                        sentencePhrase.add(new Date(currentDatePhrase, wordPtr, currentDatePhrase.size(), wordPtr));
                        wordPtr = tempPtr;
                    }
                }
                else if (!(currentPhrase.size() == 1
                           && (isPastTense(currentPhrase.get(0).text) || isPlural(currentPhrase.get(0).text)))) {
                    sentencePhrase.add(new Name(currentPhrase, phrasePos, phraseLen, phraseStubStartPtr));
                }

                wordPtr++;
            }

            phrases.add(sentencePhrase);
        }

        // find attached words(preps):
        for (int si = 0; si < tokens.size(); si++) {
            final List<Token> sentenceToken = tokens.get(si);
            final List<Phrase> sentencePhrase = phrases.get(si);
            for (int pi = 0; pi < sentencePhrase.size(); pi++) {
                getAttachedPrep(sentenceToken, sentencePhrase, pi);
            }
        }

        return new NerResultSet(phrases, tokens);
    }
1186964
public List<Expression<E>> tokenize(String string) {
        List<Expression<E>> expressions = new ArrayList<Expression<E>>();

        final Pattern whitespacePattern = Pattern.compile("\\s+");
        final Pattern unaryPattern = Pattern.compile("[*?+]");
        final Pattern minMaxPattern = Pattern.compile("\\{(\\d+),(\\d+)\\}");
        final Pattern binaryPattern = Pattern.compile("[|]");

        List<String> tokens = new ArrayList<String>();

        char stack = ' ';
        int start = 0;
        while (start < string.length()) {
            Matcher matcher;

            // skip whitespace
            if ((matcher = whitespacePattern.matcher(string))
                .region(start, string.length()).lookingAt()) {
                start = matcher.end();
                continue;
            }

            char c = string.charAt(start);
            // group, assertion, or token
            if (c == '(' || c == '<' || c == '[' || c == '$' || c == '^') {
                // group
                if (string.charAt(start) == '(') {
                    int end = indexOfClose(string, start, '(', ')');
                    if (end == -1) {
                        throw new TokenizationRegexException("unclosed parenthesis: " + start
                                + ":\"" + string.substring(start) + ")\"");
                    }

                    String group = string.substring(start + 1, end);
                    start = end + 1;

                    final Pattern namedPattern = Pattern.compile("<(\\w*)>:(.*)");
                    final Pattern unnamedPattern = Pattern.compile("\\?:(.*)");

                    // named group (matching)
                    if ((matcher = namedPattern.matcher(group)).matches()) {
                        String groupName = matcher.group(1);
                        group = matcher.group(2);
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.NamedGroup<E>(groupName, groupExpressions));
                    }
                    // unnamed group
                    else if ((matcher = unnamedPattern.matcher(group)).matches()) {
                        group = matcher.group(1);
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.NonMatchingGroup<E>(groupExpressions));
                    }
                    // group (matching)
                    else {
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.MatchingGroup<E>(groupExpressions));
                    }
                }

                // token
                else if (c == '<' || c == '[') {
                    String token = readToken(string.substring(start));
                    try {
                        // strip off enclosing characters
                        String tokenInside = token.substring(1, token.length() - 1);
                        BaseExpression<E> base = factory(tokenInside);
                        expressions.add(base);

                        start += token.length();
                    }
                    catch (Exception e) {
                        throw new TokenizationRegexException("error parsing token: " + token, e);
                    }
                }

                // assertion (^)
                else if (c == '^') {
                    expressions.add(new StartAssertion<E>());
                    start += 1;
                }

                // assertion ($)
                else if (c == '$') {
                    expressions.add(new EndAssertion<E>());
                    start += 1;
                }

                // check if we have a floating OR operator
                if (stack == '|') {
                    try {
                        stack = ' ';
                        if (expressions.size() < 2) {
                            throw new IllegalStateException(
                                    "OR operator is applied to fewer than 2 elements.");
                        }

                        Expression<E> expr1 = expressions.remove(expressions.size() - 1);
                        Expression<E> expr2 = expressions.remove(expressions.size() - 1);
                        expressions.add(new Expression.Or<E>(expr1, expr2));
                    }
                    catch (Exception e) {
                        throw new TokenizationRegexException("error parsing OR (|) operator.", e);
                    }
                }
            }
            // unary operator
            else if ((matcher = unaryPattern.matcher(string))
                     .region(start, string.length()).lookingAt()) {
                char operator = matcher.group(0).charAt(0);

                // pop the last expression
                Expression<E> base = expressions.remove(expressions.size() - 1);

                // add the operator to it
                Expression<E> expr;
                if (operator == '?') {
                    expr = new Expression.Option<E>(base);
                } else if (operator == '*') {
                    expr = new Expression.Star<E>(base);
                } else if (operator == '+') {
                    expr = new Expression.Plus<E>(base);
                }
                else {
                    throw new IllegalStateException();
                }

                expressions.add(expr);

                start = matcher.end();
            }
            // min/max operator "{x,y}"
            else if ((matcher = minMaxPattern.matcher(string))
                    .region(start, string.length()).lookingAt()) {
                int minOccurrences = Integer.parseInt(matcher.group(1));
                int maxOccurrences = Integer.parseInt(matcher.group(2));

                // pop the last expression and add operator
                Expression<E> base = expressions.remove(expressions.size() - 1);
                Expression<E> expr = new Expression.MinMax<E>(base, minOccurrences, maxOccurrences);

                expressions.add(expr);

                start = matcher.end();
            }
            // binary operator (alternation)
            else if ((matcher = binaryPattern.matcher(string))
                     .region(start, string.length()).lookingAt()) {
                tokens.add(matcher.group(0));
                stack = '|';
                start = matcher.end();
            }
            else {
                throw new TokenizationRegexException("unknown symbol: "
                        + string.substring(start));
            }
        }

        if (stack == '|') {
            throw new TokenizationRegexException("OR remains on the stack.");
        }

        return expressions;
    }
418737
def wordify(text):
    
    stopset = set(nltk.corpus.stopwords.words('english'))
    tokens = nltk.WordPunctTokenizer().tokenize(text)
    return [w for w in tokens if w not in stopset]
1352541
public static String getStringFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String tok : tokens) {
      sb.append(tok).append(" ");
    }
    return sb.toString().trim();
  }
738174
public INDArray convert(String text) {
        Tokenizer tokenizer = tokenizerFactory.create(text);
        List<String> tokens = tokenizer.getTokens();
        INDArray create = Nd4j.create(1,wordIndexMap.size());
        Counter<String> tokenizedCounter = new Counter<>();

        for(int i = 0; i < tokens.size(); i++) {
            tokenizedCounter.incrementCount(tokens.get(i),1.0);
        }

        for(int i = 0; i < tokens.size(); i++) {
            if(wordIndexMap.containsKey(tokens.get(i))) {
                int idx = wordIndexMap.get(tokens.get(i));
                int count = (int) tokenizedCounter.getCount(tokens.get(i));
                double weight = tfidfWord(tokens.get(i),count,tokens.size());
                create.putScalar(idx,weight);
            }
        }

        return create;
    }
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
16752
def span_tokenize_sents(self, strings):
        
        raise NotImplementedError("span_tokenizer and span_tokenzie_sents not yet implemented. ;)")
        for s in strings:
            yield list(self.span_tokenize(s))
686182
public CoreLabel makeToken(String tokenText, int begin, int length) {
    return makeToken(tokenText, tokenText, begin, length);
  }
414410
public List<String> extractWords(@NotNull String text, @NotNull Language language) {
        Timer timer = MetricsFactory.getMetrics().getTimer(MetricsType.TEXT_UTILS_TOKENIZE_TEXT.name());

        if (StringUtils.isEmpty(text)) {
            timer.stop();
            return Collections.emptyList();
        }

        List<String> strings = splitText(text, language);

        timer.stop();

        return strings;
    }
1021771
public String tokenizeToText() {

    final StringBuilder sb = new StringBuilder();
    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token tok : tokSentence) {
          String tokenValue = tok.getTokenValue();
          sb.append(tokenValue.trim()).append(DELIMITER);
        }
        sb.append(LINE_BREAK);
      }
    } else {
      final String[] sentences = segmenter.segmentSentence();
      final List<List<Token>> tokens = tokenizer.tokenize(sentences);
      for (final List<Token> tokSentence : tokens) {
        for (final Token token : tokSentence) {
          String tokenValue = token.getTokenValue();
          if (tokenValue.equals(RuleBasedSegmenter.PARAGRAPH)) {
            sb.append(DEFAULT_TOKEN_VALUE).append(LINE_BREAK);
          } else {
            sb.append(tokenValue.trim()).append(DELIMITER);
          }
        }
        sb.append(LINE_BREAK);
      }
    }
    return sb.toString().trim();
  }
707147
function tokenize (parser, text) {
  var textLines = text.split(LINEBREAKE_MARK)

  if (textLines[textLines.length - 1] === '') {
    textLines = textLines.splice(0, textLines.length - 1)
  }

  var rootNode = parser.createParentNode('Root')

  /**
   * Callback Function
   * @param element
   * @param index
   * @param array
   */
  function tokenizeByLine (element, index, array) {
    parser.line = index + 1
    parser.column = 1

    var paragraphNode = parser.createParentNode('Paragraph')

    // 空行ではない場合
    if (element !== '') {
      // 半角括弧を全角括弧に変換
      var str = new Jaco(element).toWideJapneseSymbol().toString()

      // 改行ノードをParagraphNodeに追加する
      parser.add(parser.createTextNode('Text', str), paragraphNode)
    }

    // 改行ノードをParagraphNodeに追加する
    parser.add(parser.createTextNode('WhiteSpace', '\n'), paragraphNode)

    // ParagraphNodeをRoodNodeに追加
    parser.add(paragraphNode, rootNode)
  }

  textLines.forEach(tokenizeByLine)

  return rootNode
}