Returns a list of tokens that contains tokenized string.

@param input the string to tokenize
@param textTrim whether to trim text
@return a list of tokens
334656
public static List<Token> tokenize(CharSequence input, boolean textTrim) {
        if (input == null) {
            throw new IllegalArgumentException("input must not be null");
        }

        int inputLen = input.length();
        if (inputLen == 0) {
            List<Token> tokens = new ArrayList<>(1);
            tokens.add(new Token(""));
            return tokens;
        }

        int status = AT_TEXT;
        int tokenStartOffset = 0; // start position of token in the stringBuffer
        char symbol = Token.PARAMETER_SYMBOL; // PARAMETER_SYMBOL or ATTRIBUTE_SYMBOL
        char c;

        StringBuilder nameBuf = new StringBuilder();
        StringBuilder valueBuf = new StringBuilder();
        StringBuilder textBuf = new StringBuilder();

        List<Token> tokens = new ArrayList<>();

        for (int i = 0; i < inputLen; i++) {
            c = input.charAt(i);

            switch (status) {
            case AT_TEXT:
                textBuf.append(c);
                if (Token.isTokenSymbol(c)) {
                    symbol = c;
                    status = AT_TOKEN_SYMBOL;
                    // abc$ --> tokenStartOffset: 3
                    tokenStartOffset = textBuf.length() - 1;
                }
                break;
            case AT_TOKEN_SYMBOL:
                textBuf.append(c);
                if (c == Token.START_BRACKET) {
                    status = AT_TOKEN_NAME;
                } else {
                    if (Token.isTokenSymbol(c)) {
                        symbol = c;
                        status = AT_TOKEN_SYMBOL;
                        // abc$ --> tokenStartOffset: 3
                        tokenStartOffset = textBuf.length() - 1;
                    } else {
                        status = AT_TEXT;
                    }
                }
                break;
            case AT_TOKEN_NAME:
            case AT_TOKEN_VALUE:
                textBuf.append(c);
                if (status == AT_TOKEN_NAME) {
                    if (c == Token.VALUE_SEPARATOR) {
                        status = AT_TOKEN_VALUE;
                        break;
                    }
                }
                if (c == Token.END_BRACKET) {
                    if (nameBuf.length() > 0 || valueBuf.length() > 0) {
                        // save previous non-token string
                        if (tokenStartOffset > 0) {
                            String text = trimBuffer(textBuf, tokenStartOffset, textTrim);
                            Token token = new Token(text);
                            tokens.add(token);
                        }

                        // save token name and default value
                        Token token = createToken(symbol, nameBuf, valueBuf);
                        tokens.add(token);

                        status = AT_TEXT;
                        textBuf.setLength(0);
                        break;
                    }

                    status = AT_TEXT;
                    break;
                }
                if (status == AT_TOKEN_NAME) {
                    if (nameBuf.length() > MAX_TOKEN_NAME_LENGTH) {
                        status = AT_TEXT;
                        nameBuf.setLength(0);
                    }
                    nameBuf.append(c);
                } else {
                    valueBuf.append(c);
                }
                break;
            }
        }

        if (textBuf.length() > 0) {
            String text = trimBuffer(textBuf, textBuf.length(), textTrim);
            Token token = new Token(text);
            tokens.add(token);
        }

        return tokens;
    }
916002
public List<String> splitMatches(String text, int minSize) {
    TrieNode node = inner.root();
    List<String> matches = new ArrayList<>();
    String accumulator = "";
    for (int i = 0; i < text.length(); i++) {
      short prevDepth = node.getDepth();
      TrieNode prevNode = node;
      node = node.getContinuation(text.charAt(i));
      if (null == node) node = inner.root();
      if (!accumulator.isEmpty() && (node.getDepth() < prevDepth || (prevNode.hasChildren() && node.getDepth() == prevDepth))) {
        if (accumulator.length() > minSize) {
          matches.add(accumulator);
          node = ((Optional<TrieNode>) inner.root().getChild(text.charAt(i))).orElse(inner.root());
        }
        accumulator = "";
      }
      else if (!accumulator.isEmpty()) {
        accumulator += text.charAt(i);
      }
      else if (accumulator.isEmpty() && node.getDepth() > prevDepth) {
        accumulator = node.getString();
      }
    }
    List<String> tokenization = new ArrayList<>();
    for (String match : matches) {
      int index = text.indexOf(match);
      assert (index >= 0);
      if (index > 0) tokenization.add(text.substring(0, index));
      tokenization.add(text.substring(index, index + match.length()));
      text = text.substring(index + match.length());
    }
    tokenization.add(text);
    return tokenization;
  }
269682
def get_tokens(self, format_string):
        
        return [t for t in self.tokenize_pattern.split(format_string) if t]
1091115
public static String tokenizeSmileys(String text) {
        for(Map.Entry<String, String> smiley : SMILEYS_MAPPING.entrySet()) {
            text = text.replaceAll(smiley.getKey(), smiley.getValue());
        }
        return text;
    }
163482
public static List<String> wonderfulTokenizer(String tokenString, String[] delimiters) {
		List<String> tokens = new ArrayList<String>();
		String currentToken = "";
		for(int i = 0; i < tokenString.length(); i++) {
			String _currItem = String.valueOf(tokenString.charAt(i));
			if(_currItem.trim().isEmpty()) continue;

			String delimiterFound = "";
			for(String d: delimiters) {
				if(d.startsWith(_currItem)) {
					if(d.length() > 1) {
						if(d.equals(tokenString.substring(i, i + d.length()))) {
							delimiterFound = d;
							i = i + (d.length() - 1);
						}
					} else delimiterFound = d;
				}
				if(!delimiterFound.isEmpty()) break;
			}

			if(!delimiterFound.isEmpty()) {
				if(!currentToken.isEmpty()) {
					tokens.add(currentToken);
					currentToken = "";
				}
				tokens.add(delimiterFound);
			} else currentToken += _currItem;
		}
		if(!currentToken.isEmpty()) tokens.add(currentToken);

		return tokens;
	}
1242510
private static String[] tokenize(String string) {
		if (string.indexOf("?") == -1 && string.indexOf("*") == -1) {
			return new String[] { string };
		}

		char[] textArray = string.toCharArray();
		List<String> tokens = new ArrayList<String>();
		StringBuilder tokenBuilder = new StringBuilder();

		for (int i = 0; i < textArray.length; i++) {
			if (textArray[i] != '?' && textArray[i] != '*') {
				// collect non wild card characters
				tokenBuilder.append(textArray[i]);
				continue;
			}

			if (tokenBuilder.length() != 0) {
				tokens.add(tokenBuilder.toString());
				tokenBuilder.setLength(0);
			}
			if (textArray[i] == '?') {
				tokens.add("?");
			} else if (tokens.size() == 0 || (i > 0 && tokens.get(tokens.size() - 1).equals("*") == false)) {
				tokens.add("*");
			}
		}

		if (tokenBuilder.length() != 0) {
			tokens.add(tokenBuilder.toString());
		}

		return (String[]) tokens.toArray(new String[tokens.size()]);
	}
240268
@Override
	public List<Token> tokenize(String s) {
		List<Token> result = new ArrayList<Token>();
		result.add(new UndefinedToken(s));

		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
			for (Pattern pattern : patterns) {
				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
					Token token = it.next();
					if (token instanceof UndefinedToken) {
						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
								pattern);
						if (replacementTokens.size() > 1) {
							it.remove();
							for (Token newToken : replacementTokens) {
								it.add(newToken);
							}
						}
					}
				}
			}
		}

		return result;
	}
1065304
def tokenize(text, format=None):
    
    text = Text(text)
    text = text.replace("\t", " ")
    tokens = re.findall(patterns, text)
    tokens = [token[0] for token in tokens]
    if format == "text":
        return " ".join(tokens)
    else:
        return tokens
877807
public static String rtrimWildcardTokens(String input) {
        String[] tokens = tokenizePathAsArray(input);
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < tokens.length; i++) {
            if (hasWildcards(tokens[i])) {
                break;
            }
            if (i > 0 && sb.charAt(sb.length() - 1) != File.separatorChar) {
                sb.append(File.separatorChar);
            }
            sb.append(tokens[i]);
        }
        return sb.toString();
    }
408086
public static String[] split(String input, String regex) {
    if (input == null) {
      return null;
    }
    String[] arr = input.split(regex);
    List<String> results = new ArrayList<>(arr.length);
    for (String a : arr) {
      if (!a.trim().isEmpty()) {
        results.add(a);
      }
    }
    return results.toArray(new String[0]);
  }
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
302459
function tokenize(text) {
  const rawTokens = text.split(/[\s\/]+/mg);
  const tokens = [];
  rawTokens.forEach(token => {
    // Strip off unwanted trivial characters
    token = token
        .trim()
        .replace(/^[_\-"'`({[<$*)}\]>.]+/, '')
        .replace(/[_\-"'`({[<$*)}\]>.]+$/, '');
    // Ignore tokens that contain weird characters
    if (/^[\w.\-]+$/.test(token)) {
      tokens.push(token.toLowerCase());
      const ngTokenMatch = /^[nN]g([A-Z]\w*)/.exec(token);
      if (ngTokenMatch) {
        tokens.push(ngTokenMatch[1].toLowerCase());
      }
    }
  });
  return tokens;
}
666334
def _split_string_to_tokens(text):
  """"""
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in xrange(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret
452977
public static String[] asStringArray(List<String> input) {
		String[] result = new String[input.size()];

		for (int i = 0; i < result.length; i++) {
			result[i] = input.get(i);
		}

		return result;
	}
1033235
public static List<String> readStringList(InputStream input) throws IOException {
        return readStringList(input, Charset.defaultCharset());
    }
165486
public static String[] splitOptions(String input) {

        //Note that we use a regex matching all "|" characters not prefixed by "\"

        //Since we define a regex for matching, the input delimiter "|" needs to be escaped, as well as "\",
        //which is even double-escaped - one escaping is due to the String, one due to the regex.

        // emulate missing lookbehinds in JS regexes by first reversing the input,
        // then using a split with lookaheads, and finally reversing the parts resulting
        // from the split
        String reverse = reverse(input);
        String[] parts = reverse.split("\\|(?!\\\\)");
        String[] finalParts = new String[parts.length];
        int lastIndex = parts.length - 1;
        for (int i = 0; i < parts.length; i++) {
            finalParts[lastIndex - i] = reverse(parts[i]);
        }
        return finalParts;

    }
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}