Constructs a list of Span objects, one for each whitespace
delimited token. Token strings can be constructed form these
spans as follows: d.substring(span.getStart(),span.getEnd());
@param d string to tokenize.
@return List is spans.
495495
public static Span[] split(String d) {
    int tokStart = -1;
    List tokens = new ArrayList();
    boolean inTok = false;

    //gather up potential tokens
    int end = d.length();
    for (int i = 0; i < end; i++) {
      if (Character.isWhitespace(d.charAt(i))) {
        if (inTok) {
          tokens.add(new Span(tokStart, i));
          inTok = false;
          tokStart = -1;
        }
      }
      else {
        if (!inTok) {
          tokStart = i;
          inTok = true;
        }
      }
    }
    if (inTok) {
      tokens.add(new Span(tokStart, end));
    }
    return (Span[]) tokens.toArray(new Span[tokens.size()]);
  }
1033430
private String [] split (final String string, final String delim)
  {
    final List <String> result = new ArrayList <> ();

    final StringTokenizer stringTokenizer = new StringTokenizer (string, delim);
    while (stringTokenizer.hasMoreTokens ())
    {
      result.add (stringTokenizer.nextToken ());
    }

    return result.toArray (new String [result.size ()]);
  }
1158046
public static String[] getTypesFromSpans(final Span[] spans,
      final String[] tokens) {
    final List<String> tagsList = new ArrayList<String>();
    for (final Span span : spans) {
      tagsList.add(span.getType());
    }
    return tagsList.toArray(new String[tagsList.size()]);
  }
17470
public Token[] tokenize(String input) 
    {
        List tokens = new ArrayList();
        int cursor = 0;
        while (cursor<input.length()) {
	    char ch = input.charAt(cursor);
	    if (Character.isWhitespace(ch)) {
                cursor++;
	    } else if (Character.isLetter(ch)) {
                StringBuffer buf = new StringBuffer("");
                while (cursor<input.length() && Character.isLetter(input.charAt(cursor))) {
                    buf.append(input.charAt(cursor));
                    cursor++;
                }
                tokens.add(internSomething(buf.toString()));
	    } else if (Character.isDigit(ch)) {
                StringBuffer buf = new StringBuffer("");
                while (cursor<input.length() && Character.isDigit(input.charAt(cursor))) {
                    buf.append(input.charAt(cursor));
                    cursor++;
                }
                tokens.add(internSomething(buf.toString()));
	    } else {
                if (!ignorePunctuation) {
                    StringBuffer buf = new StringBuffer("");
                    buf.append(ch);
                    String str = buf.toString();
                    tokens.add(internSomething(str));
                }
                cursor++;
	    }
        }
        return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
    }
276293
public List<T> tokenize() {
    // System.out.println("tokenize called");
    List<T> result = new ArrayList<T>();
    while (hasNext()) {
      result.add(next());
    }
    return result;
  }
743699
@Nonnull
    public static String[] tokenize(@Nonnull String s, @CheckForNull String delimiter) {
        return QuotedStringTokenizer.tokenize(s,delimiter);
    }
62159
public static <T> String join(String delimiter, Collection<T> strings) {
        StringBuilder builder = new StringBuilder();
        boolean first = true;
        for (T o : strings) {
            if (first) {
                first = false;
            } else {
                builder.append(delimiter);
            }
            builder.append(Objects.toString(o));
        }
        return builder.toString();
    }
1080193
private static List<String> split(String str) {
        List<String> result = new ArrayList<String>();
        int genericCount = 0;
        int startPos = 0;
        for (int i = 0; i < str.length(); i++) {
            if (str.charAt(i) == ',' && genericCount == 0) {
                result.add(str.substring(startPos, i).trim());
                startPos = i + 1;
            } else if (str.charAt(i) == '<') {
                genericCount++;
            } else if (str.charAt(i) == '>') {
                genericCount--;
            }
        }
        result.add(str.substring(startPos).trim());
        return result;
    }
1125139
public static String[] split(FacesContext context, String value, char... separators) {

        if (LangUtils.isValueBlank(value)) {
            return null;
        }

        List<String> tokens = new ArrayList<>(5);
        StringBuilder buffer = SharedStringBuilder.get(context, SHARED_SPLIT_BUFFER_KEY);

        int parenthesesCounter = 0;

        int length = value.length();
        for (int i = 0; i < length; i++) {
            char c = value.charAt(i);
            if (c == '(') {
                parenthesesCounter++;
            }

            if (c == ')') {
                parenthesesCounter--;
            }

            if (parenthesesCounter == 0) {
                boolean isSeparator = false;
                for (char separator : separators) {
                    if (c == separator) {
                        isSeparator = true;
                    }
                }

                if (isSeparator) {
                    // lets add token inside buffer to our tokens
                    tokens.add(buffer.toString());
                    // now we need to clear buffer
                    buffer.delete(0, buffer.length());
                }
                else {
                    buffer.append(c);
                }
            }
            else {
                buffer.append(c);
            }
        }

        // lets not forget about part after the separator
        tokens.add(buffer.toString());

        return tokens.toArray(new String[tokens.size()]);
    }
163482
public static List<String> wonderfulTokenizer(String tokenString, String[] delimiters) {
		List<String> tokens = new ArrayList<String>();
		String currentToken = "";
		for(int i = 0; i < tokenString.length(); i++) {
			String _currItem = String.valueOf(tokenString.charAt(i));
			if(_currItem.trim().isEmpty()) continue;

			String delimiterFound = "";
			for(String d: delimiters) {
				if(d.startsWith(_currItem)) {
					if(d.length() > 1) {
						if(d.equals(tokenString.substring(i, i + d.length()))) {
							delimiterFound = d;
							i = i + (d.length() - 1);
						}
					} else delimiterFound = d;
				}
				if(!delimiterFound.isEmpty()) break;
			}

			if(!delimiterFound.isEmpty()) {
				if(!currentToken.isEmpty()) {
					tokens.add(currentToken);
					currentToken = "";
				}
				tokens.add(delimiterFound);
			} else currentToken += _currItem;
		}
		if(!currentToken.isEmpty()) tokens.add(currentToken);

		return tokens;
	}
1305248
private synchronized int spanWithStrings(CharSequence s, int start, int spanLimit,
            SpanCondition spanCondition) {
        // Consider strings; they may overlap with the span.
        int initSize = 0;
        if (spanCondition == SpanCondition.CONTAINED) {
            // Use offset list to try all possibilities.
            initSize = maxLength16;
        }
        offsets.setMaxLength(initSize);
        int length = s.length();
        int pos = spanLimit, rest = length - spanLimit;
        int spanLength = spanLimit - start;
        int i, stringsLength = strings.size();
        for (;;) {
            if (spanCondition == SpanCondition.CONTAINED) {
                for (i = 0; i < stringsLength; ++i) {
                    int overlap = spanLengths[i];
                    if (overlap == ALL_CP_CONTAINED) {
                        continue; // Irrelevant string.
                    }
                    String string = strings.get(i);

                    int length16 = string.length();

                    // Try to match this string at pos-overlap..pos.
                    if (overlap >= LONG_SPAN) {
                        overlap = length16;
                        // While contained: No point matching fully inside the code point span.
                        overlap = string.offsetByCodePoints(overlap, -1); // Length of the string minus the last code
                                                                          // point.
                    }
                    if (overlap > spanLength) {
                        overlap = spanLength;
                    }
                    int inc = length16 - overlap; // Keep overlap+inc==length16.
                    for (;;) {
                        if (inc > rest) {
                            break;
                        }
                        // Try to match if the increment is not listed already.
                        if (!offsets.containsOffset(inc) && matches16CPB(s, pos - overlap, length, string, length16)) {
                            if (inc == rest) {
                                return length; // Reached the end of the string.
                            }
                            offsets.addOffset(inc);
                        }
                        if (overlap == 0) {
                            break;
                        }
                        --overlap;
                        ++inc;
                    }
                }
            } else /* SIMPLE */{
                int maxInc = 0, maxOverlap = 0;
                for (i = 0; i < stringsLength; ++i) {
                    int overlap = spanLengths[i];
                    // For longest match, we do need to try to match even an all-contained string
                    // to find the match from the earliest start.

                    String string = strings.get(i);

                    int length16 = string.length();

                    // Try to match this string at pos-overlap..pos.
                    if (overlap >= LONG_SPAN) {
                        overlap = length16;
                        // Longest match: Need to match fully inside the code point span
                        // to find the match from the earliest start.
                    }
                    if (overlap > spanLength) {
                        overlap = spanLength;
                    }
                    int inc = length16 - overlap; // Keep overlap+inc==length16.
                    for (;;) {
                        if (inc > rest || overlap < maxOverlap) {
                            break;
                        }
                        // Try to match if the string is longer or starts earlier.
                        if ((overlap > maxOverlap || /* redundant overlap==maxOverlap && */inc > maxInc)
                                && matches16CPB(s, pos - overlap, length, string, length16)) {
                            maxInc = inc; // Longest match from earliest start.
                            maxOverlap = overlap;
                            break;
                        }
                        --overlap;
                        ++inc;
                    }
                }

                if (maxInc != 0 || maxOverlap != 0) {
                    // Longest-match algorithm, and there was a string match.
                    // Simply continue after it.
                    pos += maxInc;
                    rest -= maxInc;
                    if (rest == 0) {
                        return length; // Reached the end of the string.
                    }
                    spanLength = 0; // Match strings from after a string match.
                    continue;
                }
            }
            // Finished trying to match all strings at pos.

            if (spanLength != 0 || pos == 0) {
                // The position is after an unlimited code point span (spanLength!=0),
                // not after a string match.
                // The only position where spanLength==0 after a span is pos==0.
                // Otherwise, an unlimited code point span is only tried again when no
                // strings match, and if such a non-initial span fails we stop.
                if (offsets.isEmpty()) {
                    return pos; // No strings matched after a span.
                }
                // Match strings from after the next string match.
            } else {
                // The position is after a string match (or a single code point).
                if (offsets.isEmpty()) {
                    // No more strings matched after a previous string match.
                    // Try another code point span from after the last string match.
                    spanLimit = spanSet.span(s, pos, SpanCondition.CONTAINED);
                    spanLength = spanLimit - pos;
                    if (spanLength == rest || // Reached the end of the string, or
                            spanLength == 0 // neither strings nor span progressed.
                    ) {
                        return spanLimit;
                    }
                    pos += spanLength;
                    rest -= spanLength;
                    continue; // spanLength>0: Match strings from after a span.
                } else {
                    // Try to match only one code point from after a string match if some
                    // string matched beyond it, so that we try all possible positions
                    // and don't overshoot.
                    spanLength = spanOne(spanSet, s, pos, rest);
                    if (spanLength > 0) {
                        if (spanLength == rest) {
                            return length; // Reached the end of the string.
                        }
                        // Match strings after this code point.
                        // There cannot be any increments below it because UnicodeSet strings
                        // contain multiple code points.
                        pos += spanLength;
                        rest -= spanLength;
                        offsets.shift(spanLength);
                        spanLength = 0;
                        continue; // Match strings from after a single code point.
                    }
                    // Match strings from after the next string match.
                }
            }
            int minOffset = offsets.popMinimum(null);
            pos += minOffset;
            rest -= minOffset;
            spanLength = 0; // Match strings from after a string match.
        }
    }
681082
public final TextBuilder p(double d) {
        if (null != __buffer) __append(d);
        else __caller.p(d);
        return this;
    }
378794
@Nonnull
    public static String join(@Nonnull Iterable<String> elements, @Nonnull String delimiter) {
        StringBuilder result = new StringBuilder();
        Iterator<String> it = elements.iterator();
        while (it.hasNext()) {
            String element = it.next();
            result.append(element);
            if (it.hasNext()) {
                result.append(delimiter);
            }
        }
        return result.toString();
    }
1373305
public final Span[] multiWordsToSpans(final String[] tokens) {
    final List<Span> multiWordsFound = new LinkedList<Span>();

    for (int offsetFrom = 0; offsetFrom < tokens.length; offsetFrom++) {
      Span multiwordFound = null;
      String tokensSearching[] = new String[] {};

      for (int offsetTo = offsetFrom; offsetTo < tokens.length; offsetTo++) {

        final int lengthSearching = offsetTo - offsetFrom + 1;
        if (lengthSearching > getMaxTokenCount()) {
          break;
        } else {
          tokensSearching = new String[lengthSearching];
          System.arraycopy(tokens, offsetFrom, tokensSearching, 0,
              lengthSearching);

          final String entryForSearch = StringUtils
              .getStringFromTokens(tokensSearching);
          final String entryValue = dictionary
              .get(entryForSearch.toLowerCase());
          if (entryValue != null) {
            multiwordFound = new Span(offsetFrom, offsetTo + 1, entryValue);
          }
        }
      }
      if (multiwordFound != null) {
        multiWordsFound.add(multiwordFound);
        offsetFrom += multiwordFound.length() - 1;
      }
    }
    return multiWordsFound.toArray(new Span[multiWordsFound.size()]);
  }
409989
protected <T extends TokenBase> List<T> createTokenList(String text) {

        if (!split) {
            return createTokenList(0, text);
        }

        List<Integer> splitPositions = getSplitPositions(text);

        if (splitPositions.isEmpty()) {
            return createTokenList(0, text);
        }

        ArrayList<T> result = new ArrayList<>();

        int offset = 0;

        for (int position : splitPositions) {
            result.addAll(this.<T>createTokenList(offset, text.substring(offset, position + 1)));
            offset = position + 1;
        }

        if (offset < text.length()) {
            result.addAll(this.<T>createTokenList(offset, text.substring(offset)));
        }

        return result;
    }
74193
public static String join ( CharSequence delimiter, CharSequence... elements ) {
        StringBuilder sb = new StringBuilder();
        for ( CharSequence element : elements ) {
            if ( sb.length() > 0 ) {
                if ( delimiter != null ) {
                    sb.append(delimiter);
                }
                else {
                    sb.append("null");
                }
            }
            sb.append(element);
        }
        return sb.toString();
    }
1186964
public List<Expression<E>> tokenize(String string) {
        List<Expression<E>> expressions = new ArrayList<Expression<E>>();

        final Pattern whitespacePattern = Pattern.compile("\\s+");
        final Pattern unaryPattern = Pattern.compile("[*?+]");
        final Pattern minMaxPattern = Pattern.compile("\\{(\\d+),(\\d+)\\}");
        final Pattern binaryPattern = Pattern.compile("[|]");

        List<String> tokens = new ArrayList<String>();

        char stack = ' ';
        int start = 0;
        while (start < string.length()) {
            Matcher matcher;

            // skip whitespace
            if ((matcher = whitespacePattern.matcher(string))
                .region(start, string.length()).lookingAt()) {
                start = matcher.end();
                continue;
            }

            char c = string.charAt(start);
            // group, assertion, or token
            if (c == '(' || c == '<' || c == '[' || c == '$' || c == '^') {
                // group
                if (string.charAt(start) == '(') {
                    int end = indexOfClose(string, start, '(', ')');
                    if (end == -1) {
                        throw new TokenizationRegexException("unclosed parenthesis: " + start
                                + ":\"" + string.substring(start) + ")\"");
                    }

                    String group = string.substring(start + 1, end);
                    start = end + 1;

                    final Pattern namedPattern = Pattern.compile("<(\\w*)>:(.*)");
                    final Pattern unnamedPattern = Pattern.compile("\\?:(.*)");

                    // named group (matching)
                    if ((matcher = namedPattern.matcher(group)).matches()) {
                        String groupName = matcher.group(1);
                        group = matcher.group(2);
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.NamedGroup<E>(groupName, groupExpressions));
                    }
                    // unnamed group
                    else if ((matcher = unnamedPattern.matcher(group)).matches()) {
                        group = matcher.group(1);
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.NonMatchingGroup<E>(groupExpressions));
                    }
                    // group (matching)
                    else {
                        List<Expression<E>> groupExpressions = this.tokenize(group);
                        expressions.add(new Expression.MatchingGroup<E>(groupExpressions));
                    }
                }

                // token
                else if (c == '<' || c == '[') {
                    String token = readToken(string.substring(start));
                    try {
                        // strip off enclosing characters
                        String tokenInside = token.substring(1, token.length() - 1);
                        BaseExpression<E> base = factory(tokenInside);
                        expressions.add(base);

                        start += token.length();
                    }
                    catch (Exception e) {
                        throw new TokenizationRegexException("error parsing token: " + token, e);
                    }
                }

                // assertion (^)
                else if (c == '^') {
                    expressions.add(new StartAssertion<E>());
                    start += 1;
                }

                // assertion ($)
                else if (c == '$') {
                    expressions.add(new EndAssertion<E>());
                    start += 1;
                }

                // check if we have a floating OR operator
                if (stack == '|') {
                    try {
                        stack = ' ';
                        if (expressions.size() < 2) {
                            throw new IllegalStateException(
                                    "OR operator is applied to fewer than 2 elements.");
                        }

                        Expression<E> expr1 = expressions.remove(expressions.size() - 1);
                        Expression<E> expr2 = expressions.remove(expressions.size() - 1);
                        expressions.add(new Expression.Or<E>(expr1, expr2));
                    }
                    catch (Exception e) {
                        throw new TokenizationRegexException("error parsing OR (|) operator.", e);
                    }
                }
            }
            // unary operator
            else if ((matcher = unaryPattern.matcher(string))
                     .region(start, string.length()).lookingAt()) {
                char operator = matcher.group(0).charAt(0);

                // pop the last expression
                Expression<E> base = expressions.remove(expressions.size() - 1);

                // add the operator to it
                Expression<E> expr;
                if (operator == '?') {
                    expr = new Expression.Option<E>(base);
                } else if (operator == '*') {
                    expr = new Expression.Star<E>(base);
                } else if (operator == '+') {
                    expr = new Expression.Plus<E>(base);
                }
                else {
                    throw new IllegalStateException();
                }

                expressions.add(expr);

                start = matcher.end();
            }
            // min/max operator "{x,y}"
            else if ((matcher = minMaxPattern.matcher(string))
                    .region(start, string.length()).lookingAt()) {
                int minOccurrences = Integer.parseInt(matcher.group(1));
                int maxOccurrences = Integer.parseInt(matcher.group(2));

                // pop the last expression and add operator
                Expression<E> base = expressions.remove(expressions.size() - 1);
                Expression<E> expr = new Expression.MinMax<E>(base, minOccurrences, maxOccurrences);

                expressions.add(expr);

                start = matcher.end();
            }
            // binary operator (alternation)
            else if ((matcher = binaryPattern.matcher(string))
                     .region(start, string.length()).lookingAt()) {
                tokens.add(matcher.group(0));
                stack = '|';
                start = matcher.end();
            }
            else {
                throw new TokenizationRegexException("unknown symbol: "
                        + string.substring(start));
            }
        }

        if (stack == '|') {
            throw new TokenizationRegexException("OR remains on the stack.");
        }

        return expressions;
    }