It takes an array of tokens and outputs a string with tokens joined by a
whitespace.

@param array
of tokens
@return string representing one sentence for each array
323913
private String getSentenceFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String token : tokens) {
      sb.append(token).append(" ");
    }
    final String sentence = sb.toString();
    return sentence;
  }
1327913
public static String join(String[] array) {
    return array == null ? null : join(asList(array));
  }
60439
public final Span[] seqToSpans(final String[] tokens) {
    final Span[] annotatedText = this.sequenceLabeler.tag(tokens);
    final List<Span> probSpans = new ArrayList<Span>(
        Arrays.asList(annotatedText));
    return probSpans.toArray(new Span[probSpans.size()]);
  }
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
181588
private String stringJoin(String[] array, int count){
    String joined = "";
    for(int i = 0; i < count; i++)
      joined += array[i];
    return joined;
  }
903387
public static String jsonJoin(String[] array) {
        int arraySize = array.length;
        int bufSize = arraySize * (array[0].length() + 3);
        StringBuilder buf = new StringBuilder(bufSize);
        for (int i = 0; i < arraySize; i++) {
            if (i > 0) {
                buf.append(',');
            }

            buf.append('"');
            buf.append(array[i]);
            buf.append('"');
        }
        return buf.toString();
    }
1371034
public static String join(String[] strings) {
        StringBuilder builder = new StringBuilder();
        for (int i = 0; i < strings.length; i++) {
            builder.append(strings[i]);
            if (i + 1 < strings.length) builder.append(",");
        }
        return builder.toString();
    }
801227
protected static function _tokenize(array $tokens) {
		$inString = false;
		$results = array();
		$cpt = 0;
		$previousTokenId = null;
		foreach ($tokens as $tokenId => $token) {
			if ($token['content'] === '"') {
				if (!$inString) {
					$token['id'] = T_START_DOUBLE_QUOTE;
					$token['name'] = 'T_START_DOUBLE_QUOTE';
				} else {
					$token['id'] = T_END_DOUBLE_QUOTE;
					$token['name'] = 'T_END_DOUBLE_QUOTE';
				}
				$inString = !$inString;
			}

			$isCurlyBrace = (
				$token['content'] === '$' &&
				isset($tokens[$tokenId + 1]) &&
				$tokens[$tokenId + 1]['content'] === '{'
			);
			if ($isCurlyBrace) {
				$token['id'] = T_DOLLAR_CURLY_BRACES;
				$token['name'] = 'T_DOLLAR_CURLY_BRACES';
			}

			$isArray = (
				!$token['id'] &&
				$token['content'] === '(' &&
				$previousTokenId !== null &&
				$tokens[$previousTokenId]['id'] === T_ARRAY
			);
			if ($isArray) {
				$token['id'] = T_ARRAY_OPEN;
				$token['name'] = 'T_ARRAY_OPEN';
			}

			$isShortArray = (
				!$token['id'] &&
				$token['content'] === '[' &&
				$previousTokenId !== null && (
					$tokens[$previousTokenId]['id'] !== T_VARIABLE &&
					$tokens[$previousTokenId]['id'] !== T_ENCAPSED_AND_WHITESPACE
				)
			);
			if ($isShortArray) {
				$token['id'] = T_SHORT_ARRAY_OPEN;
				$token['name'] = 'T_SHORT_ARRAY_OPEN';
			}

			if ($token['content'] === '(' && !$token['id']) {
				$token['id'] = T_START_BRACKET;
				$token['name'] = 'T_START_BRACKET';
			}

			if ($token['id'] !== T_WHITESPACE) {
				$previousTokenId = $tokenId;
			}

			if ($token['id'] === T_IF) {
				$i = $cpt;
				$spaces = '';
				while ($i-- > 0 && $results[$i]['id'] === T_WHITESPACE) {
					$spaces = $results[$i]['content'] . $spaces;
				}
				if ($i >= 0 && $results[$i]['id'] === T_ELSE) {
					$token['id'] = T_ELSEIF;
					$token['name'] = 'T_ELSEIF';
					$token['content'] = $results[$i]['content'] . $spaces . $token['content'];
					$cpt = $i;
				}
			}

			$results[$cpt++] = $token;
		}

		return $results;
	}
833768
function markSpacesAndComments(tokens) {
  var tokensLength = tokens.length;
  var spaces = [-1, -1];
  var type; // Current token's type

  // For every token in the token list, mark spaces and line breaks
  // as spaces (set both `ws` and `sc` flags). Mark multiline comments
  // with `sc` flag.
  // If there are several spaces or tabs or line breaks or multiline
  // comments in a row, group them: take the last one's index number
  // and save it to the first token in the group as a reference:
  // e.g., `ws_last = 7` for a group of whitespaces or `sc_last = 9`
  // for a group of whitespaces and comments.
  for (var i = 0; i < tokensLength; i++) {
    type = tokens[i].type;

    if (type === TokenType.Space ||
        type === TokenType.Tab ||
        type === TokenType.Newline) {
      markSpace(tokens, i, spaces);
    } else if (type === TokenType.CommentML) {
      markComment(tokens, i, spaces);
    } else {
      markEndOfSpacesAndComments(tokens, i, spaces);
    }
  }

  markEndOfSpacesAndComments(tokens, i, spaces);
}
847506
public String format(String[] words, String[] target) {
		StringBuilder sb = new StringBuilder();
		for(int j=0;j<words.length;j++){
			sb.append(words[j]);
			if(Chars.isWhiteSpace(words[j]))//空格不输出词性
				continue;
			sb.append("/");
			sb.append(target[j]);
			if(j<words.length-1)
				sb.append(delim);
		} 
		String res = sb.toString();
		return res;
	}
1202654
public static String join(final Object[] array, final String delimiter) {
        
        if(array == null || array.length == 0) {
            return "";
        }
        
        String value = Arrays.stream(array)
                .map(v -> v.toString())
                .collect(Collectors.joining(defaultString(delimiter)));
        
        return value;
    }
1099778
private String[] parseStringArray(final String list) {
		StringTokenizer tokenizer = new StringTokenizer(list, ",", false);
		int length = tokenizer.countTokens();
		String[] arr = new String[length];

		for (int i = 0; tokenizer.hasMoreElements(); i++) {
			arr[i] = cleanSpaces(tokenizer.nextToken());
		}

		return arr;
	}
1373305
public final Span[] multiWordsToSpans(final String[] tokens) {
    final List<Span> multiWordsFound = new LinkedList<Span>();

    for (int offsetFrom = 0; offsetFrom < tokens.length; offsetFrom++) {
      Span multiwordFound = null;
      String tokensSearching[] = new String[] {};

      for (int offsetTo = offsetFrom; offsetTo < tokens.length; offsetTo++) {

        final int lengthSearching = offsetTo - offsetFrom + 1;
        if (lengthSearching > getMaxTokenCount()) {
          break;
        } else {
          tokensSearching = new String[lengthSearching];
          System.arraycopy(tokens, offsetFrom, tokensSearching, 0,
              lengthSearching);

          final String entryForSearch = StringUtils
              .getStringFromTokens(tokensSearching);
          final String entryValue = dictionary
              .get(entryForSearch.toLowerCase());
          if (entryValue != null) {
            multiwordFound = new Span(offsetFrom, offsetTo + 1, entryValue);
          }
        }
      }
      if (multiwordFound != null) {
        multiWordsFound.add(multiwordFound);
        offsetFrom += multiwordFound.length() - 1;
      }
    }
    return multiWordsFound.toArray(new Span[multiWordsFound.size()]);
  }
1024967
public List<Tokenizer.Token> tokenize(String sentence) {
      Sequence seq = Sequence.create(Chars.asList(sentence.toCharArray())
                                          .stream()
                                          .map(Object::toString));
      Labeling labels = model.label(seq);

      int n = sentence.length();
      int[][] spans = new int[n + 1][2];
      double[] best = new double[n + 1];
      best[0] = 1d;

      for (int i = 1; i <= n; i++) {
         for (int j = Math.max(0, i - maxSpanSize); j < i; j++) {
            int w = i - j;
            String span = sentence.substring(j, j + w);
            double score = scoreWord(j, j + w, span, labels);
            double segmentScore = multi(best[i - w], score);
            logFiner("'{0}' > '{1}' : {2} : {3}",
                     sentence.substring(spans[i - w][0], spans[i - w][1]),
                     span,
                     segmentScore,
                     best[i]);
            if (segmentScore >= best[i]) {
               best[i] = segmentScore;
               spans[i][0] = j;
               spans[i][1] = j + w;
            }
         }
      }
      int i = n;
      LinkedList<Tokenizer.Token> tokens = new LinkedList<>();
      while (i > 0) {
         String token = StringUtils.trim(sentence.substring(spans[i][0], spans[i][1]));
         if (!StringUtils.isNullOrBlank(token)) {
            tokens.addFirst(
               new Tokenizer.Token(token, TokenType.UNKNOWN, spans[i][0], spans[i][0] + token.length(), 0));
         }
         i = i - (spans[i][1] - spans[i][0]);
      }
      i = 0;
      for (Tokenizer.Token token : tokens) {
         token.index = i;
         i++;
      }

      return tokens;
   }
922912
public Span[] tokenizePos(String d) {
    Span[] tokens = split(d);
    newTokens.clear();
    tokProbs.clear();
    for (int i = 0, il = tokens.length; i < il; i++) {
      Span s = tokens[i];
      String tok = d.substring(s.getStart(), s.getEnd());
      // Can't tokenize single characters
      if (tok.length() < 2) {
        newTokens.add(s);
        tokProbs.add(ONE);
      }
      else if (useAlphaNumericOptimization() && alphaNumeric.matcher(tok).matches()) {
        newTokens.add(s);
        tokProbs.add(ONE);
      }
      else {
        int start = s.getStart();
        int end = s.getEnd();
        final int origStart = s.getStart();
        double tokenProb = 1.0;
        for (int j = origStart + 1; j < end; j++) {
          double[] probs =
            model.eval(cg.getContext(new ObjectIntPair(tok, j - origStart)));
          String best = model.getBestOutcome(probs);
          //System.err.println("TokenizerME: "+tok.substring(0,j-origStart)+"^"+tok.substring(j-origStart)+" "+best+" "+probs[model.getIndex(best)]);
          tokenProb *= probs[model.getIndex(best)];
          if (best.equals(TokContextGenerator.SPLIT)) {
            newTokens.add(new Span(start, j));
            tokProbs.add(new Double(tokenProb));
            start = j;
            tokenProb = 1.0;
          }
        }
        newTokens.add(new Span(start, end));
        tokProbs.add(new Double(tokenProb));
      }
    }

    Span[] spans = new Span[newTokens.size()];
    newTokens.toArray(spans);
    return spans;
  }
1352541
public static String getStringFromTokens(final String[] tokens) {
    final StringBuilder sb = new StringBuilder();
    for (final String tok : tokens) {
      sb.append(tok).append(" ");
    }
    return sb.toString().trim();
  }
1367547
public static String join(Object[] objects, String separator) {
        String result;

        if (objects.length > 0) {
            result = objects[0].toString();
            for (int i = 1; i < objects.length; i++) {
                result += separator + objects[i].toString();
            }
        } else {
            result = "";
        }

        return result;
    }