Index fulltext char trie.

@param documents the documents
@param maxLevels the max levels
@param minWeight the min weight
@return the char trie
963259
public static CharTrie indexFulltext(Collection<String> documents, int maxLevels, int minWeight) {
    return create(documents, maxLevels, minWeight, false);
  }
846200
public static <V> LinkedList<ResultTerm<V>> segmentReverseOrder(final char[] charArray, AhoCorasickDoubleArrayTrie<V> trie)
    {
        LinkedList<ResultTerm<V>> termList = new LinkedList<ResultTerm<V>>();
        final ResultTerm<V>[] wordNet = new ResultTerm[charArray.length + 1];
        trie.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit<V>()
        {
            @Override
            public void hit(int begin, int end, V value)
            {
                if (wordNet[end] == null || wordNet[end].word.length() < end - begin)
                {
                    wordNet[end] = new ResultTerm<V>(new String(charArray, begin, end - begin), value, begin);
                }
            }
        });
        for (int i = charArray.length; i > 0;)
        {
            if (wordNet[i] == null)
            {
                StringBuilder sbTerm = new StringBuilder();
                int offset = i - 1;
                byte preCharType = CharType.get(charArray[offset]);
                while (i > 0 && wordNet[i] == null && CharType.get(charArray[i - 1]) == preCharType)
                {
                    sbTerm.append(charArray[i - 1]);
                    preCharType = CharType.get(charArray[i - 1]);
                    --i;
                }
                termList.addFirst(new ResultTerm<V>(sbTerm.reverse().toString(), null, offset));
            }
            else
            {
                termList.addFirst(wordNet[i]);
                i -= wordNet[i].word.length();
            }
        }
        return termList;
    }
506266
public Integer[][] textsToSequences(String[] texts) {
        Integer oovTokenIndex  = wordIndex.get(outOfVocabularyToken);
        String[] wordSequence;
        ArrayList<Integer[]> sequences = new ArrayList<>();
        for (String text: texts) {
            if (charLevel) {
                if (lower) {
                    text = text.toLowerCase();
                }
                wordSequence = text.split("");
            } else {
                wordSequence = textToWordSequence(text, filters, lower, split);
            }
            ArrayList<Integer> indexVector = new ArrayList<>();
            for (String word: wordSequence) {
                if (wordIndex.containsKey(word)) {
                    int index = wordIndex.get(word);
                    if (numWords != null && index >= numWords) {
                        if (oovTokenIndex != null)
                            indexVector.add(oovTokenIndex);
                    } else {
                        indexVector.add(index);
                    }
                } else if (oovTokenIndex != null) {
                    indexVector.add(oovTokenIndex);
                }
            }
            Integer[] indices = indexVector.toArray(new Integer[indexVector.size()]);
            sequences.add(indices);
        }
        return sequences.toArray(new Integer[sequences.size()][]);
    }
767384
public static String getTextContent(Document document, boolean individualTokens) {
        String textContent = null;
        if (individualTokens) {
            List<String> tokens = getTextTokens(document);
            textContent = StringUtils.join(tokens, ",");
        } else {
            textContent =
                    document.getDocumentElement().getTextContent().trim().replaceAll("\\s+", ",");
        }
        return textContent;
    }
1256882
@Override
    public void setAsText(final String text) {
        if (BeanUtils.isNull(text)){
            setValue(null);
            return;
        }
        String[] tokens = super.tokenize(text);
        char[] theValue = new char[tokens.length];

        for (int index = 0; index < tokens.length; index++) {
            if (tokens[index].length() != 1)
                throw new IllegalArgumentException("Too many (" + tokens[index].length() + ") characters: '" + tokens[index]
                        + "'");
            theValue[index] = tokens[index].charAt(0);
        }
        setValue(theValue);
    }
996503
public static Pattern atMost(final int max, final CharPredicate predicate) {
    Checks.checkMax(max);
    return new Pattern() {
      @Override
      public int match(CharSequence src, int begin, int end) {
        return matchSome(max, predicate, src, end, begin, 0);
      }
    };
  }
169478
public void addAll(Iterable<? extends DOCUMENT> documents) {
      if (documents != null) {
         for (DOCUMENT doc : documents) {
            add(doc);
         }
      }
   }
1302406
public static <T extends Levenshtein> T weightedLevenshtein(String baseTarget, CharacterSubstitution characterSubstitution) {
		return weightedLevenshtein(baseTarget, null, characterSubstitution);
	}
67401
public Collection<String> nearestLabels(LabelledDocument document, int topN) {
        if (document.getReferencedContent() != null) {
            return nearestLabels(document.getReferencedContent(), topN);
        } else
            return nearestLabels(document.getContent(), topN);
    }
762106
public static int validateGeohashMaxLevels(Integer userMaxLevels, int defaultMaxLevels) {
        int maxLevels = userMaxLevels == null ? defaultMaxLevels : userMaxLevels;
        if (maxLevels < 1 || maxLevels > GeohashPrefixTree.getMaxLevelsPossible()) {
            throw new IndexException("max_levels must be in range [1, {}], but found {}",
                                     GeohashPrefixTree.getMaxLevelsPossible(),
                                     maxLevels);
        }
        return maxLevels;
    }
915256
public void setDocuments(java.util.Collection<DocumentMetadata> documents) {
        if (documents == null) {
            this.documents = null;
            return;
        }

        this.documents = new java.util.ArrayList<DocumentMetadata>(documents);
    }
965032
public static String join(CharSequence delimiter, Iterable tokens) {
        StringBuilder sb = new StringBuilder();
        boolean firstTime = true;
        for (Object token: tokens) {
            if (firstTime) {
                firstTime = false;
            } else {
                sb.append(delimiter);
            }
            sb.append(token);
        }
        return sb.toString();
    }
1055066
protected void adjustIndexMap(Doc[] elements) {
        for (int i = 0; i < elements.length; i++) {
            if (shouldAddToIndexMap(elements[i])) {
                String name = elements[i].name();
                char ch = (name.length()==0)?
                    '*' :
                    Character.toUpperCase(name.charAt(0));
                Character unicode = new Character(ch);
                List<Doc> list = indexmap.get(unicode);
                if (list == null) {
                    list = new ArrayList<Doc>();
                    indexmap.put(unicode, list);
                }
                list.add(elements[i]);
            }
        }
    }
618434
public ArrayList<Pair<String, V>> commonPrefixSearch(String key, int offset, int maxResults)
    {
        byte[] keyBytes = key.getBytes(utf8);
        List<Pair<Integer, Integer>> pairList = commonPrefixSearch(keyBytes, offset, maxResults);
        ArrayList<Pair<String, V>> resultList = new ArrayList<Pair<String, V>>(pairList.size());
        for (Pair<Integer, Integer> pair : pairList)
        {
            resultList.add(new Pair<String, V>(new String(keyBytes, 0, pair.first), valueArray[pair.second]));
        }
        return resultList;
    }
266913
@Override
    public INDArray getWordVectors(@NonNull Collection<String> labels) {
        int indexes[] = new int[labels.size()];
        int cnt = 0;
        boolean useIndexUnknown = useUnknown && vocab.containsWord(getUNK());

        for (String label : labels) {
            if (vocab.containsWord(label)) {
                indexes[cnt] = vocab.indexOf(label);
            } else
                indexes[cnt] = useIndexUnknown ? vocab.indexOf(getUNK()) : -1;
            cnt++;
        }

        while (ArrayUtils.contains(indexes, -1)) {
            indexes = ArrayUtils.removeElement(indexes, -1);
        }
        if (indexes.length == 0) {
                return Nd4j.empty(((InMemoryLookupTable)lookupTable).getSyn0().dataType());
        }

        INDArray result = Nd4j.pullRows(lookupTable.getWeights(), 1, indexes);
        return result;
    }
971785
private void addChars(String chars) {
        for (int i = 0; i < chars.length(); i++) {
            int c = chars.codePointAt(i) - min;
            if (values[c] != -1 && values[c] != i) {
                throw new IllegalArgumentException("Duplicate characters in the encoding alphabet");
            }
            values[c] = i;
        }
    }
569179
public Optional<? extends TrieNode> getChild(char token) {
    NodeData data = getData();
    int min = data.firstChildIndex;
    int max = data.firstChildIndex + data.numberOfChildren - 1;
    while (min <= max) {
      int i = (min + max) / 2;
      TrieNode node = new TrieNode(this.trie, i, TrieNode.this);
      char c = node.getChar();
      int compare = Character.compare(c, token);
      if (c < token) {
        // node.getChar() < token
        min = i + 1;
      }
      else if (c > token) {
        // node.getChar() > token
        max = i - 1;
      }
      else {
        return Optional.of(node);
      }
    }
    //assert !getChildren().keywords(x -> x.getChar() == token).findFirst().isPresent();
    return Optional.empty();
  }