Load word vectors for the given vocab and table

@param table the weights to use
@param vocab the vocab to use
@return wordvectors based on the given parameters
547859
public static WordVectors fromTableAndVocab(WeightLookupTable table, VocabCache vocab) {
        WordVectorsImpl vectors = new WordVectorsImpl();
        vectors.setLookupTable(table);
        vectors.setVocab(vocab);
        vectors.setModelUtils(new BasicModelUtils());
        return vectors;
    }
613167
@Override
    public INDArray getWordVectorsMean(Collection<String> labels) {
        INDArray matrix = getWordVectors(labels);

        // TODO: check this (1)
        return matrix.mean(1);
    }
1031802
public boolean loadDictionary(String corpus, int termIndex, int countIndex) {
    File file = new File(corpus);
    if (!file.exists()) {
      return false;
    }

    BufferedReader br = null;
    try {
      br = Files.newBufferedReader(Paths.get(corpus), StandardCharsets.UTF_8);
    } catch (IOException ex) {
      System.out.println(ex.getMessage());
    }
    if (br == null) {
      return false;
    }
    return loadDictionary(br, termIndex, countIndex);
  }
941738
public static VocabCache<VocabWord> readVocabCache(@NonNull InputStream stream) throws IOException {
        val vocabCache = new AbstractCache.Builder<VocabWord>().build();
        val factory = new VocabWordFactory();
        boolean firstLine = true;
        long totalWordOcc = -1L;
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
            String line;
            while ((line = reader.readLine()) != null) {
                // try to treat first line as header with 3 digits
                if (firstLine) {
                    firstLine = false;
                    val split = line.split("\\ ");

                    if (split.length != 3)
                        continue;

                    try {
                        vocabCache.setTotalDocCount(Long.valueOf(split[1]));
                        totalWordOcc = Long.valueOf(split[2]);
                        continue;
                    } catch (NumberFormatException e) {
                        // no-op
                    }
                }

                val word = factory.deserialize(line);

                vocabCache.addToken(word);
                vocabCache.addWordToIndex(word.getIndex(), word.getLabel());
            }
        }

        if (totalWordOcc >= 0)
            vocabCache.setTotalWordOccurences(totalWordOcc);

        return vocabCache;
    }
272343
public static void writeWordVectors(@NonNull Glove vectors, @NonNull File file) {
        try (BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(file))) {
            writeWordVectors(vectors, fos);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
671640
def get_word_vector(self, word):
        """"""
        dim = self.get_dimension()
        b = fasttext.Vector(dim)
        self.f.getWordVector(b, word)
        return np.array(b)
512059
def load_data_with_word2vec(word2vec_list):
    
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    # vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    return build_input_data_with_word2vec(sentences_padded, labels, word2vec_list)
843092
def load_pretrained_word2vec(infile):
    """"""
    if isinstance(infile, str):
        infile = open(infile)

    word2vec_list = {}
    for idx, line in enumerate(infile):
        if idx == 0:
            vocab_size, dim = line.strip().split()
        else:
            tks = line.strip().split()
            word2vec_list[tks[0]] = map(float, tks[1:])

    return word2vec_list
965209
def load_vocab(vocab_file):
    """"""
    vocab = collections.OrderedDict()
    index = 0
    with io.open(vocab_file, 'r') as reader:
        while True:
            token = reader.readline()
            if not token:
                break
            token = token.strip()
            vocab[token] = index
            index += 1
    return vocab
1250369
public void iterateSample(Word2VecParam param, VocabWord w1, VocabWord w2, double alpha,
                    List<Triple<Integer, Integer, Integer>> changed) {
        if (w2 == null || w2.getIndex() < 0 || w1.getIndex() == w2.getIndex() || w1.getWord().equals("STOP")
                        || w2.getWord().equals("STOP") || w1.getWord().equals("UNK") || w2.getWord().equals("UNK"))
            return;
        int vectorLength = param.getVectorLength();
        InMemoryLookupTable weights = param.getWeights();
        boolean useAdaGrad = param.isUseAdaGrad();
        double negative = param.getNegative();
        INDArray table = param.getTable();
        double[] expTable = param.getExpTable().getValue();
        double MAX_EXP = 6;
        int numWords = param.getNumWords();
        //current word vector
        INDArray l1 = weights.vector(w2.getWord());


        //error for current word and context
        INDArray neu1e = Nd4j.create(vectorLength);

        for (int i = 0; i < w1.getCodeLength(); i++) {
            int code = w1.getCodes().get(i);
            int point = w1.getPoints().get(i);

            INDArray syn1 = weights.getSyn1().slice(point);

            double dot = Nd4j.getBlasWrapper().level1().dot(syn1.length(), 1.0, l1, syn1);

            if (dot < -MAX_EXP || dot >= MAX_EXP)
                continue;

            int idx = (int) ((dot + MAX_EXP) * ((double) expTable.length / MAX_EXP / 2.0));

            //score
            double f = expTable[idx];
            //gradient
            double g = (1 - code - f) * (useAdaGrad ? w1.getGradient(i, alpha, alpha) : alpha);


            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, syn1, neu1e);
            Nd4j.getBlasWrapper().level1().axpy(syn1.length(), g, l1, syn1);


            changed.add(new Triple<>(point, w1.getIndex(), -1));

        }


        changed.add(new Triple<>(w1.getIndex(), w2.getIndex(), -1));
        //negative sampling
        if (negative > 0) {
            int target = w1.getIndex();
            int label;
            INDArray syn1Neg = weights.getSyn1Neg().slice(target);

            for (int d = 0; d < negative + 1; d++) {
                if (d == 0) {

                    label = 1;
                } else {
                    nextRandom.set(nextRandom.get() * 25214903917L + 11);
                    // FIXME: int cast
                    target = table.getInt((int) (nextRandom.get() >> 16) % (int) table.length());
                    if (target == 0)
                        target = (int) nextRandom.get() % (numWords - 1) + 1;
                    if (target == w1.getIndex())
                        continue;
                    label = 0;
                }

                double f = Nd4j.getBlasWrapper().dot(l1, syn1Neg);
                double g;
                if (f > MAX_EXP)
                    g = useAdaGrad ? w1.getGradient(target, (label - 1), alpha) : (label - 1) * alpha;
                else if (f < -MAX_EXP)
                    g = label * (useAdaGrad ? w1.getGradient(target, alpha, alpha) : alpha);
                else
                    g = useAdaGrad ? w1
                                    .getGradient(target,
                                                    label - expTable[(int) ((f + MAX_EXP)
                                                                    * (expTable.length / MAX_EXP / 2))],
                                                    alpha)
                                    : (label - expTable[(int) ((f + MAX_EXP) * (expTable.length / MAX_EXP / 2))])
                                                    * alpha;
                Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, neu1e, l1);

                Nd4j.getBlasWrapper().level1().axpy(l1.length(), g, syn1Neg, l1);

                changed.add(new Triple<>(-1, -1, label));

            }
        }


        Nd4j.getBlasWrapper().level1().axpy(l1.length(), 1.0f, neu1e, l1);


    }
569002
public TextBuilder link(final String text, final Table table) {
        this.curParagraphBuilder.link(text, table);
        return this;
    }
941839
@Override
    public Collection<String> wordsNearest(INDArray words, int top) {
        Counter<String> distances = new Counter<>();

        for (String s : vocabCache.words()) {
            INDArray otherVec = lookupTable.vector(s);
            double sim = Transforms.cosineSim(Transforms.unitVec(words.dup()), Transforms.unitVec(otherVec.dup()));
            distances.incrementCount(s, (float) sim);
        }

        distances.keepTopNElements(top);
        return distances.keySetSorted();
    }
45418
void sortVocab()
    {
        Arrays.sort(vocab, 0, vocabSize);

        // re-build vocabIndexMap
        final int size = vocabSize;
        trainWords = 0;
        table = new int[size];
        for (int i = 0; i < size; i++)
        {
            VocabWord word = vocab[i];
            // Words occuring less than min_count times will be discarded from the vocab
            if (word.cn < config.getMinCount())
            {
                table[vocabIndexMap.get(word.word)] = -4;
                vocabSize--;
            }
            else
            {
                // Hash will be re-computed, as after the sorting it is not actual
                table[vocabIndexMap.get(word.word)] = i;
                setVocabIndexMap(word, i);
            }
        }
        // lose weight
        vocabIndexMap = null;

        VocabWord[] nvocab = new VocabWord[vocabSize];
        System.arraycopy(vocab, 0, nvocab, 0, vocabSize);

    }
971506
private Map<String,SemanticVector> loadText(File sspaceFile) 
        throws IOException {

        LOGGER.info("loading text TSS from " + sspaceFile);
        
        BufferedReader br = new BufferedReader(new FileReader(sspaceFile));
        String[] header = br.readLine().split("\\s+");
        int words = Integer.parseInt(header[0]);
        dimensions = Integer.parseInt(header[1]);
        
        Map<String,SemanticVector> wordToSemantics = 
            new HashMap<String,SemanticVector>(words, 2f);
        
        // read in each word
        for (String line = null; (line = br.readLine()) != null; ) {
            String[] wordAndSemantics = line.split("\\|");
            String word = wordAndSemantics[0];
            SemanticVector semantics = new SemanticVector(dimensions);

            LOGGER.info("loading " + wordAndSemantics.length + 
                " timesteps for word " + word); 

            for (int i = 1; i < wordAndSemantics.length; ++i) {
                String[] timeStepAndValues = wordAndSemantics[i].split(" ");
                long timeStep = Long.parseLong(timeStepAndValues[0]);
                updateTimeRange(timeStep);

                // Load that time step's vector.  Note that we make the
                // assumption here that even though the T-Space is serialized in
                // a dense format, that the vector data is actually sparse, and
                // so it will be more efficient to store it as such.
                Map<Integer,Double> sparseArray = new IntegerMap<Double>();

                for (int j = 1; j < timeStepAndValues.length; ++j) {
                    sparseArray.put(Integer.valueOf(j-1), 
                            Double.valueOf(timeStepAndValues[j]));
                }
                semantics.setSemantics(timeStep, sparseArray);
            }
            wordToSemantics.put(word,semantics);
        }
    
        return wordToSemantics;
    }
318570
public boolean loadDictionary(InputStream corpus, int termIndex, int countIndex) {
    if (corpus == null) {
      return false;
    }
    BufferedReader br = new BufferedReader(new InputStreamReader(corpus, StandardCharsets.UTF_8));
    return loadDictionary(br, termIndex, countIndex);
  }
421603
@Override
    public List<String> similarWordsInVocabTo(String word, double accuracy) {
        List<String> ret = new ArrayList<>();
        for (String s : vocabCache.words()) {
            if (MathUtils.stringSimilarity(word, s) >= accuracy)
                ret.add(s);
        }
        return ret;
    }
1324045
def read_vocab(args, column_name):
  
  vocab_path = os.path.join(args.analysis,
                            feature_transforms.VOCAB_ANALYSIS_FILE % column_name)

  if not file_io.file_exists(vocab_path):
    return []

  vocab, _ = feature_transforms.read_vocab_file(vocab_path)
  return vocab