一系列文档的倒排词频

@param documentVocabularies 词表
@param smooth               平滑参数，视作额外有一个文档，该文档含有smooth个每个词语
@param addOne               tf-idf加一平滑
@param <TERM>               词语类型
@return 一个词语->倒排文档的Map
325762
public static <TERM> Map<TERM, Double> idf(Iterable<Iterable<TERM>> documentVocabularies,
                                               boolean smooth, boolean addOne)
    {
        Map<TERM, Integer> df = new HashMap<TERM, Integer>();
        int d = smooth ? 1 : 0;
        int a = addOne ? 1 : 0;
        int n = d;
        for (Iterable<TERM> documentVocabulary : documentVocabularies)
        {
            n += 1;
            for (TERM term : documentVocabulary)
            {
                Integer t = df.get(term);
                if (t == null) t = d;
                df.put(term, t + 1);
            }
        }
        Map<TERM, Double> idf = new HashMap<TERM, Double>();
        for (Map.Entry<TERM, Integer> e : df.entrySet())
        {
            TERM term = e.getKey();
            double f = e.getValue();
            idf.put(term, Math.log(n / f) + a);
        }
        return idf;
    }
364978
def compute_tf(self, sentences):
        
        content_words = self._get_all_content_words_in_doc(sentences)
        content_words_count = len(content_words)
        content_words_freq = self._compute_word_freq(content_words)
        content_word_tf = dict((w, f / content_words_count) for w, f in content_words_freq.items())
        return content_word_tf
800003
static public String getDictionaryDetail(PdfDictionary dic, int depth){
        StringBuffer builder = new StringBuffer();
        builder.append('(');
        List subDictionaries = new ArrayList();
        for (Iterator i = dic.getKeys().iterator(); i.hasNext(); ) {
            PdfName key = (PdfName)i.next();
            PdfObject val = dic.getDirectObject(key);
            if (val.isDictionary())
                subDictionaries.add(key);
            builder.append(key);
            builder.append('=');
            builder.append(val);
            builder.append(", ");
        }
        builder.setLength(builder.length()-2);
        builder.append(')');
        PdfName pdfSubDictionaryName;
        for (Iterator it = subDictionaries.iterator(); it.hasNext(); ) {
        	pdfSubDictionaryName = (PdfName)it.next();
            builder.append('\n');
            for(int i = 0; i < depth+1; i++){
                builder.append('\t');
            }
            builder.append("Subdictionary ");
            builder.append(pdfSubDictionaryName);
            builder.append(" = ");
            builder.append(getDictionaryDetail(dic.getAsDict(pdfSubDictionaryName), depth+1));
        }
        return builder.toString();
    }
833993
function createWordDivs(transferWords) {
  // Clear collectButtonsDiv first.
  while (collectButtonsDiv.firstChild) {
    collectButtonsDiv.removeChild(collectButtonsDiv.firstChild);
  }
  datasetViz = new DatasetViz(transferRecognizer,
                              collectButtonsDiv,
                              MIN_EXAMPLES_PER_CLASS,
                              startTransferLearnButton,
                              downloadAsFileButton,
                              transferDurationMultiplier);

  const wordDivs = {};
  for (const word of transferWords) {
    const wordDiv = document.createElement('div');
    wordDiv.classList.add('word-div');
    wordDivs[word] = wordDiv;
    wordDiv.setAttribute('word', word);
    const button = document.createElement('button');
    button.setAttribute('isFixed', 'true');
    button.style['display'] = 'inline-block';
    button.style['vertical-align'] = 'middle';

    const displayWord = word === BACKGROUND_NOISE_TAG ? 'noise' : word;

    button.textContent = `${displayWord} (0)`;
    wordDiv.appendChild(button);
    wordDiv.className = 'transfer-word';
    collectButtonsDiv.appendChild(wordDiv);
    collectWordButtons[word] = button;

    let durationInput;
    if (word === BACKGROUND_NOISE_TAG) {
      // Create noise duration input.
      durationInput = document.createElement('input');
      durationInput.setAttribute('isFixed', 'true');
      durationInput.value = '10';
      durationInput.style['width'] = '100px';
      wordDiv.appendChild(durationInput);
      // Create time-unit span for noise duration.
      const timeUnitSpan = document.createElement('span');
      timeUnitSpan.setAttribute('isFixed', 'true');
      timeUnitSpan.classList.add('settings');
      timeUnitSpan.style['vertical-align'] = 'middle';
      timeUnitSpan.textContent = 'seconds';
      wordDiv.appendChild(timeUnitSpan);
    }

    button.addEventListener('click', async () => {
      disableAllCollectWordButtons();
      removeNonFixedChildrenFromWordDiv(wordDiv);

      const collectExampleOptions = {};
      let durationSec;
      let intervalJob;
      let progressBar;

      if (word === BACKGROUND_NOISE_TAG) {
        // If the word type is background noise, display a progress bar during
        // sound collection and do not show an incrementally updating
        // spectrogram.
        // _background_noise_ examples are special, in that user can specify
        // the length of the recording (in seconds).
        collectExampleOptions.durationSec =
            Number.parseFloat(durationInput.value);
        durationSec = collectExampleOptions.durationSec;

        const barAndJob = createProgressBarAndIntervalJob(wordDiv, durationSec);
        progressBar = barAndJob.progressBar;
        intervalJob = barAndJob.intervalJob;
      } else {
        // If this is not a background-noise word type and if the duration
        // multiplier is >1 (> ~1 s recoding), show an incrementally
        // updating spectrogram in real time.
        collectExampleOptions.durationMultiplier = transferDurationMultiplier;
        let tempSpectrogramData;
        const tempCanvas = document.createElement('canvas');
        tempCanvas.style['margin-left'] = '132px';
        tempCanvas.height = 50;
        wordDiv.appendChild(tempCanvas);

        collectExampleOptions.snippetDurationSec = 0.1;
        collectExampleOptions.onSnippet = async (spectrogram) => {
          if (tempSpectrogramData == null) {
            tempSpectrogramData = spectrogram.data;
          } else {
            tempSpectrogramData = SpeechCommands.utils.concatenateFloat32Arrays(
                [tempSpectrogramData, spectrogram.data]);
          }
          plotSpectrogram(
              tempCanvas, tempSpectrogramData, spectrogram.frameSize,
              spectrogram.frameSize, {pixelsPerFrame: 2});
        }
      }

      collectExampleOptions.includeRawAudio =
          includeTimeDomainWaveformCheckbox.checked;
      const spectrogram = await transferRecognizer.collectExample(
          word, collectExampleOptions);


      if (intervalJob != null) {
        clearInterval(intervalJob);
      }
      if (progressBar != null) {
        wordDiv.removeChild(progressBar);
      }
      const examples = transferRecognizer.getExamples(word)
      const example = examples[examples.length - 1];
      await datasetViz.drawExample(
          wordDiv, word, spectrogram, example.example.rawAudio, example.uid);
      enableAllCollectWordButtons();
    });
  }
  return wordDivs;
}
805523
public static Map<String, String> getNamespaceMapping(final Document document) {
        Map<String, String> map = new HashMap<String, String>();
        map.put("xmlns", "http://www.w3.org/2000/xmlns/");
        map.put("xml", "http://www.w3.org/XML/1998/namespace");
//      if (childName.equals("xmlns") || childName.startsWith("xmlns:")) {
//      return "http://www.w3.org/2000/xmlns/";
//  }
//  if (childName.startsWith("xml:")) {
//      return "http://www.w3.org/XML/1998/namespace";
//  }

        Element root = document.getDocumentElement();
        if (root == null) {
            // No document, no namespaces.
            return map;
        }

        fillNSMapWithPrefixesDeclaredInElement(map, root);
        return map;
    }
351242
def _build_from_category_spacy_doc_iter(self, category_doc_iter):
        '''
        
        '''
        term_idx_store = IndexStore()
        category_idx_store = IndexStore()
        metadata_idx_store = IndexStore()
        X, mX, y = self._get_features_and_labels_from_documents_and_indexes \
            (category_doc_iter,
             category_idx_store,
             term_idx_store,
             metadata_idx_store)
        return TermDocMatrix(X,
                             mX,
                             y,
                             term_idx_store=term_idx_store,
                             category_idx_store=category_idx_store,
                             metadata_idx_store=metadata_idx_store)
538577
public static DocumentNode toDocumentNode(List<Object> documentNodeParams)
    {
        DocumentNode node = new DocumentNode((String) documentNodeParams.get(NODE_TITLE_INDEX));
        node.setIsExecutable((Boolean) documentNodeParams.get(NODE_EXECUTABLE_INDEX));
        node.setCanBeImplemented((Boolean) documentNodeParams.get(NODE_CAN_BE_IMPLEMENTED_INDEX));
        if (documentNodeParams.size() == 5) {
            node.setURL((String) documentNodeParams.get(NODE_URL_INDEX));
        }

        Hashtable children = (Hashtable) documentNodeParams.get(NODE_CHILDREN_INDEX);
        @SuppressWarnings("unchecked")
        Collection<Vector<Object>> values = children.values();
        for (Vector<Object> nodeParams : values) {

            if(nodeParams.size() > 5 ) {
                node.addChildren(ReferenceNode.toReferenceNode(nodeParams));
            } else {
                node.addChildren(toDocumentNode(nodeParams));
            }
        }

        return node;
    }
602725
function addToOperationsList(bulkOperation, docType, document) {
  // Get the bsonSize
  const bsonSize = bson.calculateObjectSize(document, {
    checkKeys: false,

    // Since we don't know what the user selected for BSON options here,
    // err on the safe side, and check the size with ignoreUndefined: false.
    ignoreUndefined: false
  });
  // Throw error if the doc is bigger than the max BSON size
  if (bsonSize >= bulkOperation.s.maxBatchSizeBytes)
    throw toError('document is larger than the maximum size ' + bulkOperation.s.maxBatchSizeBytes);
  // Holds the current batch
  bulkOperation.s.currentBatch = null;
  // Get the right type of batch
  if (docType === common.INSERT) {
    bulkOperation.s.currentBatch = bulkOperation.s.currentInsertBatch;
  } else if (docType === common.UPDATE) {
    bulkOperation.s.currentBatch = bulkOperation.s.currentUpdateBatch;
  } else if (docType === common.REMOVE) {
    bulkOperation.s.currentBatch = bulkOperation.s.currentRemoveBatch;
  }

  const maxKeySize = bulkOperation.s.maxKeySize;

  // Create a new batch object if we don't have a current one
  if (bulkOperation.s.currentBatch == null)
    bulkOperation.s.currentBatch = new Batch(docType, bulkOperation.s.currentIndex);

  // Check if we need to create a new batch
  if (
    bulkOperation.s.currentBatch.size + 1 >= bulkOperation.s.maxWriteBatchSize ||
    bulkOperation.s.currentBatch.sizeBytes + maxKeySize + bsonSize >=
      bulkOperation.s.maxBatchSizeBytes ||
    bulkOperation.s.currentBatch.batchType !== docType
  ) {
    // Save the batch to the execution stack
    bulkOperation.s.batches.push(bulkOperation.s.currentBatch);

    // Create a new batch
    bulkOperation.s.currentBatch = new Batch(docType, bulkOperation.s.currentIndex);
  }

  // We have an array of documents
  if (Array.isArray(document)) {
    throw toError('operation passed in cannot be an Array');
  }

  bulkOperation.s.currentBatch.operations.push(document);
  bulkOperation.s.currentBatch.originalIndexes.push(bulkOperation.s.currentIndex);
  bulkOperation.s.currentIndex = bulkOperation.s.currentIndex + 1;

  // Save back the current Batch to the right type
  if (docType === common.INSERT) {
    bulkOperation.s.currentInsertBatch = bulkOperation.s.currentBatch;
    bulkOperation.s.bulkResult.insertedIds.push({
      index: bulkOperation.s.bulkResult.insertedIds.length,
      _id: document._id
    });
  } else if (docType === common.UPDATE) {
    bulkOperation.s.currentUpdateBatch = bulkOperation.s.currentBatch;
  } else if (docType === common.REMOVE) {
    bulkOperation.s.currentRemoveBatch = bulkOperation.s.currentBatch;
  }

  // Update current batch size
  bulkOperation.s.currentBatch.size += 1;
  bulkOperation.s.currentBatch.sizeBytes += maxKeySize + bsonSize;

  // Return bulkOperation
  return bulkOperation;
}
874647
public String predict(List<VocabWord> document) {
        /*
            This code was transferred from original ParagraphVectors DL4j implementation, and yet to be tested
         */
        if (document.isEmpty())
            throw new IllegalStateException("Document has no words inside");

        /*
        INDArray arr = Nd4j.create(document.size(), this.layerSize);
        for (int i = 0; i < document.size(); i++) {
            arr.putRow(i, getWordVectorMatrix(document.get(i).getWord()));
        }*/

        INDArray docMean = inferVector(document); //arr.mean(0);
        Counter<String> distances = new Counter<>();

        for (String s : labelsSource.getLabels()) {
            INDArray otherVec = getWordVectorMatrix(s);
            double sim = Transforms.cosineSim(docMean, otherVec);
            distances.incrementCount(s, (float) sim);
        }

        return distances.argMax();
    }
200201
function improveFunc(doclet) {
    doclet.signature = doclet.name + '(';
    _.each(doclet.params, function(p, i) {
      if (!(p && p.type && p.type.names)) {
        logger.debug('Bad parameter', p, doclet.longname);
        return;
      }
      p.signature = ':param ' +
        p.type && p.type.names && p.type.names.join('|');
      p.signature += ' ' + p.name;

      if (p.optional) {
        doclet.signature += '[';
      }
      if (i > 0) {
        doclet.signature += ', ';
      }
      doclet.signature += p.name;
      if (p.optional) {
        doclet.signature += ']';
      }
      return p.name;
    });
    doclet.signature += ')';

    _.each(doclet.returns, function(r) {
      if (!(r && r.type && r.type.names)) {
        logger.debug('Bad return', r, doclet.longname);
        return;
      }
      r.signature = ':return ' +
        r.type && r.type.names && r.type.names.join('|');
    });
  }
1313035
public Map<String, Object> getDocument(String collectionName, String docId) {
        Map<String, Map<String,Object>> docs = DDPStateSingleton.getInstance()
                .getCollection(collectionName);
        if (docs != null) {
            return docs.get(docId);
        }
        return null;
    }
45645
public static List<String> getKeywordList(String document, int size)
    {
        return new TermFrequencyCounter().getKeywords(document, size);
    }
939120
public String asString(Document doc) {
        try {
            DOMSource domSource = new DOMSource(doc);
            StringWriter writer = new StringWriter();
            StreamResult result = new StreamResult(writer);
            TransformerFactory tf = TransformerFactory.newInstance();
            Transformer transformer = tf.newTransformer();
            transformer.transform(domSource, result);
            return writer.toString();
        } catch (TransformerException e) {
            throw new IllegalStateException(e);
        }
    }
1235926
public Text add(String id, String title, String body) {
        ArrayList<String> bag = new ArrayList<>();
        
        for (String sentence : splitter.split(body)) {
            String[] tokens = tokenizer.split(sentence);
            for (int i = 0; i < tokens.length; i++) {
                tokens[i] = tokens[i].toLowerCase();
            }

            for (String w : tokens) {
                boolean keep = true;
                if (punctuations != null && punctuations.contains(w)) {
                    keep = false;
                } else if (stopWords != null && stopWords.contains(w)) {
                    keep = false;
                }

                if (keep) {
                    size++;
                    bag.add(w);
                    
                    Integer f = freq.get(w);
                    if (f == null) {
                        f = 1;
                    } else {
                        f = f + 1;
                    }

                    freq.put(w, f);
                }
            }

            for (int i = 0; i < tokens.length - 1; i++) {
                String w1 = tokens[i];
                String w2 = tokens[i + 1];
                
                if (freq.containsKey(w1) && freq.containsKey(w2)) {
                    Bigram bigram = new Bigram(w1, w2);
                    Integer f = freq2.get(bigram);
                    if (f == null) {
                        f = 1;
                    } else {
                        f = f + 1;
                    }

                    freq2.put(bigram, f);
                }
            }
        }

        String[] words = new String[bag.size()];
        for (int i = 0; i < words.length; i++) {
            words[i] = bag.get(i);
        }

        SimpleText doc = new SimpleText(id, title, body, words);
        docs.add(doc);

        for (String term : doc.unique()) {
            List<SimpleText> hit = invertedFile.get(term);
            if (hit == null) {
                hit = new ArrayList<>();
                invertedFile.put(term, hit);
            }
            hit.add(doc);
        }

        return doc;
    }
1297429
function documentParam(param, paramDictionary, procedureOrService) {
    let typeString = getTypeStringFromCode(param.type, null, param);
    let name = getParamName(param);
    let options = {
        type: param.type,
        paramDictionary: paramDictionary,
        paramName: name
    };
    let description = getParamDescription(options);
    if (description !== '') {
        return ' * @param ' + typeString + ' ' + name + ' - ' + description;
    }
    return ' * @param ' + typeString + ' ' + name;
}
166718
public static <TERM> Map<TERM, Double> tf(Collection<TERM> document, TfType type)
    {
        Map<TERM, Double> tf = new HashMap<TERM, Double>();
        for (TERM term : document)
        {
            Double f = tf.get(term);
            if (f == null) f = 0.0;
            tf.put(term, f + 1);
        }
        if (type != TfType.NATURAL)
        {
            for (TERM term : tf.keySet())
            {
                switch (type)
                {
                    case LOGARITHM:
                        tf.put(term, 1 + Math.log(tf.get(term)));
                        break;
                    case BOOLEAN:
                        tf.put(term, tf.get(term) == 0.0 ? 0.0 : 1.0);
                        break;
                }
            }
        }
        return tf;
    }
1224849
function createIndex(addFn) {
  lunr.QueryLexer.termSeparator = lunr.tokenizer.separator = /\s+/;
  return lunr(/** @this */function() {
    this.ref('path');
    this.field('titleWords', {boost: 10});
    this.field('headingWords', {boost: 5});
    this.field('members', {boost: 4});
    this.field('keywords', {boost: 2});
    addFn(this);
  });
}