Load a n-gram language model for mathematics in ARPA format which gets
    shipped with hwrt.

    Returns
    -------
    A NgramLanguageModel object
425483
def load_model():
    
    logging.info("Load language model...")
    ngram_arpa_t = pkg_resources.resource_filename('hwrt',
                                                   'misc/ngram.arpa.tar.bz2')
    with tarfile.open(ngram_arpa_t, 'r:bz2') as tar:
        tarfolder = tempfile.mkdtemp()
        tar.extractall(path=tarfolder)
    ngram_arpa_f = os.path.join(tarfolder, 'ngram.arpa')
    with open(ngram_arpa_f) as f:
        content = f.read()
    ngram_model = NgramLanguageModel()
    ngram_model.load_from_arpa_str(content)
    return ngram_model
374985
function nGram(n) {
  if (typeof n !== 'number' || isNaN(n) || n < 1 || n === Infinity) {
    throw new Error('`' + n + '` is not a valid argument for n-gram')
  }

  return grams

  // Create n-grams from a given value.
  function grams(value) {
    var nGrams = []
    var index

    if (value === null || value === undefined) {
      return nGrams
    }

    value = value.slice ? value : String(value)
    index = value.length - n + 1

    if (index < 1) {
      return nGrams
    }

    while (index--) {
      nGrams[index] = value.slice(index, index + n)
    }

    return nGrams
  }
}
1045776
def _loadable_models():
    
    classes = [
        pyphi.Direction,
        pyphi.Network,
        pyphi.Subsystem,
        pyphi.Transition,
        pyphi.labels.NodeLabels,
        pyphi.models.Cut,
        pyphi.models.KCut,
        pyphi.models.NullCut,
        pyphi.models.Part,
        pyphi.models.Bipartition,
        pyphi.models.KPartition,
        pyphi.models.Tripartition,
        pyphi.models.RepertoireIrreducibilityAnalysis,
        pyphi.models.MaximallyIrreducibleCauseOrEffect,
        pyphi.models.MaximallyIrreducibleCause,
        pyphi.models.MaximallyIrreducibleEffect,
        pyphi.models.Concept,
        pyphi.models.CauseEffectStructure,
        pyphi.models.SystemIrreducibilityAnalysis,
        pyphi.models.ActualCut,
        pyphi.models.AcRepertoireIrreducibilityAnalysis,
        pyphi.models.CausalLink,
        pyphi.models.Account,
        pyphi.models.AcSystemIrreducibilityAnalysis
    ]
    return {cls.__name__: cls for cls in classes}
589776
private void loadLanguageInfo() {
        // load coarse map
        coarseMap = new HashMap<>();
        try {
            try (BufferedReader br = new BufferedReader(new FileReader(options.unimapFile))) {
                String str;
                while ((str = br.readLine()) != null) {
                    String[] data = str.split("\\s+");
                    coarseMap.put(data[0], data[1]);
                }
            }

            coarseMap.put("<root-POS>", "ROOT");
        } catch (Exception e) {
            logger.warn("Couldn't find coarse POS map for this language");
        }

        // fill conj word
        conjWord = new HashSet<>();
        conjWord.add("and");
        conjWord.add("or");

    }
172899
def language_model(self,verbose=True):
        
        # use a generator in corpus
        prev = None
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            if prev:
                self.letter2[prev][next_letter] += 1
                if ( verbose ) :
                    print(prev)
                    print(next_letter)
                    print( self.letter2[prev][next_letter] )
            prev = next_letter #update always
        return
836954
public void load(String modelFile) throws IOException, ClassNotFoundException {
		System.out.println("加载N-Gram模型:" + modelFile);
		System.out.println("load ...");
		System.out.println(modelFile);
		ObjectInputStream in = new ObjectInputStream(new BufferedInputStream(
				new GZIPInputStream(new FileInputStream(modelFile))));
		n = (Integer) in.readObject(); 
		System.out.println("ngram " + n);
		in.close();
		System.out.println("load end");
	}
742387
def _generate_ngrams_with_context_helper(ngrams_iter: iter, ngrams_len: int) -> map:
        """"""
        return map(lambda term: (term[1], term[0], term[0] + ngrams_len), enumerate(ngrams_iter))
492681
def language_model(self,verbose=True):
        
        # use a generator in corpus
        p2 = None
        p1 = None
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            if p2:
                trig = p2+p1+next_letter
                self.letter3[trig] = 1 + self.letter3.get(trig,0)
            p2 = p1
            p1 = next_letter #update always
        return
534615
def _load_network(self):
    
    self.embeddings = load_embeddings(self.lang, type='cw')
    self.model = load_pos_model(lang=self.lang, version=2)

    def predict_proba(input_):
      hidden = np.tanh(np.dot(input_, self.model["W1"]) + self.model["b1"])
      output =  np.dot(hidden, self.model["W2"]) + self.model["b2"]
      scores = np.exp(output)
      probs = scores/scores.sum()
      return probs
    return predict_proba
912146
def update_model_languages(self, model_class, languages):
        
        # get the langs we have to add to the TransModelLanguage
        qs = TransLanguage.objects.filter(code__in=languages)
        new_langs = [lang for lang in qs]
        if not new_langs:
            return
        mod_lan, created = TransModelLanguage.objects.get_or_create(
            model='{} - {}'.format(model_class._meta.app_label, model_class._meta.model.__name__.lower()),
        )

        exist_langs_codes = [lang.code for lang in mod_lan.languages.all()]
        for lang in new_langs:
            if lang.code not in exist_langs_codes:
                try:
                    mod_lan.languages.add(lang)
                except IntegrityError:
                    pass
1148745
def main(model=None, output_dir=None, n_iter=100):
    """"""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pipe(ner, last=True)
    # otherwise, get it so we can add labels
    else:
        ner = nlp.get_pipe("ner")

    # add labels
    for _, annotations in TRAIN_DATA:
        for ent in annotations.get("entities"):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
    with nlp.disable_pipes(*other_pipes):  # only train NER
        # reset and initialize the weights randomly – but only if we're
        # training a new model
        if model is None:
            nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            # batch up the examples using spaCy's minibatch
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(
                    texts,  # batch of texts
                    annotations,  # batch of annotations
                    drop=0.5,  # dropout - make it harder to memorise data
                    losses=losses,
                )
            print("Losses", losses)

    # test the trained model
    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
        print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        for text, _ in TRAIN_DATA:
            doc = nlp2(text)
            print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
            print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])
1336922
def init_model(
    lang,
    output_dir,
    freqs_loc=None,
    clusters_loc=None,
    jsonl_loc=None,
    vectors_loc=None,
    prune_vectors=-1,
):
    
    if jsonl_loc is not None:
        if freqs_loc is not None or clusters_loc is not None:
            settings = ["-j"]
            if freqs_loc:
                settings.append("-f")
            if clusters_loc:
                settings.append("-c")
            msg.warn(
                "Incompatible arguments",
                "The -f and -c arguments are deprecated, and not compatible "
                "with the -j argument, which should specify the same "
                "information. Either merge the frequencies and clusters data "
                "into the JSONL-formatted file (recommended), or use only the "
                "-f and -c files, without the other lexical attributes.",
            )
        jsonl_loc = ensure_path(jsonl_loc)
        lex_attrs = srsly.read_jsonl(jsonl_loc)
    else:
        clusters_loc = ensure_path(clusters_loc)
        freqs_loc = ensure_path(freqs_loc)
        if freqs_loc is not None and not freqs_loc.exists():
            msg.fail("Can't find words frequencies file", freqs_loc, exits=1)
        lex_attrs = read_attrs_from_deprecated(freqs_loc, clusters_loc)

    with msg.loading("Creating model..."):
        nlp = create_model(lang, lex_attrs)
    msg.good("Successfully created model")
    if vectors_loc is not None:
        add_vectors(nlp, vectors_loc, prune_vectors)
    vec_added = len(nlp.vocab.vectors)
    lex_added = len(nlp.vocab)
    msg.good(
        "Sucessfully compiled vocab",
        "{} entries, {} vectors".format(lex_added, vec_added),
    )
    if not output_dir.exists():
        output_dir.mkdir()
    nlp.to_disk(output_dir)
    return nlp
726047
def load_from_arpa_str(self, arpa_str):
        
        data_found = False
        end_found = False
        in_ngram_block = 0
        for i, line in enumerate(arpa_str.split("\n")):
            if not end_found:
                if not data_found:
                    if "\\data\\" in line:
                        data_found = True
                else:
                    if in_ngram_block == 0:
                        if line.startswith("ngram"):
                            ngram_type, count = line.split("=")
                            _, n = ngram_type.split(" ")
                            n = int(n)
                            self.ngrams[n] = {'data': {},
                                              'count': count}
                        elif line.startswith("\\"):
                            n = int(line.split("-")[0][1:])
                            in_ngram_block = n
                        else:
                            continue  # Empty line
                    elif in_ngram_block > 0:
                        if "\\end\\" in line:
                            end_found = True
                        elif line.startswith("\\"):
                            n = int(line.split("-")[0][1:])
                            in_ngram_block = n
                        elif len(line) <= 1:
                            continue
                        else:
                            data = line.split("\t")
                            probability = Decimal(data[0])
                            ngram = data[1:]
                            if len(ngram) != n:
                                raise Exception(("ARPA language file is "
                                                 "inconsistant. Line %i has "
                                                 "only %i items, but should "
                                                 "have %i items.") %
                                                (i, len(ngram), n))
                            rest = ngram
                            append_to = self.ngrams[n]['data']
                            while len(rest) > 1:
                                first, rest = rest[0], rest[1:]
                                if first not in append_to:
                                    append_to[first] = {}
                                append_to = append_to[first]
                            if rest[0] in append_to:
                                raise Exception(("Duplicate entry for "
                                                 "ngram %s") % ngram)
                            append_to[rest[0]] = probability
            else:
                if line.startswith("info: "):
                    logging.info(line[6:])
1315352
def recognise_model(feature_filename,
                    symbollist_filename,
                    model_directory,
                    recognition_filename,
                    pronunciation_dictionary_filename,
                    list_words_filename='',
                    cmllr_directory=None,
                    tokens_count=None,
                    hypotheses_count=1,
                    htk_trace=0):
    
    # Normalize UTF-8 to avoid Mac problems
    temp_dictionary_filename = utf8_normalize(pronunciation_dictionary_filename)

    # Create language word list
    if list_words_filename:
        list_words = parse_wordlist(list_words_filename)
    else:
        list_words = sorted(parse_dictionary(temp_dictionary_filename).keys())

    # Create language model
    temp_directory = config.project_path('tmp', create=True)
    grammar_filename = config.path(temp_directory, 'grammar_words')
    wdnet_filename = config.path(temp_directory, 'wdnet')
    logging.debug('Create language model')
    create_language_model(list_words,
                          grammar_filename,
                          wdnet_filename)

    # Handle the Adaptation parameters
    cmllr_arguments = ''
    if cmllr_directory:
        if not os.path.isdir(cmllr_directory):
            logging.error('CMLLR adapatation directory not found: {}'.format(cmllr_directory))

        cmllr_arguments = "-J {} mllr2 -h '*/*_s%.mfc' -k -J {}".format(
            os.path.abspath(config.path(cmllr_directory, 'xforms')),
            os.path.abspath(config.path(cmllr_directory, 'classes')))
    # Handle the N-Best parameters
    hypotheses_count = hypotheses_count or 1
    tokens_count = tokens_count or int(math.ceil(hypotheses_count / 5.0))
    if hypotheses_count == 1 and tokens_count == 1:
        nbest_arguments = ""
    else:
        nbest_arguments = "-n {tokens_count} {hypotheses_count} ".format(tokens_count=tokens_count,
                                                                         hypotheses_count=hypotheses_count)

    # Run the HTK command
    config.htk_command("HVite -A -l '*' -T {htk_trace} "
                       "-H {model_directory}/macros -H {model_directory}/hmmdefs "
                       "-i {recognition_filename} -S {feature_filename} "
                       "{cmllr_arguments} -w {wdnet_filename} "
                       "{nbest_arguments} "
                       "-p 0.0 -s 5.0 "
                       "{pronunciation_dictionary_filename} "
                       "{symbollist_filename}".format(htk_trace=htk_trace,
                                                      model_directory=model_directory,
                                                      recognition_filename=recognition_filename,
                                                      feature_filename=feature_filename,
                                                      symbollist_filename=symbollist_filename,
                                                      nbest_arguments=nbest_arguments,
                                                      pronunciation_dictionary_filename=temp_dictionary_filename,
                                                      wdnet_filename=wdnet_filename,
                                                      cmllr_arguments=cmllr_arguments))

    # Remove temporary files
    os.remove(temp_dictionary_filename)
    os.remove(wdnet_filename)
    os.remove(grammar_filename)
1275523
def create_language_model(list_words,
                          grammar_filename,
                          wdnet_filename):
    """"""

    # Create the temporary grammar file
    grammar = u''
    grammar += u'$possible_words = '
    grammar += u' | '.join([word for word in list_words])
    grammar += u';\n'

    grammar += u'( [SENT-START] ( $possible_words ) [SENT-END] )\n'

    with codecs.open(grammar_filename, 'w', 'utf-8') as f:
        f.write(grammar)

    # Use HParse to create the wdnet file
    config.htk_command('HParse -A {} {}'.format(grammar_filename, wdnet_filename))

    return
708537
def load_lang_model(self):
        
        if self.lang in self.languages:
            if not Spacy.model_installed(self.lang):
                download(self.lang)
            model = spacy.load(self.lang)
        elif self.lang in self.alpha_languages:
            language_module = importlib.import_module(f"spacy.lang.{self.lang}")
            language_method = getattr(language_module, self.alpha_languages[self.lang])
            model = language_method()
        self.model = model
665173
def load(fh, model):
    
    graphs = penman.load(fh, cls=XMRSCodec)
    xs = [model.from_triples(g.triples()) for g in graphs]
    return xs