builds a Tamil bigram letter model
492681
def language_model(self,verbose=True):
        
        # use a generator in corpus
        p2 = None
        p1 = None
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            if p2:
                trig = p2+p1+next_letter
                self.letter3[trig] = 1 + self.letter3.get(trig,0)
            p2 = p1
            p1 = next_letter #update always
        return
556198
def text_cnn_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.batch_size = 4096
  hparams.max_length = 256
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.learning_rate_schedule = "legacy"
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 4000
  hparams.initializer_gain = 1.0
  hparams.num_hidden_layers = 6
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.98
  hparams.num_sampled_classes = 0
  hparams.label_smoothing = 0.1
  hparams.shared_embedding_and_softmax_weights = True
  hparams.symbol_modality_num_shards = 16

  # Add new ones like this.
  hparams.add_hparam("filter_sizes", [2, 3, 4, 5])
  hparams.add_hparam("num_filters", 128)
  hparams.add_hparam("output_dropout", 0.4)
  return hparams
455422
def _keyword_expander(word, language, lemmatized=False, threshold=0.70):
    
    try:
        from cltk.vector.word2vec import get_sims
    except ImportError as imp_err:
        print(imp_err)
        raise
    similar_vectors = get_sims(word, language, lemmatized=lemmatized, threshold=threshold)

    return similar_vectors
471004
def BiLSTM(nO, nI):
    """"""
    return Bidirectional(LSTM(nO // 2, nI), LSTM(nO // 2, nI))
1148745
def main(model=None, output_dir=None, n_iter=100):
    """"""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank("en")  # create blank Language class
        print("Created blank 'en' model")

    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if "ner" not in nlp.pipe_names:
        ner = nlp.create_pipe("ner")
        nlp.add_pipe(ner, last=True)
    # otherwise, get it so we can add labels
    else:
        ner = nlp.get_pipe("ner")

    # add labels
    for _, annotations in TRAIN_DATA:
        for ent in annotations.get("entities"):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != "ner"]
    with nlp.disable_pipes(*other_pipes):  # only train NER
        # reset and initialize the weights randomly – but only if we're
        # training a new model
        if model is None:
            nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            # batch up the examples using spaCy's minibatch
            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
            for batch in batches:
                texts, annotations = zip(*batch)
                nlp.update(
                    texts,  # batch of texts
                    annotations,  # batch of annotations
                    drop=0.5,  # dropout - make it harder to memorise data
                    losses=losses,
                )
            print("Losses", losses)

    # test the trained model
    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
        print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        for text, _ in TRAIN_DATA:
            doc = nlp2(text)
            print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
            print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])
318728
def transliterate(text,lang1_code,lang2_code):
        
        if (lang1_code in langinfo.SCRIPT_RANGES) and (lang2_code in langinfo.SCRIPT_RANGES):
            
            # if Sinhala is source, do a mapping to Devanagari first 
            if lang1_code=='si': 
                text=sdt.sinhala_to_devanagari(text)
                lang1_code='hi'

            # if Sinhala is target, make Devanagiri the intermediate target
            org_lang2_code=''
            if lang2_code=='si': 
                lang2_code='hi'
                org_lang2_code='si'

            trans_lit_text=[]
            for c in text: 
                newc=c
                offset=ord(c)-langinfo.SCRIPT_RANGES[lang1_code][0]
                if offset >=langinfo.COORDINATED_RANGE_START_INCLUSIVE and offset <= langinfo.COORDINATED_RANGE_END_INCLUSIVE:
                    if lang2_code=='ta': 
                        # tamil exceptions 
                        offset=UnicodeIndicTransliterator._correct_tamil_mapping(offset)
                    newc=py23char(langinfo.SCRIPT_RANGES[lang2_code][0]+offset)

                trans_lit_text.append(newc)        

            # if Sinhala is source, do a mapping to Devanagari first 
            if org_lang2_code=='si': 
                return sdt.devanagari_to_sinhala(''.join(trans_lit_text))

            return (''.join(trans_lit_text))
        else:
            return text
80813
def get_bigram_pair_string(self, text):
        
        bigram_pairs = []

        if len(text) <= 2:
            text_without_punctuation = text.translate(self.punctuation_table)
            if len(text_without_punctuation) >= 1:
                text = text_without_punctuation

        document = self.nlp(text)

        if len(text) <= 2:
            bigram_pairs = [
                token.lemma_.lower() for token in document
            ]
        else:
            tokens = [
                token for token in document if token.is_alpha and not token.is_stop
            ]

            if len(tokens) < 2:
                tokens = [
                    token for token in document if token.is_alpha
                ]

            for index in range(1, len(tokens)):
                bigram_pairs.append('{}:{}'.format(
                    tokens[index - 1].pos_,
                    tokens[index].lemma_.lower()
                ))

        if not bigram_pairs:
            bigram_pairs = [
                token.lemma_.lower() for token in document
            ]

        return ' '.join(bigram_pairs)
98683
def cbow_fasttext_batch(centers, contexts, num_tokens, subword_lookup, dtype,
                        index_dtype):
    """"""
    _, contexts_row, contexts_col = contexts
    data, row, col = subword_lookup(contexts_row, contexts_col)
    centers = mx.nd.array(centers, dtype=index_dtype)
    contexts = mx.nd.sparse.csr_matrix(
        (data, (row, col)), dtype=dtype,
        shape=(len(centers), num_tokens))  # yapf: disable
    return centers, contexts
769124
def _get_ngrams(n, text):
  
  ngram_set = set()
  text_length = len(text)
  max_index_ngram_start = text_length - n
  for i in range(max_index_ngram_start + 1):
    ngram_set.add(tuple(text[i:i + n]))
  return ngram_set
190016
def get_lang_data(self):
        

        root = os.path.expanduser('~')
        csv_dir_path = os.path.join(root, 'cltk_data/sanskrit/model/sanskrit_models_cltk/phonetics')

        all_phonetic_csv = os.path.join(csv_dir_path, 'all_script_phonetic_data.csv')
        tamil_csv = os.path.join(csv_dir_path, 'tamil_script_phonetic_data.csv')

        # Make helper function for this
        with open(all_phonetic_csv,'r') as f:
            reader = csv.reader(f, delimiter = ',', quotechar = '"')
            next(reader, None) # Skip headers
            all_phonetic_data = [row for row in reader]

        with open(tamil_csv,'r') as f:
            reader = csv.reader(f, delimiter = ',', quotechar = '"')
            next(reader, None) # Skip headers
            # tamil_phonetic_data = [row[PHONETIC_VECTOR_START_OFFSET:] for row in reader]
            tamil_phonetic_data = [row for row in reader]

        # Handle better?
        all_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in all_phonetic_data]
        tamil_phonetic_data = [[int(cell) if cell=='0' or cell=='1' else cell for cell in row] for row in tamil_phonetic_data]

        all_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in all_phonetic_data])
        tamil_phonetic_vectors = np.array([row[PHONETIC_VECTOR_START_OFFSET:] for row in tamil_phonetic_data])

        phonetic_vector_length = all_phonetic_vectors.shape[1]

        return all_phonetic_data, tamil_phonetic_data, all_phonetic_vectors, tamil_phonetic_vectors, phonetic_vector_length
1010358
def attention_lm_moe_unscramble_base():
  """"""
  hparams = attention_lm_no_moe_small()
  hparams.use_inputs = True
  hparams.min_length_bucket = 1024
  hparams.max_length = 1024
  hparams.batch_size = 5000
  hparams.layer_prepostprocess_dropout = 0.0
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  return hparams
1010523
def build_and_train(iterations, log_stride, test=False):
  """"""

  # Test mode settings.
  batch_size = 2 if test else FLAGS.batch_size
  num_mems = 2 if test else FLAGS.num_mems
  num_heads = 1 if test else FLAGS.num_mems
  num_blocks = 1 if test else FLAGS.num_mems
  head_size = 4 if test else FLAGS.head_size
  max_length = 3 if test else FLAGS.max_length
  max_nest = 2 if test else FLAGS.max_nest
  mlp_size = (20,) if test else (256, 256, 256, 256)

  with tf.Graph().as_default():
    t0 = time.time()

    # Initialize the dataset.
    lte_train = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest)
    lte_test = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest, mode=learn_to_execute.Mode.TEST)
    train_data_iter = lte_train.make_one_shot_iterator().get_next()
    test_data_iter = lte_test.make_one_shot_iterator().get_next()
    output_size = lte_train.state.vocab_size

    # Create the model.
    core = snt.RelationalMemory(
        mem_slots=num_mems,
        head_size=head_size,
        num_heads=num_heads,
        num_blocks=num_blocks,
        gate_style=FLAGS.gate_style)
    final_mlp = snt.nets.MLP(
        output_sizes=mlp_size,
        activate_final=True)
    model = SequenceModel(
        core=core,
        target_size=output_size,
        final_mlp=final_mlp)
    tf.logging.info("Instantiated models ({:3f})".format(time.time() - t0))

    # Define the loss & accuracy.
    def loss_fn(inputs, targets, input_sequence_length, output_sequence_length):
      """Creates the loss and the exports."""
      logits = model(
          inputs, targets, input_sequence_length, output_sequence_length)
      targets = tf.cast(targets, tf.int32)
      sq_sz_out_max = targets.shape[0].value

      # Create a mask to ignore accuracy on buffer characters.
      sequence_sizes = tf.cast(output_sequence_length, tf.float32)
      lengths_transposed = tf.expand_dims(sequence_sizes, 1)
      range_row = tf.expand_dims(
          tf.range(0, sq_sz_out_max, 1, dtype=tf.float32), 0)
      mask = tf.cast(tf.transpose(tf.less(range_row, lengths_transposed)),
                     tf.float32)

      # Compute token accuracy and solved.
      correct = tf.equal(tf.argmax(logits, 2), tf.argmax(targets, 2))
      solved = tf.reduce_all(tf.boolean_mask(correct, tf.squeeze(mask)), axis=0)
      token_acc = tf.reduce_sum(tf.cast(correct, tf.float32) * mask)
      token_acc /= tf.reduce_sum(sequence_sizes)

      # Compute Loss.
      mask = tf.cast(tf.tile(tf.expand_dims(mask, 2), (1, 1, logits.shape[2])),
                     tf.float32)
      masked_logits = logits * mask
      masked_target = tf.cast(targets, tf.float32) * mask
      logits_flat = tf.reshape(masked_logits,
                               [sq_sz_out_max * batch_size, -1])
      target_flat = tf.reshape(masked_target,
                               [sq_sz_out_max * batch_size, -1])
      xent = tf.nn.softmax_cross_entropy_with_logits(logits=logits_flat,
                                                     labels=target_flat)
      loss = tf.reduce_mean(xent)
      return loss, token_acc, solved

    # Get training step counter.
    global_step = tf.train.get_or_create_global_step()

    # Create the optimizer.
    learning_rate_op = tf.reduce_max([
        tf.train.exponential_decay(
            FLAGS.learning_rate,
            global_step,
            decay_steps=FLAGS.epochs // 100,
            decay_rate=0.9,
            staircase=False),
        FLAGS.min_learning_rate
    ])
    optimizer = tf.train.AdamOptimizer(learning_rate_op)

    # Compute loss, accuracy & the step op.
    inputs, targets, _, input_lengths, output_lengths = train_data_iter
    train_loss, train_acc, train_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    step_op = optimizer.minimize(train_loss, global_step=global_step)
    inputs, targets, _, input_lengths, output_lengths = test_data_iter
    _, test_acc, test_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    tf.logging.info("Created losses and optimizers ({:3f})".format(
        time.time() - t0))

    # Begin Training.
    t0 = time.time()
    tf.logging.info("Starting training ({:3f})".format(time.time() - t0))
    with tf.train.SingularMonitoredSession() as sess:
      for it in six.moves.range(iterations):
        sess.run([step_op, learning_rate_op])
        if it % log_stride == 0:
          loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v = sess.run([
              train_loss, train_acc, test_acc, train_sol, test_sol])
          elapsed = time.time() - t0
          tf.logging.info(
              "iter: {:2d}, train loss {:3f}; train acc {:3f}; test acc {:3f};"
              " train solved {:3f}; test solved {:3f}; ({:3f})".format(
                  it, loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v,
                  elapsed))
340981
def num2tamilstr( *args ):
    
    number = args[0]
    if len(args) < 2:
        filenames = []
    else:
        filenames = args[1]
    if len(args) ==3:
        tensSpecial = args[2]
    else:
        tensSpecial='BASIC'
    if not any( filter( lambda T: isinstance( number, T), [str,unicode,int, long, float]) ) or isinstance(number,complex):
        raise Exception('num2tamilstr input has to be a long or integer or float')
    if float(number) > long(1e12):
        raise Exception('num2tamilstr input is too large')
    if float(number) < 0:
        return u"- "+num2tamilstr( -number )

    units = (u'பூஜ்ஜியம்', u'ஒன்று', u'இரண்டு', u'மூன்று', u'நான்கு', u'ஐந்து', u'ஆறு', u'ஏழு', u'எட்டு', u'ஒன்பது', u'பத்து') # 0-10
    units_suffix = (u'பூஜ்ஜியம்', u'தொன்று', u'திரண்டு', u'மூன்று', u'நான்கு', u'தைந்து', u'தாறு', u'தேழு', u'தெட்டு', u'தொன்பது', u'பத்து') # 0-10
    units_suffix_nine = (u'பூஜ்ஜியம்', u'றொன்று', u'றிரண்டு', u'மூன்று', u'நான்கு', u'றைந்து', u'றாறு', u'றேழு', u'றெட்டு', u'றொன்பது', u'பத்து') # 0-10
    tween = [1.0,2.0,5.0,6.0,7.0,8.0,9.0]
    teens = (u'பதினொன்று', u'பனிரண்டு', u'பதிமூன்று', u'பதினான்கு', u'பதினைந்து',u'பதினாறு', u'பதினேழு', u'பதினெட்டு', u'பத்தொன்பது') # 11-19
    tens = (u'பத்து', u'இருபது', u'முப்பது', u'நாற்பது', u'ஐம்பது',u'அறுபது', u'எழுபது', u'எண்பது', u'தொன்னூறு') # 10-90
    tens_full_prefix = (u'இருபத்து', u'முப்பத்து', u'நாற்பத்து', u'ஐம்பத்து', u'அறுபத்து', u'எழுபத்து', u'எண்பத்து', u'தொன்னூற்று') # 10+-90+
    tens_prefix = (u'இருபத்', u'முப்பத்', u'நாற்பத்', u'ஐம்பத்', u'அறுபத்', u'எழுபத்', u'எண்பத்', u'தொன்னூற்') # 10+-90+
    hundreds = ( u'நூறு', u'இருநூறு', u'முன்னூறு', u'நாநூறு',u'ஐநூறு', u'அறுநூறு', u'எழுநூறு', u'எண்ணூறு', u'தொள்ளாயிரம்') #100 - 900
    hundreds_suffix = (u'நூற்றி', u'இருநூற்று', u'முன்னூற்று', u'நாநூற்று', u'ஐநூற்று', u'அறுநூற்று', u'எழுநூற்று', u'எண்ணூற்று',u'தொள்ளாயிரத்து') #100+ - 900+

    one_thousand_prefix = u'ஓர்'
    thousands = (u'ஆயிரம்',u'ஆயிரத்து')

    one_prefix = u'ஒரு'
    lakh = (u'இலட்சம்',u'இலட்சத்து')
    crore = (u'கோடி',u'கோடியே')

    pulli = u'புள்ளி'
    n_one = 1.0
    n_ten = 10.0
    n_hundred = 100.0
    n_thousand = 1000.0
    n_lakh = 100.0*n_thousand
    n_crore = (100.0*n_lakh)
    # handle fractional parts
    if float(number) > 0.0 and float(number) < 1.0:
        rval = []
        rval.append(pulli)
        filenames.append( 'pulli' )
        number_str = str(number).replace('0.','')
        for digit in number_str:
            filenames.append( "units_%d"%int(digit))
            rval.append( units[int(digit)] )
        return u' '.join(rval)

    if isinstance(number,str) or isinstance(number,unicode):
        result = u""
        number = number.strip()
        assert(len(args) == 1)
        assert(len(number) > 0)
        is_negative = number[0] == "-"
        if  is_negative:
            number = number[1:]
        frac_part = u""
        if number.find(".") >= 0:
            rat_part,frac_part = number.split(".")
            frac_part = num2tamilstr(u"0."+frac_part)
        else:
            rat_part = number
        if len(rat_part) > 0:
            result = num2tamilstr(float(rat_part))
        result = result +u" "+ frac_part
        return is_negative and "-" + result.strip() or result.strip()

    suffix_base = { n_crore: crore, n_lakh : lakh, n_thousand : thousands}
    suffix_file_map = {n_crore: "crore", n_lakh : "lakh", n_thousand : "thousands"}

    file_map = {n_crore :["one_prefix","crore_0"],
                n_lakh : ["one_prefix","lakh_0"],
                n_thousand :  ["one_thousand_prefix", "thousands_0"],
               n_hundred : ["hundreds_0"], #special
               n_ten : ["units_10"],
               n_one : ["units_1"]}

    num_map = {n_crore : [one_prefix,crore[0]],
               n_lakh  : [one_prefix,lakh[0]],
               n_thousand : [one_thousand_prefix, thousands[0]],
               n_hundred : [hundreds[0]], #special
               n_ten : [units[10]],
               n_one : [units[1]]}

    all_bases = [n_crore, n_lakh, n_thousand, n_hundred, n_ten,n_one]
    allowed_bases = list(filter( lambda base: number >= base, all_bases ))
    if len(allowed_bases) >= 1:
        n_base = allowed_bases[0]
        if number == n_base:
            if tensSpecial=='BASIC':
                filenames.extend(file_map[n_base])
                return u" ".join(num_map[n_base])
            elif tensSpecial=='NINES':
                filenames.extend(file_map[n_base])
                return units_suffix_nine[long(number%10)]
            else:
                filenames.extend(file_map[n_base])
                return units_suffix[long(number%10)]
        quotient_number = long( number/n_base )
        residue_number = number - n_base*quotient_number
        #print number, n_base, quotient_number, residue_number, tensSpecial
        if n_base == n_one:
            if isinstance(number,float):
                int_part = long(number%10)
                frac = number - float(int_part)
                filenames.append("units_%d"%int_part)
                if abs(frac) > 1e-30:
                    if tensSpecial=='BASIC':
                        return units[int_part]+u' ' + num2tamilstr(frac,filenames)
                    elif tensSpecial=='NINES':
                        return units_suffix_nine[int_part]+u' ' + num2tamilstr(frac,filenames)
                    else:
                        return units_suffix[int_part]+u' ' + num2tamilstr(frac,filenames)

                else:
                    if tensSpecial=='BASIC':
                        return units[int_part]
                    elif tensSpecial=='NINES':
                        return units_suffix_nine[int_part]
                    else:
                        return units_suffix[int_part]
            else:
                if tensSpecial=='BASIC':
                    filenames.append("units_%d"%number)
                    return units[number]
                elif tensSpecial=='NINES':
                    filenames.append("units_%d"%number)
                    return units_suffix_nine[number]
                else:
                    filenames.append("units_%d"%number)
                    return units_suffix[number]

        elif n_base == n_ten:
            if residue_number < 1.0:
                filenames.append("tens_%d"%(quotient_number-1))
                if residue_number == 0.0:
                    return tens[quotient_number-1]
                #else: //seems not reachable.
                #    numeral = tens[quotient_number-1]
            elif number < 20:
                filenames.append("teens_%d"%(number-10))
                residue_number = math.fmod(number,1)
                teen_number = int(math.floor(number - 10))
                if residue_number > 1e-30:
                    return teens[teen_number-1] +u' ' + num2tamilstr(residue_number,filenames)
                else:
                    return teens[teen_number-1]+u' '
            if residue_number < 1.0:
                filenames.append( "tens_%d"%(quotient_number-1) )
                numeral = tens[quotient_number-1]+u' '
            else:
                if residue_number in tween:
                    filenames.append( "tens_prefix_%d"%(quotient_number-2) )
                    numeral = tens_prefix[quotient_number-2]
                    tensSpecial='SPECIAL'
                    if (quotient_number==9):
                        tensSpecial = 'NINES'
                else:
                    filenames.append( "tens_prefix_%d"%(quotient_number-2) )
                    numeral = tens_full_prefix[quotient_number-2]+u' '
        elif n_base == n_hundred:
            if residue_number == 0:
                filenames.append("hundreds_%d"%(quotient_number-1))
                return hundreds[quotient_number-1]+u' '
            if residue_number < 1.0:
                filenames.append( "hundreds_%d"%(quotient_number-1) )
                numeral = hundreds[quotient_number-1]+u' '
            else:
                filenames.append("hundreds_suffix_%d"%(quotient_number-1))
                numeral = hundreds_suffix[quotient_number-1]+u' '
        else:
            if ( quotient_number == 1 ):
                if n_base == n_thousand:
                    filenames.append("one_thousand_prefix")
                    numeral = one_thousand_prefix
                else:
                    filenames.append("one_prefix")
                    numeral = one_prefix
            else:
                numeral = num2tamilstr( quotient_number, filenames )
        if n_base >= n_thousand:
            suffix = suffix_base[n_base][long(residue_number >= 1)]
            suffix_filename = "%s_%d"%(suffix_file_map[n_base],long(residue_number >= 1))
            if residue_number == 0:
                filenames.append(suffix_filename)
                return numeral + u' ' + suffix+u' '
            filenames.append(suffix_filename)
            numeral = numeral + u' ' + suffix+u' '
        residue_numeral = num2tamilstr( residue_number, filenames, tensSpecial)
        #return numeral+u' '+residue_numeral
        return numeral+residue_numeral
    # number has to be zero
    filenames.append("units_0")
    return units[0]
394545
def _generate_substrings(self, ngram, size):
        
        text = Text(ngram, self._tokenizer)
        substrings = []
        for sub_size, ngrams in text.get_ngrams(1, size-1):
            for sub_ngram, count in ngrams.items():
                substrings.extend([sub_ngram] * count)
        return substrings
462340
Word2VecModel train(Log log, TrainingProgressListener listener, Iterable<List<String>> sentences) throws InterruptedException {
		try (ProfilingTimer timer = ProfilingTimer.createLoggingSubtasks(log, "Training word2vec")) {
			final Multiset<String> counts;
			
			try (AC ac = timer.start("Acquiring word frequencies")) {
				listener.update(Stage.ACQUIRE_VOCAB, 0.0);
				counts = (vocab.isPresent())
							? vocab.get()
							: count(Iterables.concat(sentences));
			}
			
			final ImmutableMultiset<String> vocab;
			try (AC ac = timer.start("Filtering and sorting vocabulary")) {
				listener.update(Stage.FILTER_SORT_VOCAB, 0.0);
				vocab = filterAndSort(counts);
			}
			
			final Map<String, HuffmanNode> huffmanNodes;
			try (AC task = timer.start("Create Huffman encoding")) {
				huffmanNodes = new HuffmanCoding(vocab, listener).encode();
			}
			
			final NeuralNetworkModel model;
			try (AC task = timer.start("Training model %s", neuralNetworkConfig)) {
				model = neuralNetworkConfig.createTrainer(vocab, huffmanNodes, listener).train(sentences);
			}
			
			return new Word2VecModel(vocab.elementSet(), model.layerSize(), Doubles.concat(model.vectors()));
		}
	}
1334718
def _build_unicode_character_database(self):
        
        filename = "UnicodeData.txt"
        current_dir = os.path.abspath(os.path.dirname(__file__))
        tag = re.compile(r"<\w+?>")
        with codecs.open(os.path.join(current_dir, filename), mode="r", encoding="utf-8") as fp:
            for line in fp:
                if not line.strip():
                    continue
                data = line.strip().split(";")
                # Replace the start/end range markers with their proper derived names.
                if data[1].endswith((u"First>", u"Last>")) and _is_derived(int(data[0], 16)):
                    data[1] = _get_nr_prefix(int(data[0], 16))
                    if data[1].startswith("HANGUL SYLLABLE"):  # For Hangul syllables, use naming rule NR1
                        data[1] += _get_hangul_syllable_name(int(data[0], 16))
                    else:  # Others should use naming rule NR2
                        data[1] += data[0]
                data[3] = int(data[3])  # Convert the Canonical Combining Class value into an int.
                if data[5]:  # Convert the contents of the decomposition into characters, preserving tag info.
                    data[5] = u" ".join([_hexstr_to_unichr(s) if not tag.match(s) else s for s in data[5].split()])
                for i in [6, 7, 8]:  # Convert the decimal, digit and numeric fields to either ints or fractions.
                    if data[i]:
                        if "/" in data[i]:
                            data[i] = Fraction(data[i])
                        else:
                            data[i] = int(data[i])
                for i in [12, 13, 14]:  # Convert the uppercase, lowercase and titlecase fields to characters.
                    if data[i]:
                        data[i] = _hexstr_to_unichr(data[i])
                lookup_name = _uax44lm2transform(data[1])
                uc_data = UnicodeCharacter(u"U+" + data[0], *data[1:])
                self._unicode_character_database[int(data[0], 16)] = uc_data
                self._name_database[lookup_name] = uc_data
        # Fill out the "compressed" ranges of UnicodeData.txt i.e. fill out the remaining characters per the Name
        # Derivation Rules.  See the Unicode Standard, ch. 4, section 4.8, Unicode Name Property
        for lookup_range, prefix_string in _nr_prefix_strings.items():
            exemplar = self._unicode_character_database.__getitem__(lookup_range[0])
            for item in lookup_range:
                hex_code = _padded_hex(item)
                new_name = prefix_string
                if prefix_string.startswith("HANGUL SYLLABLE"):  # For Hangul, use naming rule NR1
                    new_name += _get_hangul_syllable_name(item)
                else:  # Everything else uses naming rule NR2
                    new_name += hex_code
                uc_data = exemplar._replace(code=u"U+" + hex_code, name=new_name)
                self._unicode_character_database[item] = uc_data
                self._name_database[_uax44lm2transform(new_name)] = uc_data
387860
def edit_distance(wordA,wordB):
    "
    if not type(wordA) is list:
        lettersA = tamil.utf8.get_letters(wordA)
    else:
        lettersA = wordA
    
    if not type(wordB) is list:
        lettersB = tamil.utf8.get_letters(wordB)
    else:
        lettersB = wordB
    n_A = len(lettersA)
    n_B = len(lettersB)
    dist_AB = [[0 for i in range(0,n_B+1)] for i in range(0,(n_A+1))]
    # Target prefix reached by insertion
    for j in range(1,n_B+1):
        dist_AB[0][j] = j
    for i in range(1,n_A+1):
        dist_AB[i][0] = i
    for j in range(1,n_B+1):
        for i in range(1,n_A+1):
            if (lettersA[i-1] == lettersB[j-1]):
                new_dist = dist_AB[i-1][j-1]
            else:
                new_dist = min( [dist_AB[i-1][j]+1, dist_AB[i][j-1]+1, dist_AB[i-1][j-1]+1] ) #del, ins, or sub
            dist_AB[i][j] = new_dist 
    return dist_AB[-1][-1]