Hyper parameters specifics for long sequence generation.
190427
def transformer_moe_8k():
  """"""
  hparams = transformer_moe_base()

  hparams.batch_size = 8192
  hparams.max_length = 0  # max_length == batch_size
  hparams.eval_drop_long_sequences = True
  hparams.min_length_bucket = 256  # Avoid cyclic problems for big batches

  hparams.default_ff = "sep"
  hparams.hidden_size = 1024

  return hparams
1183350
function LSTM(linalg, params) {
  const Vector = linalg.Vector;

  function makeState() {
    return new Vector(params.nNodes * params.nLayers * 2);
  }

  function resetState(state) {
    linalg.zero(state, state);
    return state;
  }

  function copyState(state) {
    return new Vector(state);
  }

  function forward(inState, byte, outState) {
    var input = byteToVector(byte);
    for (var n = 0; n < params.nLayers; n++) {
      input = forwardLayer(indexState(inState, 2 * n),
                           indexState(inState, 2 * n + 1),
                           input,
                           params.affines[2 * n],
                           params.affines[2 * n + 1],
                           indexState(outState, 2 * n),
                           indexState(outState, 2 * n + 1));
    }
    return outState;
  }

  function predict(state) {
    var topH = indexState(state, 2 * params.nLayers - 1);
    var probs = params.affines[2 * params.nLayers](topH);
    linalg.exp(probs, probs);
    normalize(probs, probs);
    return probs;
  }

  function forwardLayer(prevC, prevH, x, i2h, h2h, nextC, nextH) {
    var allInputSums = linalg.vecAddElems(i2h(x), h2h(prevH));

    var inGate = linalg.sigmoid(indexState(allInputSums, 0));
    var forgetGate = linalg.sigmoid(indexState(allInputSums, 1));
    var outGate = linalg.sigmoid(indexState(allInputSums, 2));
    var inTransform = linalg.tanh(indexState(allInputSums, 3));

    linalg.vecAddElems(linalg.vecMultElems(forgetGate, prevC),
                       linalg.vecMultElems(inGate, inTransform),
                       nextC);
    linalg.vecMultElems(outGate, linalg.tanh(nextC), nextH);

    //console.log(nextC);
    //console.log(nextH);
    return nextH;
  }

  function indexState(state, n) {
    return state.subarray(n * params.nNodes, (n + 1) * params.nNodes);
  }

  function normalize(inVec, outVec) {
    var sum = inVec.reduce((x, y) => x + y);
    return linalg.scalarMult(inVec, 1 / sum, outVec);
  }

  function byteToIndex(byte) {
    return params.vocab[byte];
  }

  function indexToByte(index) {
    return params.ivocab[index];
  }

  function byteToVector(byte) {
    var vec = new Vector(params.affines[0].inLength);
    vec[byteToIndex(byte)] = 1.;
    return vec;
  }

  return {
    makeState: makeState,
    copyState: copyState,
    resetState: resetState,
    forward: forward,
    predict: predict,
    indexToByte: indexToByte,
    byteToIndex: byteToIndex,
  };
}
234146
def example_generator(self, encoder, tmp_dir, task_id):
    
    filepaths = self.text_filepaths_for_task(tmp_dir, task_id)
    if task_id >= self.num_train_shards:
      # this is dev data - limit the total length.
      max_chars_per_file = self.max_dev_chars // (
          self.num_dev_shards * len(filepaths))
    else:
      max_chars_per_file = None
    tokens = []
    for ftext in self.file_generator(
        filepaths, max_chars_per_file=max_chars_per_file):
      tokens.extend(encoder.encode(ftext))
      pos = 0
      while pos + self.sequence_length <= len(tokens):
        yield {"targets": tokens[pos:pos + self.sequence_length]}
        pos += self.sequence_length
      if pos > 0:
        tokens = tokens[pos:]
    if self.remainder_policy == "pad":
      if tokens:
        targets = tokens + [0] * (self.sequence_length - len(tokens))
        yield {"targets": targets}
    else:
      assert self.remainder_policy == "drop"
107736
def autoencoder_ordered_text():
  """"""
  hparams = autoencoder_ordered_discrete()
  hparams.bottleneck_bits = 1024
  hparams.bottleneck_shared_bits = 1024-64
  hparams.bottleneck_shared_bits_start_warmup = 75000
  hparams.bottleneck_shared_bits_stop_warmup = 275000
  hparams.num_hidden_layers = 7
  hparams.batch_size = 1024
  hparams.autoregressive_mode = "conv5"
  hparams.max_hidden_size = 1024
  hparams.bottom = {
      "inputs": modalities.identity_bottom,
      "targets": modalities.identity_bottom,
  }
  hparams.top = {
      "targets": modalities.identity_top,
  }
  hparams.sample_height = 128
  hparams.sample_width = 1
  return hparams
1012485
def _fetch_bounded_seq(self, var, start, end, window_size, boundary):
        
        var_len = end - start - window_size

        start = start if start >= boundary[0] else boundary[0]
        end = end if end <= boundary[1] else boundary[1]
        if start >= end:
            return ""

        seq = self.hdp.get_seq(var.ac, start, end)

        if len(seq) < end - start and len(seq) < var_len:
            raise HGVSInvalidVariantError(
                "Variant span is outside sequence bounds ({var})".format(var=var))

        return seq
62850
def cluster_seqs(seqs,
                 neighbor_join=False,
                 params={},
                 add_seq_names=True,
                 WorkingDir=tempfile.gettempdir(),
                 SuppressStderr=None,
                 SuppressStdout=None,
                 max_chars=1000000,
                 max_hours=1.0,
                 constructor=PhyloNode,
                 clean_up=True
                 ):
    
    num_seqs = len(seqs)
    if num_seqs < 2:
        raise ValueError, "Muscle requres 2 or more sequences to cluster."


    num_chars = sum(map(len, seqs))
    if num_chars > max_chars:
        params["-maxiters"] = 2
        params["-diags1"] = True
        params["-sv"] = True
        #params["-distance1"] = "kmer6_6"
        #params["-distance1"] = "kmer20_3"
        #params["-distance1"] = "kbit20_3"
        print "lots of chars, using fast align", num_chars


    params["-maxhours"] = max_hours
    #params["-maxiters"] = 10

    #cluster_type = "upgmb"
    #if neighbor_join:
    #    cluster_type = "neighborjoining"

    params["-clusteronly"] = True
    params["-tree1"] = get_tmp_filename(WorkingDir)

    muscle_res = muscle_seqs(seqs,
                 params=params,
                 add_seq_names=add_seq_names,
                 WorkingDir=WorkingDir,
                 SuppressStderr=SuppressStderr,
                 SuppressStdout=SuppressStdout)

    tree = DndParser(muscle_res["Tree1Out"], constructor=constructor)

    if clean_up:
        muscle_res.cleanUp()
    return tree
1053406
def super_lm_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.hidden_size = 512
  hparams.moe_hidden_sizes = "512"
  hparams.batch_size = 16384
  hparams.max_length = 0
  # All hyperparameters ending in "dropout" are automatically set to 0.0
  # when not in training mode.
  hparams.layer_prepostprocess_dropout = 0.0
  hparams.symbol_dropout = 0.1
  hparams.add_hparam("attention_dropout", 0.0)
  hparams.label_smoothing = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 8000
  hparams.initializer_gain = 1.0
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.shared_embedding_and_softmax_weights = False
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  # we only want one data shard.
  hparams.no_data_parallelism = True
  # bypass the symbol modality so that we can use model parallelism.
  hparams.bottom = {
      "inputs": modalities.identity_bottom,
      "targets": modalities.identity_bottom,
  }
  hparams.top = {
      "targets": modalities.identity_top,
  }
  hparams.add_hparam("filter_size", 512)
  hparams.add_hparam("mix_fraction", 0.5)
  # attention-related flags
  hparams.add_hparam("multihead_attention_num_heads", 4)
  hparams.add_hparam("multihead_attention_key_channels", 0)
  hparams.add_hparam("multihead_attention_value_channels", 0)
  hparams.add_hparam("pos", "timing")  # timing, none
  hparams.add_hparam(
      "layers", ("n,att,m,d,a," "n,ffn,m,d,a,") * 4 + "n,ffn,d")
  # Number of model shards - each one has separate parameters.
  # Changing this number invalidates checkpoints.
  hparams.add_hparam("num_model_shards", 8)
  hparams.add_hparam("diet_experts", False)
  return hparams
1083947
def sequencetyper(self):
        """"""
        logging.info('Performing sequence typing')
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                if type(sample[self.analysistype].allelenames) == list:
                    # Initialise variables
                    header = 0
                    genome = sample.name
                    # Initialise self.bestmatch[genome] with an int that will eventually be replaced by the # of matches
                    self.bestmatch[genome] = defaultdict(int)
                    if sample[self.analysistype].profile != 'NA':
                        # Create the profiledata variable to avoid writing self.profiledata[self.analysistype]
                        profiledata = sample[self.analysistype].profiledata
                        # Calculate the number of allele matches between each sequence type and the results
                        best_seqtype = dict()
                        for sequencetype in sample[self.analysistype].profiledata:
                            # Initialise a counter
                            count = 0
                            # Iterate through each gene for the sequence type
                            for gene, refallele in sample[self.analysistype].profiledata[sequencetype].items():
                                # Use the gene to extract the calculated allele
                                allele = self.matchdict[genome][gene]
                                # Increment the count on a match
                                if refallele == allele:
                                    count += 1
                            # Add the sequence type to the set of sequence types with the number of matches as the key
                            try:
                                best_seqtype[count].add(sequencetype)
                            except KeyError:
                                best_seqtype[count] = set()
                                best_seqtype[count].add(sequencetype)
                        # Find the highest number of matches from the dictionary
                        best = sorted(best_seqtype.items(), key=operator.itemgetter(0), reverse=True)[0][1]
                        # Deal with multiple allele matches
                        for gene in sample[self.analysistype].allelenames:
                            # Clear the appropriate count and lists
                            multiallele = list()
                            multipercent = list()
                            # Go through the alleles in plusdict
                            for allele in self.plusdict[genome][gene]:
                                percentid = list(self.plusdict[genome][gene][allele].keys())[0]
                                # "N" alleles screw up the allele splitter function
                                if allele not in ['N', 'NA']:
                                    # Append as appropriate - alleleNumber is treated as an integer for proper sorting
                                    multiallele.append(int(allele))
                                    multipercent.append(percentid)
                                # If the allele is "N"
                                else:
                                    # Append "N" and a percent identity of 0
                                    multiallele.append("N")
                                    multipercent.append(0)
                            # Populate self.bestdict with genome, gene, alleles joined with a space (this was made like
                            # this because allele is a list generated by the .iteritems() above
                            try:
                                self.bestdict[genome][gene][" ".join(str(allele)
                                                                     for allele in sorted(multiallele))] = \
                                    multipercent[0]
                            except IndexError:
                                self.bestdict[genome][gene]['NA'] = 0
                            # Find the profile with the most alleles in common with the query genome
                            for sequencetype in best:
                                # The number of genes in the analysis
                                header = len(profiledata[sequencetype])
                                # refallele is the allele number of the sequence type
                                refallele = profiledata[sequencetype][gene]
                                # If there are multiple allele matches for a gene in the reference profile e.g. 10 692
                                if len(refallele.split(" ")) > 1:
                                    # Map the split (on a space) alleles as integers - if they are treated as integers,
                                    # the alleles will sort properly
                                    intrefallele = map(int, refallele.split(" "))
                                    # Create a string of the joined, sorted alleles
                                    sortedrefallele = " ".join(str(allele) for allele in sorted(intrefallele))
                                else:
                                    # Use the reference allele as the sortedRefAllele
                                    sortedrefallele = refallele
                                for allele, percentid in self.bestdict[genome][gene].items():
                                    # If the allele in the query genome matches the allele in the reference profile, add
                                    # the result to the bestmatch dictionary. Genes with multiple alleles were sorted
                                    # the same, strings with multiple alleles will match: 10 692 will never be 692 10
                                    if allele == sortedrefallele and float(percentid) == 100.00:
                                        # Increment the number of matches to each profile
                                        self.bestmatch[genome][sequencetype] += 1
                                    # Special handling of BACT000060 and BACT000065 genes for E. coli and BACT000014
                                    # for Listeria. When the reference profile has an allele of 'N', and the query
                                    # allele doesn't, set the allele to 'N', and count it as a match
                                    elif sortedrefallele == 'N' and allele != 'N':
                                        # Increment the number of matches to each profile
                                        self.bestmatch[genome][sequencetype] += 1
                                    # Consider cases with multiple allele matches
                                    elif len(allele.split(' ')) > 1:
                                        # Also increment the number of matches if one of the alleles matches the
                                        # reference allele e.g. 41 16665 will match either 41 or 16665
                                        if sortedrefallele != 'N' and allele != 'N':
                                            match = False
                                            for sub_allele in allele.split(' '):
                                                if sub_allele == refallele:
                                                    match = True
                                            if match:
                                                # Increment the number of matches to each profile
                                                self.bestmatch[genome][sequencetype] += 1
                                    elif allele == sortedrefallele and sortedrefallele == 'N':
                                        # Increment the number of matches to each profile
                                        self.bestmatch[genome][sequencetype] += 1
                        # Get the best number of matches
                        # From: https://stackoverflow.com/questions/613183/sort-a-python-dictionary-by-value
                        try:
                            sortedmatches = sorted(self.bestmatch[genome].items(), key=operator.itemgetter(1),
                                                   reverse=True)[0][1]
                        # If there are no matches, set :sortedmatches to zero
                        except IndexError:
                            sortedmatches = 0
                        # Otherwise, the query profile matches the reference profile
                        if int(sortedmatches) == header:
                            # Iterate through best match
                            for sequencetype, matches in self.bestmatch[genome].items():
                                if matches == sortedmatches:
                                    for gene in profiledata[sequencetype]:
                                        # Populate resultProfile with the genome, best match to profile, # of matches
                                        # to the profile, gene, query allele(s), reference allele(s), and % identity
                                        self.resultprofile[genome][sequencetype][sortedmatches][gene][
                                            list(self.bestdict[genome][gene]
                                                 .keys())[0]] = str(list(self.bestdict[genome][gene].values())[0])
                                    sample[self.analysistype].sequencetype = sequencetype
                                    sample[self.analysistype].matchestosequencetype = matches
                        # If there are fewer matches than the total number of genes in the typing scheme
                        elif 0 < int(sortedmatches) < header:
                            mismatches = []
                            # Iterate through the sequence types and the number of matches in bestDict for each genome
                            for sequencetype, matches in self.bestmatch[genome].items():
                                # If the number of matches for a profile matches the best number of matches
                                if matches == sortedmatches:
                                    # Iterate through the gene in the analysis
                                    for gene in profiledata[sequencetype]:
                                        # Get the reference allele as above
                                        refallele = profiledata[sequencetype][gene]
                                        # As above get the reference allele split and ordered as necessary
                                        if len(refallele.split(" ")) > 1:
                                            intrefallele = map(int, refallele.split(" "))
                                            sortedrefallele = " ".join(str(allele) for allele in sorted(intrefallele))
                                        else:
                                            sortedrefallele = refallele
                                        # Populate self.mlstseqtype with the genome, best match to profile, # of matches
                                        # to the profile, gene, query allele(s), reference allele(s), and % identity
                                        if self.analysistype == 'mlst':
                                            self.resultprofile[genome][sequencetype][sortedmatches][gene][
                                                list(self.bestdict[genome][gene]
                                                     .keys())[0]] = str(list(self.bestdict[genome][gene].values())[0])
                                        else:
                                            self.resultprofile[genome][sequencetype][sortedmatches][gene][
                                                list(self.bestdict[genome][gene].keys())[0]] \
                                                = str(list(self.bestdict[genome][gene].values())[0])
                                            #
                                            if sortedrefallele != list(self.bestdict[sample.name][gene].keys())[0]:
                                                mismatches.append(
                                                    ({gene: ('{} ({})'.format(list(self.bestdict[sample.name][gene]
                                                                                   .keys())[0], sortedrefallele))}))
                                        sample[self.analysistype].mismatchestosequencetype = mismatches
                                        sample[self.analysistype].sequencetype = sequencetype
                                        sample[self.analysistype].matchestosequencetype = matches
                        elif sortedmatches == 0:
                            for gene in sample[self.analysistype].allelenames:
                                # Populate the results profile with negative values for sequence type and sorted matches
                                self.resultprofile[genome]['NA'][sortedmatches][gene]['NA'] = 0
                            # Add the new profile to the profile file (if the option is enabled)
                            sample[self.analysistype].sequencetype = 'NA'
                            sample[self.analysistype].matchestosequencetype = 'NA'
                            sample[self.analysistype].mismatchestosequencetype = 'NA'
                        else:
                            sample[self.analysistype].matchestosequencetype = 'NA'
                            sample[self.analysistype].mismatchestosequencetype = 'NA'
                            sample[self.analysistype].sequencetype = 'NA'
                else:
                    sample[self.analysistype].matchestosequencetype = 'NA'
                    sample[self.analysistype].mismatchestosequencetype = 'NA'
                    sample[self.analysistype].sequencetype = 'NA'

            else:
                sample[self.analysistype].matchestosequencetype = 'NA'
                sample[self.analysistype].mismatchestosequencetype = 'NA'
                sample[self.analysistype].sequencetype = 'NA'
            # Clear out the reverse_profiledata attribute - frozen sets can not be .json encoded
            try:
                delattr(sample[self.analysistype], 'reverse_profiledata')
            except AttributeError:
                pass
1176321
def generate_antisense_sequence(sequence):
    """"""
    dna_antisense = {
        'A': 'T',
        'T': 'A',
        'C': 'G',
        'G': 'C'
    }
    antisense = [dna_antisense[x] for x in sequence[::-1]]
    return ''.join(antisense)
73950
def _seqfeature_to_coral(feature):
    '''

    '''
    # Some genomic sequences don't have a label attribute
    # TODO: handle genomic cases differently than others. Some features lack
    # a label but should still be incorporated somehow.
    qualifiers = feature.qualifiers
    if 'label' in qualifiers:
        feature_name = qualifiers['label'][0]
    elif 'locus_tag' in qualifiers:
        feature_name = qualifiers['locus_tag'][0]
    else:
        raise FeatureNameError('Unrecognized feature name')
    # Features with gaps are special, require looking at subfeatures
    # Assumption: subfeatures are never more than one level deep
    if feature.location_operator == 'join':
        # Feature has gaps. Have to figure out start/stop from subfeatures,
        # calculate gap indices. A nested feature model may be required
        # eventually.
        # Reorder the sub_feature list by start location
        # Assumption: none of the subfeatures overlap so the last entry in
        # the reordered list also has the final stop point of the feature.
        # FIXME: Getting a deprecation warning about using sub_features
        # instead of feature.location being a CompoundFeatureLocation
        reordered = sorted(feature.location.parts,
                           key=lambda location: location.start)
        starts = [int(location.start) for location in reordered]
        stops = [int(location.end) for location in reordered]
        feature_start = starts.pop(0)
        feature_stop = stops.pop(-1)
        starts = [start - feature_start for start in starts]
        stops = [stop - feature_start for stop in stops]
        feature_gaps = list(zip(stops, starts))
    else:
        # Feature doesn't have gaps. Ignore subfeatures.
        feature_start = int(feature.location.start)
        feature_stop = int(feature.location.end)
        feature_gaps = []
    feature_type = _process_feature_type(feature.type)
    if feature.location.strand == -1:
        feature_strand = 1
    else:
        feature_strand = 0
    if 'gene' in qualifiers:
        gene = qualifiers['gene']
    else:
        gene = []
    if 'locus_tag' in qualifiers:
        locus_tag = qualifiers['locus_tag']
    else:
        locus_tag = []
    coral_feature = coral.Feature(feature_name, feature_start,
                                  feature_stop, feature_type,
                                  gene=gene, locus_tag=locus_tag,
                                  qualifiers=qualifiers,
                                  strand=feature_strand,
                                  gaps=feature_gaps)
    return coral_feature
1143147
def generate_flat_data(self):
    
    # Construct the string statements.
    all_statements = []
    all_targets = []
    self.sequence_sizes_in = []
    self.sequence_sizes_out = []
    for _ in six.moves.range(self._batch_size):
      length, nest = self.curriculum_obj.fetch()
      seq_size_in = self._max_seq_length
      # Generate batch within max length.
      is_valid_sample = False
      tries_remaining = 10
      while not is_valid_sample:
        value, code = generate_code(length, nest, self._ops)
        tokens_in, seq_size_in = self.tokenize(
            code, self._max_seq_length, self._token_by_char)
        tokens_out, seq_size_out = self.tokenize(
            value, self._max_seq_length, self._token_by_char)
        is_valid_sample = self._max_seq_length >= seq_size_in
        if is_valid_sample:
          self.sequence_sizes_in.append(seq_size_in)
          self.sequence_sizes_out.append(seq_size_out)
        if tries_remaining == 0:
          raise ValueError("Could not generate a sample below the allowable "
                           "maximum, consider reducing either max_length or "
                           "max_nest.")
        else:
          tries_remaining -= 1
      all_statements += tokens_in
      all_targets += tokens_out
    # Store the flattened data.
    self.flat_data = np.array(all_statements, dtype=np.int64)
    self.num_tokens = self.flat_data.shape[0]
    self.flat_targets = np.array(all_targets, dtype=np.int64)
    self.num_tokens_target = self.flat_targets.shape[0]
    self.start_token = np.array(self.tokenize(
        [get_start_token()], 1)[0], dtype=np.int64)
    self.end_token = np.array(self.tokenize(
        [get_end_token()], 1)[0], dtype=np.int64)
879318
def convert_examples_to_features(examples, seq_length, tokenizer):
    """"""

    features = []
    for (ex_index, example) in enumerate(examples):
        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)

        if tokens_b:
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > seq_length - 2:
                tokens_a = tokens_a[0:(seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids:   0   0  0    0    0     0      0   0    1  1  1   1  1   1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids:   0   0   0   0  0     0   0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambigiously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = []
        input_type_ids = []
        tokens.append("[CLS]")
        input_type_ids.append(0)
        for token in tokens_a:
            tokens.append(token)
            input_type_ids.append(0)
        tokens.append("[SEP]")
        input_type_ids.append(0)

        if tokens_b:
            for token in tokens_b:
                tokens.append(token)
                input_type_ids.append(1)
            tokens.append("[SEP]")
            input_type_ids.append(1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        while len(input_ids) < seq_length:
            input_ids.append(0)
            input_mask.append(0)
            input_type_ids.append(0)

        assert len(input_ids) == seq_length
        assert len(input_mask) == seq_length
        assert len(input_type_ids) == seq_length

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("unique_id: %s" % (example.unique_id))
            logger.info("tokens: %s" % " ".join([str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids]))

        features.append(
            InputFeatures(
                unique_id=example.unique_id,
                tokens=tokens,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids))
    return features
946108
def bytenet_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.batch_size = 2048
  hparams.hidden_size = 768
  hparams.dropout = 0.2
  hparams.symbol_dropout = 0.2
  hparams.label_smoothing = 0.1
  hparams.clip_grad_norm = 2.0
  hparams.num_hidden_layers = 4
  hparams.kernel_height = 3
  hparams.kernel_width = 1
  hparams.learning_rate_decay_scheme = "exp"
  hparams.learning_rate = 0.05
  hparams.learning_rate_warmup_steps = 3000
  hparams.initializer_gain = 1.0
  hparams.weight_decay = 3.0
  hparams.num_sampled_classes = 0
  hparams.sampling_method = "argmax"
  hparams.optimizer_adam_epsilon = 1e-6
  hparams.optimizer_adam_beta1 = 0.85
  hparams.optimizer_adam_beta2 = 0.997
  hparams.add_hparam("num_block_repeat", 4)
  return hparams
680171
def _iter_content_generator(response, decode_unicode):
    """"""
    for chunk in response.iter_content(100 * 1024, decode_unicode=decode_unicode):
        if decode_unicode:
            # Replace CRLF newlines with LF, Python will handle
            # platform specific newlines if written to file.
            chunk = chunk.replace('\r\n', '\n')
            # Chunk could be ['...\r', '\n...'], stril trailing \r
            chunk = chunk.rstrip('\r')
        yield chunk
50586
def build_hgvsp(self):
        

        variants = []

        if not self._is_ambiguous and len(self._alt_seq) > 0:

            do_delins = True
            if self._ref_seq == self._alt_seq:
                # Silent p. variant
                start = self._alt_data.variant_start_aa
                if start - 1 < len(self._ref_seq):
                    deletion = self._ref_seq[start - 1]
                    insertion = deletion
                else:
                    start = ""
                    deletion = ""
                    insertion = ""
                    self._is_frameshift = False
                variants.append({"start": start, "ins": insertion, "del": deletion})
                do_delins = False
            elif self._is_substitution:
                if len(self._ref_seq) == len(self._alt_seq):
                    diff_pos = [(i, self._ref_seq[i], self._alt_seq[i])
                                for i in range(len(self._ref_seq))
                                if self._ref_seq[i] != self._alt_seq[i]]
                    if len(diff_pos) == 1:
                        (start, deletion, insertion) = diff_pos[0]
                        variants.append({"start": start + 1, "ins": insertion, "del": deletion})
                        do_delins = False

            if do_delins:
                if self._alt_data.is_frameshift:
                    start = self._alt_data.variant_start_aa - 1
                    while self._ref_seq[start] == self._alt_seq[start]:
                        start += 1
                    insertion = list(self._alt_seq[start:])
                    deletion = list(self._ref_seq[start:])
                    variants.append({"start": start + 1, "ins": insertion, "del": deletion})

                else:    # non-frameshifting delins or dup
                    # get size diff from diff in ref/alt lengths
                    start = self._alt_data.variant_start_aa - 1
                    delta = len(self._alt_seq) - len(self._ref_seq)
                    while self._ref_seq[start] == self._alt_seq[start]:
                        start += 1
                    offset = start + abs(delta)

                    if delta > 0:    # net insertion
                        insertion = list(self._alt_seq[start:offset])
                        deletion = []
                        ref_sub = self._ref_seq[start:]
                        alt_sub = self._alt_seq[offset:]
                    elif delta < 0:    # net deletion
                        insertion = []
                        deletion = list(self._ref_seq[start:offset])
                        ref_sub = self._ref_seq[offset:]
                        alt_sub = self._alt_seq[start:]
                    else:
                        insertion = []
                        deletion = []
                        ref_sub = self._ref_seq[start:]
                        alt_sub = self._alt_seq[start:]

                    # from start, get del/ins out to last difference
                    diff_indices = [i for i in range(len(ref_sub)) if ref_sub[i] != alt_sub[i]]
                    if not diff_indices and not deletion and insertion[0] == '*':
                        diff_indices.append(0)
                    if diff_indices:
                        max_diff = diff_indices[-1] + 1
                        insertion.extend(list(alt_sub[:max_diff]))
                        deletion.extend(list(ref_sub[:max_diff]))

                    variants.append({"start": start + 1, "ins": insertion, "del": deletion})

            if DBG:
                print(variants)

        if self._is_ambiguous:
            var_ps = [
                self._create_variant(
                    None,
                    None,
                    '',
                    '',
                    acc=self._protein_accession,
                    is_ambiguous=self._is_ambiguous)
            ]
        elif len(self._alt_seq) == 0:
            var_ps = [
                self._create_variant(
                    None,
                    None,
                    '',
                    '',
                    acc=self._protein_accession,
                    is_ambiguous=self._is_ambiguous,
                    is_no_protein=True)
            ]
        else:
            var_ps = [
                self._convert_to_sequence_variants(x, self._protein_accession) for x in variants
            ]

        if len(var_ps) > 1:
            raise HGVSError("Got multiple AA variants - not supported")
        return var_ps[0]
429758
def generate_i
      return (begin
        bytes = (time_low << 96) + (time_mid << 80) +
          (time_hi_and_version << 64) + (clock_seq_hi_and_reserved << 56) +
          (clock_seq_low << 48)
        for i in 0..5
          bytes += (nodes[i] << (40 - (i * 8)))
        end
        bytes
      end)
    end
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict