A Toil job function performing ADAM preprocessing on a single sample
425726
def static_adam_preprocessing_dag(job, inputs, sample, output_dir, suffix=''):
    
    inputs.sample = sample
    inputs.output_dir = output_dir
    inputs.suffix = suffix

    if inputs.master_ip is not None or inputs.run_local:
        if not inputs.run_local and inputs.master_ip == 'auto':
            # Static, standalone Spark cluster managed by uberscript
            spark_on_toil = False
            scale_up = job.wrapJobFn(scale_external_spark_cluster, 1)
            job.addChild(scale_up)
            spark_work = job.wrapJobFn(download_run_and_upload,
                                       inputs.master_ip, inputs, spark_on_toil)
            scale_up.addChild(spark_work)
            scale_down = job.wrapJobFn(scale_external_spark_cluster, -1)
            spark_work.addChild(scale_down)
        else:
            # Static, external Spark cluster
            spark_on_toil = False
            spark_work = job.wrapJobFn(download_run_and_upload,
                                       inputs.master_ip, inputs, spark_on_toil)
            job.addChild(spark_work)
    else:
        # Dynamic subclusters, i.e. Spark-on-Toil
        spark_on_toil = True
        cores = multiprocessing.cpu_count()
        master_ip = spawn_spark_cluster(job,
                                        False, # Sudo
                                        inputs.num_nodes-1,
                                        cores=cores,
                                        memory=inputs.memory)
        spark_work = job.wrapJobFn(download_run_and_upload,
                                   master_ip, inputs, spark_on_toil)
        job.addChild(spark_work)
716071
def _parse_proto(prototxt_fname):
    
    proto = caffe_parser.read_prototxt(prototxt_fname)

    # process data layer
    input_name, input_dim, layers = _get_input(proto)
    # only support single input, so always use `data` as the input data
    mapping = {input_name: 'data'}
    need_flatten = {input_name: False}
    symbol_string = "import mxnet as mx\ndata = mx.symbol.Variable(name='data')\n"

    flatten_count = 0
    output_name = ""
    prev_name = None
    _output_name = {}

    # convert reset layers one by one
    for i, layer in enumerate(layers):
        type_string = ''
        param_string = ''
        skip_layer = False
        name = re.sub('[-/]', '_', layer.name)
        for k in range(len(layer.bottom)):
            if layer.bottom[k] in _output_name:
                _output_name[layer.bottom[k]]['count'] = _output_name[layer.bottom[k]]['count']+1
            else:
                _output_name[layer.bottom[k]] = {'count':0}
        for k in range(len(layer.top)):
            if layer.top[k] in _output_name:
                _output_name[layer.top[k]]['count'] = _output_name[layer.top[k]]['count']+1
            else:
                _output_name[layer.top[k]] = {'count':0, 'name':name}
        if layer.type == 'Convolution' or layer.type == 4:
            type_string = 'mx.symbol.Convolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Deconvolution' or layer.type == 39:
            type_string = 'mx.symbol.Deconvolution'
            param_string = _convert_conv_param(layer.convolution_param)
            need_flatten[name] = True
        if layer.type == 'Pooling' or layer.type == 17:
            type_string = 'mx.symbol.Pooling'
            param_string = _convert_pooling_param(layer.pooling_param)
            need_flatten[name] = True
        if layer.type == 'ReLU' or layer.type == 18:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='relu'"
            param = layer.relu_param
            if hasattr(param, 'negative_slope'):
                if param.negative_slope > 0:
                    type_string = 'mx.symbol.LeakyReLU'
                    param_string = "act_type='leaky', slope=%f" % param.negative_slope
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'TanH' or layer.type == 23:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='tanh'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Sigmoid' or layer.type == 19:
            type_string = 'mx.symbol.Activation'
            param_string = "act_type='sigmoid'"
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'LRN' or layer.type == 15:
            type_string = 'mx.symbol.LRN'
            param = layer.lrn_param
            param_string = "alpha=%f, beta=%f, knorm=%f, nsize=%d" % (
                param.alpha, param.beta, param.k, param.local_size)
            need_flatten[name] = True
        if layer.type == 'InnerProduct' or layer.type == 14:
            type_string = 'mx.symbol.FullyConnected'
            param = layer.inner_product_param
            param_string = "num_hidden=%d, no_bias=%s" % (
                param.num_output, not param.bias_term)
            need_flatten[name] = False
        if layer.type == 'Dropout' or layer.type == 6:
            type_string = 'mx.symbol.Dropout'
            param = layer.dropout_param
            param_string = "p=%f" % param.dropout_ratio
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Softmax' or layer.type == 20:
            type_string = 'mx.symbol.SoftmaxOutput'
        if layer.type == 'Flatten' or layer.type == 8:
            type_string = 'mx.symbol.Flatten'
            need_flatten[name] = False
        if layer.type == 'Split' or layer.type == 22:
            type_string = 'split'  # will process later
        if layer.type == 'Concat' or layer.type == 3:
            type_string = 'mx.symbol.Concat'
            need_flatten[name] = True
        if layer.type == 'Crop':
            type_string = 'mx.symbol.Crop'
            need_flatten[name] = True
            param_string = 'center_crop=True'
        if layer.type == 'BatchNorm':
            type_string = 'mx.symbol.BatchNorm'
            param = layer.batch_norm_param
            # CuDNN requires eps to be greater than 1e-05
            # We compensate for this change in convert_model
            epsilon = param.eps
            if (epsilon <= 1e-05):
                epsilon = 1e-04
            # if next layer is scale, don't fix gamma
            fix_gamma = layers[i+1].type != 'Scale'
            param_string = 'use_global_stats=%s, fix_gamma=%s, eps=%f' % (
                param.use_global_stats, fix_gamma, epsilon)
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Scale':
            assert layers[i-1].type == 'BatchNorm'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
            skip_layer = True
            prev_name = re.sub('[-/]', '_', layers[i-1].name)
        if layer.type == 'PReLU':
            type_string = 'mx.symbol.LeakyReLU'
            param = layer.prelu_param
            param_string = "act_type='prelu', slope=%f" % param.filler.value
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]
        if layer.type == 'Eltwise':
            type_string = 'mx.symbol.broadcast_add'
            param = layer.eltwise_param
            param_string = ""
            need_flatten[name] = False
        if layer.type == 'Reshape':
            type_string = 'mx.symbol.Reshape'
            need_flatten[name] = False
            param = layer.reshape_param
            param_string = "shape=(%s)" % (','.join(param.shape.dim),)
        if layer.type == 'AbsVal':
            type_string = 'mx.symbol.abs'
            need_flatten[name] = need_flatten[mapping[layer.bottom[0]]]

        if skip_layer:
            assert len(layer.bottom) == 1
            symbol_string += "%s = %s\n" % (name, prev_name)
        elif type_string == '':
            raise ValueError('Unknown layer %s!' % layer.type)
        elif type_string != 'split':
            bottom = layer.bottom
            if param_string != "":
                param_string = ", " + param_string
            if len(bottom) == 1:
                if need_flatten[mapping[bottom[0]]] and type_string == 'mx.symbol.FullyConnected':
                    flatten_name = "flatten_%d" % flatten_count
                    symbol_string += "%s=mx.symbol.Flatten(name='%s', data=%s)\n" % (
                        flatten_name, flatten_name, mapping[bottom[0]])
                    flatten_count += 1
                    need_flatten[flatten_name] = False
                    bottom[0] = flatten_name
                    mapping[bottom[0]] = bottom[0]
                symbol_string += "%s = %s(name='%s', data=%s %s)\n" % (
                    name, type_string, name, mapping[bottom[0]], param_string)
            else:
                if layer.type == 'Eltwise' and param.operation == 1 and len(param.coeff) > 0:
                    symbol_string += "%s = " % name
                    symbol_string += " + ".join(["%s * %s" % (
                        mapping[bottom[i]], param.coeff[i]) for i in range(len(param.coeff))])
                    symbol_string += "\n"
                else:
                    symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % (
                        name, type_string, name, ','.join(
                            [mapping[x] for x in bottom]), param_string)
        for j in range(len(layer.top)):
            mapping[layer.top[j]] = name
        output_name = name
    output_name = []
    for i in _output_name:
        if 'name' in _output_name[i] and _output_name[i]['count'] == 0:
            output_name.append(_output_name[i]['name'])

    return symbol_string, output_name, input_dim
494489
def generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim=100, activation='sigmoid'):
    '''
    '''
    BatchNorm = mx.sym.BatchNorm
    rand = mx.sym.Variable('rand')

    rand = mx.sym.Reshape(rand, shape=(-1, z_dim, 1, 1))

    g1 = mx.sym.Deconvolution(rand, name='gen1', kernel=(5,5), stride=(2,2),target_shape=(2,2), num_filter=ngf*8, no_bias=no_bias)
    gbn1 = BatchNorm(g1, name='genbn1', fix_gamma=fix_gamma, eps=eps)
    gact1 = mx.sym.Activation(gbn1, name="genact1", act_type="relu")

    g2 = mx.sym.Deconvolution(gact1, name='gen2', kernel=(5,5), stride=(2,2),target_shape=(4,4), num_filter=ngf*4, no_bias=no_bias)
    gbn2 = BatchNorm(g2, name='genbn2', fix_gamma=fix_gamma, eps=eps)
    gact2 = mx.sym.Activation(gbn2, name='genact2', act_type='relu')

    g3 = mx.sym.Deconvolution(gact2, name='gen3', kernel=(5,5), stride=(2,2), target_shape=(8,8), num_filter=ngf*2, no_bias=no_bias)
    gbn3 = BatchNorm(g3, name='genbn3', fix_gamma=fix_gamma, eps=eps)
    gact3 = mx.sym.Activation(gbn3, name='genact3', act_type='relu')

    g4 = mx.sym.Deconvolution(gact3, name='gen4', kernel=(5,5), stride=(2,2), target_shape=(16,16), num_filter=ngf, no_bias=no_bias)
    gbn4 = BatchNorm(g4, name='genbn4', fix_gamma=fix_gamma, eps=eps)
    gact4 = mx.sym.Activation(gbn4, name='genact4', act_type='relu')

    g5 = mx.sym.Deconvolution(gact4, name='gen5', kernel=(5,5), stride=(2,2), target_shape=(32,32), num_filter=nc, no_bias=no_bias)
    gout = mx.sym.Activation(g5, name='genact5', act_type=activation)

    return gout
69253
def transformer_wikitext103_l4k_v0():
  """"""
  hparams = transformer_big()

  # Adafactor uses less memory than Adam.
  # switch to Adafactor with its recommended learning rate scheme.
  hparams.optimizer = "Adafactor"
  hparams.learning_rate_schedule = "rsqrt_decay"
  hparams.learning_rate_warmup_steps = 10000

  hparams.num_heads = 4
  hparams.max_length = 4096
  hparams.batch_size = 4096
  hparams.shared_embedding_and_softmax_weights = False

  hparams.num_hidden_layers = 8
  hparams.attention_dropout = 0.1
  hparams.layer_prepostprocess_dropout = 0.2
  hparams.relu_dropout = 0.1
  hparams.label_smoothing = 0.0

  # Using noise broadcast in the dropout layers saves memory during training.
  hparams.attention_dropout_broadcast_dims = "0,1"  # batch, heads
  hparams.relu_dropout_broadcast_dims = "1"  # length
  hparams.layer_prepostprocess_dropout_broadcast_dims = "1"  # length

  # Avoid an expensive concat on TPU.
  # >1 shards helps with faster parameter distribution on multi-GPU machines
  hparams.symbol_modality_num_shards = 1

  return hparams
1084368
def iterate(self, params, repetition, iteration):
    
    print("\nStarting iteration",iteration)
    print("Learning rate:", self.learningRate if self.lr_scheduler is None
                                              else self.lr_scheduler.get_lr())
    t1 = time.time()
    ret = {}

    # Update dataset epoch when using pre-processed speech dataset
    if self.use_preprocessed_dataset:
      t2 = time.time()
      self.train_loader.dataset.next_epoch()
      self.validation_loader.dataset.next_epoch()
      self.test_loader.dataset.next_epoch()
      self.bg_noise_loader.dataset.next_epoch()
      print("Dataset Load time = {0:.3f} secs, ".format(time.time() - t2))

    # Update learning rate using learning rate scheduler if configured
    if self.lr_scheduler is not None:
      # ReduceLROnPlateau lr_scheduler step should be called after validation,
      # all other lr_schedulers should be called before training
      if params["lr_scheduler"] != "ReduceLROnPlateau":
        self.lr_scheduler.step()

    self.train(params, epoch=iteration, repetition=repetition)

    # Run validation test
    if self.validation_loader is not None:
      validation = self.test(params, self.validation_loader)

      # ReduceLROnPlateau step should be called after validation
      if params["lr_scheduler"] == "ReduceLROnPlateau":
        self.lr_scheduler.step(validation["test_loss"])

      ret["validation"] = validation
      print("Validation: error=", validation["testerror"],
            "entropy=", validation["entropy"],
            "loss=", validation["test_loss"])
      ret.update({"validationerror": validation["testerror"]})

    # Run test set
    if self.test_loader is not None:
      testResults = self.test(params, self.test_loader)
      ret["testResults"] = testResults
      print("Test: error=", testResults["testerror"],
            "entropy=", testResults["entropy"],
            "loss=", testResults["test_loss"])
      ret.update({"testerror": testResults["testerror"]})

    # Run bg noise set
    if self.bg_noise_loader is not None:
      bgResults = self.test(params, self.bg_noise_loader)
      ret["bgResults"] = bgResults
      print("BG noise error=", bgResults["testerror"])
      ret.update({"bgerror": bgResults["testerror"]})

    ret.update({"elapsedTime": time.time() - self.startTime})
    ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                  else self.lr_scheduler.get_lr()})

    # Run noise set
    if params.get("run_noise_tests", False):
      ret.update(self.runNoiseTests(params))
      print("Noise test results: totalCorrect=", ret["totalCorrect"],
            "Test error=", ret["testerror"], ", entropy=", ret["entropy"])

    ret.update({"elapsedTime": time.time() - self.startTime})
    ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                  else self.lr_scheduler.get_lr()})

    print("Iteration time= {0:.3f} secs, "
          "total elapsed time= {1:.3f} mins".format(
            time.time() - t1,ret["elapsedTime"]/60.0))

    return ret
932964
def preprocess(train_dataset, output_dir, eval_dataset, checkpoint):
    """"""

    import apache_beam as beam
    from google.datalab.utils import LambdaJob
    from . import _preprocess

    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL
    job_id = ('preprocess-image-classification-' +
              datetime.datetime.now().strftime('%y%m%d-%H%M%S'))
    # Project is needed for bigquery data source, even in local run.
    options = {
        'project': _util.default_project(),
    }
    opts = beam.pipeline.PipelineOptions(flags=[], **options)
    p = beam.Pipeline('DirectRunner', options=opts)
    _preprocess.configure_pipeline(p, train_dataset, eval_dataset, checkpoint, output_dir, job_id)
    job = LambdaJob(lambda: p.run().wait_until_finish(), job_id)
    return job
467926
def parse_rev_args(receive_msg):
    
    global trainloader
    global testloader
    global net
    global criterion
    global optimizer

    # Loading Data
    logger.debug("Preparing data..")

    transform_train, transform_test = utils.data_transforms_cifar10(args)

    trainset = torchvision.datasets.CIFAR10(
        root="./data", train=True, download=True, transform=transform_train
    )
    trainloader = torch.utils.data.DataLoader(
        trainset, batch_size=args.batch_size, shuffle=True, num_workers=2
    )

    testset = torchvision.datasets.CIFAR10(
        root="./data", train=False, download=True, transform=transform_test
    )
    testloader = torch.utils.data.DataLoader(
        testset, batch_size=args.batch_size, shuffle=False, num_workers=2
    )

    # Model
    logger.debug("Building model..")
    net = build_graph_from_json(receive_msg)

    net = net.to(device)
    criterion = nn.CrossEntropyLoss()
    if device == "cuda" and torch.cuda.device_count() > 1:
        net = torch.nn.DataParallel(net)

    if args.optimizer == "SGD":
        optimizer = optim.SGD(
            net.parameters(), lr=args.learning_rate, momentum=0.9, weight_decay=5e-4
        )
    if args.optimizer == "Adadelta":
        optimizer = optim.Adadelta(net.parameters(), lr=args.learning_rate)
    if args.optimizer == "Adagrad":
        optimizer = optim.Adagrad(net.parameters(), lr=args.learning_rate)
    if args.optimizer == "Adam":
        optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
    if args.optimizer == "Adamax":
        optimizer = optim.Adamax(net.parameters(), lr=args.learning_rate)
    if args.optimizer == "RMSprop":
        optimizer = optim.RMSprop(net.parameters(), lr=args.learning_rate)


    return 0
522538
def adam_convert(job, master_ip, inputs, in_file, in_snps, adam_file, adam_snps, spark_on_toil):
    

    log.info("Converting input BAM to ADAM.")
    call_adam(job, master_ip,
              ["transform", in_file, adam_file],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    in_file_name = in_file.split("/")[-1]
    remove_file(master_ip, in_file_name, spark_on_toil)

    log.info("Converting known sites VCF to ADAM.")

    call_adam(job, master_ip,
              ["vcf2adam", "-only_variants", in_snps, adam_snps],
              memory=inputs.memory,
              run_local=inputs.run_local,
              native_adam_path=inputs.native_adam_path)

    in_snps_name = in_snps.split("/")[-1]
    remove_file(master_ip, in_snps_name, spark_on_toil)
820025
def prepare_data(args):
    
    rnn_type = args.config.get("arch", "rnn_type")
    num_rnn_layer = args.config.getint("arch", "num_rnn_layer")
    num_hidden_rnn_list = json.loads(args.config.get("arch", "num_hidden_rnn_list"))

    batch_size = args.config.getint("common", "batch_size")

    if rnn_type == 'lstm':
        init_c = [('l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l]))
                  for l in range(num_rnn_layer)]
        init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                  for l in range(num_rnn_layer)]
    elif rnn_type == 'bilstm':
        forward_init_c = [('forward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l]))
                          for l in range(num_rnn_layer)]
        backward_init_c = [('backward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l]))
                           for l in range(num_rnn_layer)]
        init_c = forward_init_c + backward_init_c
        forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                          for l in range(num_rnn_layer)]
        backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                           for l in range(num_rnn_layer)]
        init_h = forward_init_h + backward_init_h
    elif rnn_type == 'gru':
        init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                  for l in range(num_rnn_layer)]
    elif rnn_type == 'bigru':
        forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                          for l in range(num_rnn_layer)]
        backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l]))
                           for l in range(num_rnn_layer)]
        init_h = forward_init_h + backward_init_h
    else:
        raise Exception('network type should be one of the lstm,bilstm,gru,bigru')

    if rnn_type == 'lstm' or rnn_type == 'bilstm':
        init_states = init_c + init_h
    elif rnn_type == 'gru' or rnn_type == 'bigru':
        init_states = init_h
    return init_states
221528
def train(net, X_train, y_train, epochs, verbose_epoch, learning_rate,
          weight_decay, batch_size):
    """"""
    dataset_train = gluon.data.ArrayDataset(X_train, y_train)
    data_iter_train = gluon.data.DataLoader(dataset_train, batch_size,
                                            shuffle=True)
    trainer = gluon.Trainer(net.collect_params(), 'adam',
                            {'learning_rate': learning_rate,
                             'wd': weight_decay})
    net.initialize(force_reinit=True)
    for epoch in range(epochs):
        for data, label in data_iter_train:
            with autograd.record():
                output = net(data)
                loss = square_loss(output, label)
            loss.backward()
            trainer.step(batch_size)
            avg_loss = get_rmse_log(net, X_train, y_train)
        if epoch > verbose_epoch:
            print("Epoch %d, train loss: %f" % (epoch, avg_loss))
    return avg_loss
1006817
function lineToTodoMapper (filename, sha, conf) {
  return function lineToTodo (line) {
    return _.merge({
      "file":   filename,
      "sha":    sha,
      "line":   line.ln
    }, extractTodoTitle(line.content, conf));
  };
}
1035823
def _example_generator(filename):
  
  for idx, line in enumerate(tf.gfile.Open(filename, "rb")):
    if idx == 0: continue  # skip header
    line = text_encoder.to_unicode_utf8(line.strip())
    split_line = line.split("\t")
    # Works for both splits even though dev has some extra human labels.
    yield {
        "premise": split_line[8],
        "hypothesis": split_line[9],
        "label": split_line[-1]
    }
796340
def iterate(self, params, repetition, iteration):
    
    try:
      print("\nStarting iteration",iteration)
      t1 = time.time()
      ret = {}

      # Update learning rate using learning rate scheduler if configured
      if self.lr_scheduler is not None:
        # ReduceLROnPlateau lr_scheduler step should be called after validation,
        # all other lr_schedulers should be called before training
        if params["lr_scheduler"] != "ReduceLROnPlateau":
          self.lr_scheduler.step()

      self.train(params, epoch=iteration)

      # Run validation test
      if self.validation_loader is not None:
        validation = self.test(params, self.validation_loader)

        # ReduceLROnPlateau step should be called after validation
        if params["lr_scheduler"] == "ReduceLROnPlateau":
          self.lr_scheduler.step(validation["test_loss"])

        ret["validation"] = validation
        print("Validation: Test error=", validation["testerror"],
              "entropy=", validation["entropy"])

      # Run noise test
      if (params["test_noise_every_epoch"] or
          iteration == params["iterations"] - 1):
        ret.update(self.runNoiseTests(params))
        print("Noise test results: totalCorrect=", ret["totalCorrect"],
              "Test error=", ret["testerror"], ", entropy=", ret["entropy"])
        if ret["totalCorrect"] > 100000 and ret["testerror"] > 98.3:
          print("*******")
          print(params)

      ret.update({"elapsedTime": time.time() - self.startTime})
      ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                    else self.lr_scheduler.get_lr()})

      print("Iteration time= {0:.3f} secs, "
            "total elapsed time= {1:.3f} mins".format(
              time.time() - t1,ret["elapsedTime"]/60.0))

    except Exception as e:
      # Tracebacks are not printed if using multiprocessing so we do it here
      tb = sys.exc_info()[2]
      traceback.print_tb(tb)
      raise RuntimeError("Something went wrong in iterate", e)

    return ret
442119
def forward_for_single_feature_map(
            self, anchors, box_cls, box_regression):
        
        device = box_cls.device
        N, _, H, W = box_cls.shape
        A = box_regression.size(1) // 4
        C = box_cls.size(1) // A

        # put in the same format as anchors
        box_cls = permute_and_flatten(box_cls, N, A, C, H, W)
        box_cls = box_cls.sigmoid()

        box_regression = permute_and_flatten(box_regression, N, A, 4, H, W)
        box_regression = box_regression.reshape(N, -1, 4)

        num_anchors = A * H * W

        candidate_inds = box_cls > self.pre_nms_thresh

        pre_nms_top_n = candidate_inds.view(N, -1).sum(1)
        pre_nms_top_n = pre_nms_top_n.clamp(max=self.pre_nms_top_n)

        results = []
        for per_box_cls, per_box_regression, per_pre_nms_top_n, \
        per_candidate_inds, per_anchors in zip(
            box_cls,
            box_regression,
            pre_nms_top_n,
            candidate_inds,
            anchors):

            # Sort and select TopN
            # TODO most of this can be made out of the loop for
            # all images. 
            # TODO:Yang: Not easy to do. Because the numbers of detections are
            # different in each image. Therefore, this part needs to be done
            # per image. 
            per_box_cls = per_box_cls[per_candidate_inds]
 
            per_box_cls, top_k_indices = \
                    per_box_cls.topk(per_pre_nms_top_n, sorted=False)

            per_candidate_nonzeros = \
                    per_candidate_inds.nonzero()[top_k_indices, :]

            per_box_loc = per_candidate_nonzeros[:, 0]
            per_class = per_candidate_nonzeros[:, 1]
            per_class += 1

            detections = self.box_coder.decode(
                per_box_regression[per_box_loc, :].view(-1, 4),
                per_anchors.bbox[per_box_loc, :].view(-1, 4)
            )

            boxlist = BoxList(detections, per_anchors.size, mode="xyxy")
            boxlist.add_field("labels", per_class)
            boxlist.add_field("scores", per_box_cls)
            boxlist = boxlist.clip_to_image(remove_empty=False)
            boxlist = remove_small_boxes(boxlist, self.min_size)
            results.append(boxlist)

        return results
30448
def data_loader(self, sentences, shuffle=False):
        """"""
        dataset = BertEmbeddingDataset(sentences, self.transform)
        return DataLoader(dataset=dataset, batch_size=self.batch_size, shuffle=shuffle)
865627
def mnist_tutorial(nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
                   train_end=-1, test_end=-1, learning_rate=LEARNING_RATE):
  
  # Train a pytorch MNIST model
  torch_model = PytorchMnistModel()
  if torch.cuda.is_available():
    torch_model = torch_model.cuda()
  report = AccuracyReport()

  train_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=True, download=True,
                     transform=transforms.ToTensor()),
      batch_size=batch_size, shuffle=True)
  test_loader = torch.utils.data.DataLoader(
      datasets.MNIST('data', train=False, transform=transforms.ToTensor()),
      batch_size=batch_size)

  # Truncate the datasets so that our test run more quickly
  train_loader.dataset.train_data = train_loader.dataset.train_data[
      :train_end]
  test_loader.dataset.test_data = test_loader.dataset.test_data[:test_end]

  # Train our model
  optimizer = optim.Adam(torch_model.parameters(), lr=learning_rate)
  train_loss = []

  total = 0
  correct = 0
  step = 0
  for _epoch in range(nb_epochs):
    for xs, ys in train_loader:
      xs, ys = Variable(xs), Variable(ys)
      if torch.cuda.is_available():
        xs, ys = xs.cuda(), ys.cuda()
      optimizer.zero_grad()
      preds = torch_model(xs)
      loss = F.nll_loss(preds, ys)
      loss.backward()  # calc gradients
      train_loss.append(loss.data.item())
      optimizer.step()  # update gradients

      preds_np = preds.cpu().detach().numpy()
      correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
      total += train_loader.batch_size
      step += 1
      if total % 1000 == 0:
        acc = float(correct) / total
        print('[%s] Training accuracy: %.2f%%' % (step, acc * 100))
        total = 0
        correct = 0

  # Evaluate on clean data
  total = 0
  correct = 0
  for xs, ys in test_loader:
    xs, ys = Variable(xs), Variable(ys)
    if torch.cuda.is_available():
      xs, ys = xs.cuda(), ys.cuda()

    preds = torch_model(xs)
    preds_np = preds.cpu().detach().numpy()

    correct += (np.argmax(preds_np, axis=1) == ys.cpu().detach().numpy()).sum()
    total += len(xs)

  acc = float(correct) / total
  report.clean_train_clean_eval = acc
  print('[%s] Clean accuracy: %.2f%%' % (step, acc * 100))

  # We use tf for evaluation on adversarial data
  sess = tf.Session()
  x_op = tf.placeholder(tf.float32, shape=(None, 1, 28, 28,))

  # Convert pytorch model to a tf_model and wrap it in cleverhans
  tf_model_fn = convert_pytorch_model_to_tf(torch_model)
  cleverhans_model = CallableModelWrapper(tf_model_fn, output_layer='logits')

  # Create an FGSM attack
  fgsm_op = FastGradientMethod(cleverhans_model, sess=sess)
  fgsm_params = {'eps': 0.3,
                 'clip_min': 0.,
                 'clip_max': 1.}
  adv_x_op = fgsm_op.generate(x_op, **fgsm_params)
  adv_preds_op = tf_model_fn(adv_x_op)

  # Run an evaluation of our model against fgsm
  total = 0
  correct = 0
  for xs, ys in test_loader:
    adv_preds = sess.run(adv_preds_op, feed_dict={x_op: xs})
    correct += (np.argmax(adv_preds, axis=1) == ys.cpu().detach().numpy()).sum()
    total += test_loader.batch_size

  acc = float(correct) / total
  print('Adv accuracy: {:.3f}'.format(acc * 100))
  report.clean_train_adv_eval = acc
  return report
1335060
def single_discriminator(x, filters=128, kernel_size=8,
                         strides=4, pure_mean=False):
  """"""
  with tf.variable_scope("discriminator"):
    net = layers().Conv2D(
        filters, kernel_size, strides=strides, padding="SAME", name="conv1")(x)
    if pure_mean:
      net = tf.reduce_mean(net, [1, 2])
    else:
      net = mean_with_attention(net, "mean_with_attention")
    return net