Load training and test data.
181323
def ld_mnist():
  """"""

  def convert_types(image, label):
    image = tf.cast(image, tf.float32)
    image /= 255
    return image, label

  dataset, info = tfds.load('mnist', data_dir='gs://tfds-data/datasets', with_info=True,
                            as_supervised=True)
  mnist_train, mnist_test = dataset['train'], dataset['test']
  mnist_train = mnist_train.map(convert_types).shuffle(10000).batch(128)
  mnist_test = mnist_test.map(convert_types).batch(128)
  return EasyDict(train=mnist_train, test=mnist_test)
512743
def _load_models(self) -> None:
        """"""
        if self._models is None:
            logging.info('Loading %d models', len(self._model_paths))

            def load_model(model_path: str):
                logging.debug('\tloading %s', model_path)
                if path.isdir(model_path):
                    model_path = path.join(model_path, CXF_CONFIG_FILE)
                config = load_config(model_path)
                config['model']['inputs'] = self._inputs
                config['model']['outputs'] = self._outputs

                return create_model(config, output_dir=None, dataset=self._dataset,
                                    restore_from=path.dirname(model_path))

            self._models = list(map(load_model, self._model_paths))
1239189
def loadalldatas():
    """"""
    dependency_order = ['common', 'profiles', 'blog', 'democomments']
    for app in dependency_order:
        project.recursive_load(os.path.join(paths.project_paths.manage_root, app))
499728
def load_data(train_path='./data/regression.train', test_path='./data/regression.test'):
    '''
    
    '''
    print('Load data...')
    df_train = pd.read_csv(train_path, header=None, sep='\t')
    df_test = pd.read_csv(test_path, header=None, sep='\t')
    num = len(df_train)
    split_num = int(0.9 * num)

    y_train = df_train[0].values
    y_test = df_test[0].values
    y_eval = y_train[split_num:]
    y_train = y_train[:split_num]

    X_train = df_train.drop(0, axis=1).values
    X_test = df_test.drop(0, axis=1).values
    X_eval = X_train[split_num:, :]
    X_train = X_train[:split_num, :]

    # create dataset for lightgbm
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_eval = lgb.Dataset(X_eval, y_eval, reference=lgb_train)

    return lgb_train, lgb_eval, X_test, y_test
1334885
def LoadFits(name):
    
    if name not in fits_collection.keys():
        raise Exception('Invalid fit name : %s'%name)
    else:
        testPath = DataPath() + '/' + fits_collection[name].data_url.split('/')[-1]
        if (not os.path.isfile(testPath)):
            DownloadData(name)
        fit = fits_collection[name].fit_class(name.split('surfinBH')[-1])
        print('Loaded %s fit.'%name)
        return fit
216663
def build_trainer(nn, ds, verbosity=1):
    """"""
    return pb.supervised.trainers.rprop.RPropMinusTrainer(nn, dataset=ds, batchlearning=True, verbose=bool(verbosity))
1070459
def _get_raw_data(subset):
  """"""
  raw_data = _LOADED.get(subset)
  if raw_data is not None:
    return raw_data, _LOADED["vocab"]
  else:
    train_data, valid_data, test_data, vocab = ptb_reader.ptb_raw_data(
        FLAGS.data_path)
    _LOADED.update({
        "train": np.array(train_data),
        "valid": np.array(valid_data),
        "test": np.array(test_data),
        "vocab": vocab
    })
    return _LOADED[subset], vocab
793367
def load_data(self, pdbid):
        """"""
        f = urlopen("http://projects.biotec.tu-dresden.de/plip-rest/pdb/%s?format=xml" % pdbid.lower())
        self.doc = etree.parse(f)
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
1316670
def load_data(limit=0, split=0.8):
    """"""
    # Partition off part of the train data for evaluation
    train_data, _ = thinc.extra.datasets.imdb()
    random.shuffle(train_data)
    train_data = train_data[-limit:]
    texts, labels = zip(*train_data)
    cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
    split = int(len(train_data) * split)
    return (texts[:split], cats[:split]), (texts[split:], cats[split:])
820519
def __load(self, path):
        """"""
        try:
            path = os.path.abspath(path)
            with open(path, 'rb') as df:
                # loaded_dataset = pickle.load(df)
                self.__data, self.__classes, self.__labels, \
                self.__dtype, self.__description, \
                self.__num_features, self.__feature_names = pickle.load(df)

            # ensure the loaded dataset is valid
            self.__validate(self.__data, self.__classes, self.__labels)

        except IOError as ioe:
            raise IOError('Unable to read the dataset from file: {}', format(ioe))
        except:
            raise
1010523
def build_and_train(iterations, log_stride, test=False):
  """"""

  # Test mode settings.
  batch_size = 2 if test else FLAGS.batch_size
  num_mems = 2 if test else FLAGS.num_mems
  num_heads = 1 if test else FLAGS.num_mems
  num_blocks = 1 if test else FLAGS.num_mems
  head_size = 4 if test else FLAGS.head_size
  max_length = 3 if test else FLAGS.max_length
  max_nest = 2 if test else FLAGS.max_nest
  mlp_size = (20,) if test else (256, 256, 256, 256)

  with tf.Graph().as_default():
    t0 = time.time()

    # Initialize the dataset.
    lte_train = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest)
    lte_test = learn_to_execute.LearnToExecute(
        batch_size, max_length, max_nest, mode=learn_to_execute.Mode.TEST)
    train_data_iter = lte_train.make_one_shot_iterator().get_next()
    test_data_iter = lte_test.make_one_shot_iterator().get_next()
    output_size = lte_train.state.vocab_size

    # Create the model.
    core = snt.RelationalMemory(
        mem_slots=num_mems,
        head_size=head_size,
        num_heads=num_heads,
        num_blocks=num_blocks,
        gate_style=FLAGS.gate_style)
    final_mlp = snt.nets.MLP(
        output_sizes=mlp_size,
        activate_final=True)
    model = SequenceModel(
        core=core,
        target_size=output_size,
        final_mlp=final_mlp)
    tf.logging.info("Instantiated models ({:3f})".format(time.time() - t0))

    # Define the loss & accuracy.
    def loss_fn(inputs, targets, input_sequence_length, output_sequence_length):
      """Creates the loss and the exports."""
      logits = model(
          inputs, targets, input_sequence_length, output_sequence_length)
      targets = tf.cast(targets, tf.int32)
      sq_sz_out_max = targets.shape[0].value

      # Create a mask to ignore accuracy on buffer characters.
      sequence_sizes = tf.cast(output_sequence_length, tf.float32)
      lengths_transposed = tf.expand_dims(sequence_sizes, 1)
      range_row = tf.expand_dims(
          tf.range(0, sq_sz_out_max, 1, dtype=tf.float32), 0)
      mask = tf.cast(tf.transpose(tf.less(range_row, lengths_transposed)),
                     tf.float32)

      # Compute token accuracy and solved.
      correct = tf.equal(tf.argmax(logits, 2), tf.argmax(targets, 2))
      solved = tf.reduce_all(tf.boolean_mask(correct, tf.squeeze(mask)), axis=0)
      token_acc = tf.reduce_sum(tf.cast(correct, tf.float32) * mask)
      token_acc /= tf.reduce_sum(sequence_sizes)

      # Compute Loss.
      mask = tf.cast(tf.tile(tf.expand_dims(mask, 2), (1, 1, logits.shape[2])),
                     tf.float32)
      masked_logits = logits * mask
      masked_target = tf.cast(targets, tf.float32) * mask
      logits_flat = tf.reshape(masked_logits,
                               [sq_sz_out_max * batch_size, -1])
      target_flat = tf.reshape(masked_target,
                               [sq_sz_out_max * batch_size, -1])
      xent = tf.nn.softmax_cross_entropy_with_logits(logits=logits_flat,
                                                     labels=target_flat)
      loss = tf.reduce_mean(xent)
      return loss, token_acc, solved

    # Get training step counter.
    global_step = tf.train.get_or_create_global_step()

    # Create the optimizer.
    learning_rate_op = tf.reduce_max([
        tf.train.exponential_decay(
            FLAGS.learning_rate,
            global_step,
            decay_steps=FLAGS.epochs // 100,
            decay_rate=0.9,
            staircase=False),
        FLAGS.min_learning_rate
    ])
    optimizer = tf.train.AdamOptimizer(learning_rate_op)

    # Compute loss, accuracy & the step op.
    inputs, targets, _, input_lengths, output_lengths = train_data_iter
    train_loss, train_acc, train_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    step_op = optimizer.minimize(train_loss, global_step=global_step)
    inputs, targets, _, input_lengths, output_lengths = test_data_iter
    _, test_acc, test_sol = loss_fn(
        inputs, targets, input_lengths, output_lengths)
    tf.logging.info("Created losses and optimizers ({:3f})".format(
        time.time() - t0))

    # Begin Training.
    t0 = time.time()
    tf.logging.info("Starting training ({:3f})".format(time.time() - t0))
    with tf.train.SingularMonitoredSession() as sess:
      for it in six.moves.range(iterations):
        sess.run([step_op, learning_rate_op])
        if it % log_stride == 0:
          loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v = sess.run([
              train_loss, train_acc, test_acc, train_sol, test_sol])
          elapsed = time.time() - t0
          tf.logging.info(
              "iter: {:2d}, train loss {:3f}; train acc {:3f}; test acc {:3f};"
              " train solved {:3f}; test solved {:3f}; ({:3f})".format(
                  it, loss_v, train_acc_v, test_acc_v, train_sol_v, test_sol_v,
                  elapsed))
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
545618
def main(feature_folder, create_learning_curve=False):
    """"""

    # Read the feature description file
    with open(os.path.join(feature_folder, "info.yml"), 'r') as ymlfile:
        feature_description = yaml.load(ymlfile)

    # Get preprocessed .pickle file from model description file
    path_to_data = os.path.join(utils.get_project_root(),
                                feature_description['data-source'])
    if os.path.isdir(path_to_data):
        path_to_data = os.path.join(path_to_data, "data.pickle")
    target_paths = {'traindata': os.path.join(feature_folder,
                                              "traindata.hdf5"),
                    'validdata': os.path.join(feature_folder,
                                              "validdata.hdf5"),
                    'testdata': os.path.join(feature_folder,
                                             "testdata.hdf5")}

    feature_list = features.get_features(feature_description['features'])
    mult_queue = data_multiplication.get_data_multiplication_queue(
        feature_description['data-multiplication'])

    # Set everything up for the creation of the 3 hdf5 (test, validation,
    # training).

    os.chdir(feature_folder)
    logging.info("Start creation of hdf5-files...")
    logging.info("Get sets from '%s' ...", path_to_data)
    (training_set, validation_set, test_set, formula_id2index,
     preprocessing_queue, index2latex) = get_sets(path_to_data)

    training_set = training_set_multiplication(training_set, mult_queue)
    _create_index_formula_lookup(formula_id2index, feature_folder, index2latex)

    # Output data for documentation
    print("Classes (nr of symbols): %i" % len(formula_id2index))
    preprocessing.print_preprocessing_list(preprocessing_queue)
    features.print_featurelist(feature_list)

    logging.info("Start creating hdf5 files")

    # Get the dimension of the feature vector
    input_features = sum(map(lambda n: n.get_dimension(), feature_list))

    # Traindata has to come first because of feature normalization
    for dataset_name, dataset, is_traindata in \
        [("traindata", training_set, True),
         ("testdata", test_set, False),
         ("validdata", validation_set, False)]:
        t0 = time.time()
        logging.info("Start preparing '%s' ...", dataset_name)
        prepared, translation = prepare_dataset(dataset,
                                                formula_id2index,
                                                feature_list,
                                                is_traindata)
        logging.info("%s length: %i", dataset_name, len(prepared))
        logging.info("start 'make_hdf5'x ...")
        make_hdf5(dataset_name,
                  input_features,
                  prepared,
                  os.path.join(feature_folder, target_paths[dataset_name]),
                  create_learning_curve)

        _create_translation_file(feature_folder,
                                 dataset_name,
                                 translation,
                                 formula_id2index)

        t1 = time.time() - t0
        logging.info("%s was written. Needed %0.2f seconds", dataset_name, t1)
        gc.collect()
    utils.create_run_logfile(feature_folder)
64418
def load_mnist_data(args):
    '''
    
    '''
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train]
    x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test]
    y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train]
    y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test]

    LOG.debug('x_train shape: %s', (x_train.shape,))
    LOG.debug('x_test shape: %s', (x_test.shape,))

    return x_train, y_train, x_test, y_test
79170
def load_dataloader(self):
        

        input_transform = transforms.Compose([transforms.ToTensor(), \
                                             transforms.Normalize((0.7136, 0.4906, 0.3283), \
                                                                  (0.1138, 0.1078, 0.0917))])
        training_dataset = LipsDataset(self.image_path,
                                       self.align_path,
                                       mode='train',
                                       transform=input_transform,
                                       seq_len=self.seq_len)

        self.train_dataloader = mx.gluon.data.DataLoader(training_dataset,
                                                         batch_size=self.batch_size,
                                                         shuffle=True,
                                                         num_workers=self.num_workers)

        valid_dataset = LipsDataset(self.image_path,
                                    self.align_path,
                                    mode='valid',
                                    transform=input_transform,
                                    seq_len=self.seq_len)

        self.valid_dataloader = mx.gluon.data.DataLoader(valid_dataset,
                                                         batch_size=self.batch_size,
                                                         shuffle=True,
                                                         num_workers=self.num_workers)
220437
def load(data, udf, data_dir, overwrite):
    """"""
    con = make_ibis_client(ENV)

    # validate our environment before performing possibly expensive operations
    if not can_write_to_hdfs(con):
        raise IbisError('Failed to write to HDFS; check your settings')
    if udf and not can_build_udfs():
        raise IbisError('Build environment does not support building UDFs')

    # load the data files
    if data:
        load_impala_data(con, str(data_dir), overwrite)
    else:
        logger.info('Skipping Ibis test data load (--no-data)')

    # build and upload the UDFs
    if udf:
        already_loaded = is_udf_loaded(con)
        logger.info('Attempting to build and load test UDFs')
        if already_loaded and not overwrite:
            logger.info('UDFs already loaded and not overwriting; moving on')
        else:
            if already_loaded:
                logger.info('UDFs already loaded; attempting to overwrite')
            logger.info('Building UDFs')
            build_udfs()
            logger.info('Uploading UDFs')
            upload_udfs(con)
    else:
        logger.info('Skipping UDF build/load (--no-udf)')