Loads and preprocessed data for the MR dataset.
    Returns input vectors, labels, vocabulary, and inverse vocabulary.
964331
def load_data():
    
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    x, y = build_input_data(sentences_padded, labels, vocabulary)
    return [x, y, vocabulary, vocabulary_inv]
1051841
def load_data(self):
        
        print('Extracting training data from csv...')
        images = list()
        labels = list()
        emotion_index_map = dict()
        with open(self.datapath) as csv_file:
            reader = csv.reader(csv_file, delimiter=',', quotechar='"')
            for row in reader:
                label_class = row[self.csv_label_col]
                if label_class not in self.target_emotion_map.keys():
                    continue
                label_class = self.target_emotion_map[label_class]
                if label_class not in emotion_index_map.keys():
                    emotion_index_map[label_class] = len(emotion_index_map.keys())
                labels.append(label_class)

                image = np.asarray([int(pixel) for pixel in row[self.csv_image_col].split(' ')], dtype=np.uint8).reshape(self.image_dimensions)
                image = self._reshape(image)
                images.append(image)

        vectorized_labels = self._vectorize_labels(emotion_index_map, labels)
        self._check_data_not_empty(images)

        return self._load_dataset(np.array(images), np.array(vectorized_labels), emotion_index_map)
167994
def load(self, data: dict):
        
        for i, label in enumerate(self.keys):
            key = label.cget('text')
            if key in data.keys():
                entry_was_enabled = True if \
                    self.values[i].cget('state') == 'normal' else False
                if not entry_was_enabled:
                    self.values[i].config(state='normal')

                self.values[i].delete(0, tk.END)
                self.values[i].insert(0, str(data[key]))

                if not entry_was_enabled:
                    self.values[i].config(state='disabled')
1169753
def get_data_iters_and_vocabs(args: argparse.Namespace,
                              model_folder: Optional[str]) -> Tuple['data_io.BaseParallelSampleIter',
                                                                    List[vocab.Vocab], vocab.Vocab, model.ModelConfig]:
    

    model_config = model.SockeyeModel.load_config(os.path.join(args.model, C.CONFIG_NAME))

    if args.max_seq_len is None:
        max_seq_len_source = model_config.config_data.max_seq_len_source
        max_seq_len_target = model_config.config_data.max_seq_len_target
    else:
        max_seq_len_source, max_seq_len_target = args.max_seq_len

    batch_num_devices = 1 if args.use_cpu else sum(-di if di < 0 else 1 for di in args.device_ids)

    # Load the existing vocabs created when starting the training run.
    source_vocabs = vocab.load_source_vocabs(model_folder)
    target_vocab = vocab.load_target_vocab(model_folder)

    sources = [args.source] + args.source_factors
    sources = [str(os.path.abspath(source)) for source in sources]

    score_iter = data_io.get_scoring_data_iters(
        sources=sources,
        target=os.path.abspath(args.target),
        source_vocabs=source_vocabs,
        target_vocab=target_vocab,
        batch_size=args.batch_size,
        batch_num_devices=batch_num_devices,
        max_seq_len_source=max_seq_len_source,
        max_seq_len_target=max_seq_len_target)

    return score_iter, source_vocabs, target_vocab, model_config
247653
def load_data_batch(self, data_batch):
        """"""
        _load_data(data_batch, self.data_arrays)
        _load_label(data_batch, self.label_arrays)
970981
def _load(self, metrix):
    
    if isinstance(metrix, TimeSeries):
      return metrix
    if isinstance(metrix, dict):
      return TimeSeries(metrix)
    return TimeSeries(utils.read_csv(metrix))
220093
def get_word_vectors(vocab):
    
    wv = get_data('word2vec')
    vectors = np.array(len(vocab), len(wv['the']))
    for i, tok in enumerate(vocab):
        word = tok[0]
        variations = (word, word.lower(), word.lower()[:-1])
        for w in variations:
            if w in wv:
                vectors[i, :] = wv[w]
        if not np.sum(np.abs(vectors[i])):
            logger.warning('Unable to find {}, {}, or {} in word2vec.'.format(*variations))
    return vectors
414989
def load(fname):
    """"""

    content = _open(fname).read()
    if PY2:
      state = pickle.loads(content)
    else:
      state = pickle.loads(content, encoding='latin1')
    voc, vec = state
    if len(voc) == 2:
      words, counts = voc
      word_count = dict(zip(words, counts))
      vocab = CountedVocabulary(word_count=word_count)
    else:
      vocab = OrderedVocabulary(voc)
    return Embedding(vocabulary=vocab, vectors=vec)
395365
def data_preprocess(data, size=None, min_axis=None, batch=False):
    
    if batch:
        return [data_preprocess(el, size=size, min_axis=min_axis, batch=False) for el in data]

    if isinstance(data, string_types):
        if file_exists(data):
            # probably a path to an image
            preprocessed = Image.open(data)
        else:
            # base 64 encoded image data, a url, or raw content
            # send raw data to the server and let the server infer type
            b64_or_url = re.sub('^data:image/.+;base64,', '', data)
            return b64_or_url

    elif isinstance(data, Image.Image):
        # data is image from PIL
        preprocessed = data

    elif type(data).__name__ == "ndarray":
        # data is likely image from numpy/scipy
        if "float" in str(data.dtype) and data.min() >= 0 and data.max() <= 1:
            data *= 255.
        try:
            preprocessed = Image.fromarray(data.astype("uint8"))
        except TypeError:
            raise IndicoError(
                "Please ensure the numpy array is in a format by PIL. "
                "Values must be between 0 and 1 or between 0 and 255 in greyscale, rgb, or rgba format."
            )

    else:
        # at this point we are unsure of the type -- it could be malformatted text or image data.
        raise IndicoError(
            "Invalid input datatype: `{}`. "
            "Ensure input data is one of the following types: "
            "`str`, `unicode`, `PIL.Image`, `np.ndarray`.".format(
                data.__class__.__name__
            )
        )

    #
    if size or min_axis:
        preprocessed = resize_image(preprocessed, size, min_axis)

    # standardize on b64 encoding for sending image data over the wire
    temp_output = BytesIO()
    preprocessed.save(temp_output, format='PNG')
    temp_output.seek(0)
    output_s = temp_output.read()
    return base64.b64encode(output_s).decode('utf-8') if PY3 else base64.b64encode(output_s)
600320
def _preprocess_input(self):
    """"""
    # Compute the sets of MTF dimensions used in operations/tensors.

    # a {string: frozenset(string)}, mapping operation name to MTF dimension
    # names.
    self._operation_name_to_mtf_dimension_set = {}
    # a {string: frozenset(string)}, mapping tensor name to MTF dimension names.
    self._tensor_name_to_mtf_dimension_set = {}

    for operation_name in self._graph.get_all_operation_names():
      self._operation_name_to_mtf_dimension_set[operation_name] = frozenset(
          set(self._graph.get_operation_mtf_dimension_names(
              operation_name)).intersection(
                  self._layout_validator.splittable_mtf_dimension_names))
    for tensor_name in self._graph.get_all_tensor_names():
      self._tensor_name_to_mtf_dimension_set[tensor_name] = frozenset(
          set(self._graph.get_tensor_mtf_dimension_names(tensor_name))
          .intersection(self._layout_validator.splittable_mtf_dimension_names))

    self._operation_mtf_dimension_sets = set(
        self._operation_name_to_mtf_dimension_set.values())
    self._mtf_dimension_sets = self._operation_mtf_dimension_sets | set(
        self._tensor_name_to_mtf_dimension_set.values())

    # Compute possible assignments for each set of MTF dimensions.
    self._assignments = {}  # indexed by MTF dimension set
    for mtf_dimension_set in self._mtf_dimension_sets:
      self._assignments[mtf_dimension_set] = _generate_assignments(
          mtf_dimension_set, self._layout_validator.mesh_dimension_name_to_size)
88266
def _load(cls,
              config: Params,
              serialization_dir: str,
              weights_file: str = None,
              cuda_device: int = -1) -> 'Model':
        
        weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS)

        # Load vocabulary from file
        vocab_dir = os.path.join(serialization_dir, 'vocabulary')
        # If the config specifies a vocabulary subclass, we need to use it.
        vocab_params = config.get("vocabulary", Params({}))
        vocab_choice = vocab_params.pop_choice("type", Vocabulary.list_available(), True)
        vocab = Vocabulary.by_name(vocab_choice).from_files(vocab_dir)

        model_params = config.get('model')

        # The experiment config tells us how to _train_ a model, including where to get pre-trained
        # embeddings from.  We're now _loading_ the model, so those embeddings will already be
        # stored in our weights.  We don't need any pretrained weight file anymore, and we don't
        # want the code to look for it, so we remove it from the parameters here.
        remove_pretrained_embedding_params(model_params)
        model = Model.from_params(vocab=vocab, params=model_params)

        # If vocab+embedding extension was done, the model initialized from from_params
        # and one defined by state dict in weights_file might not have same embedding shapes.
        # Eg. when model embedder module was transferred along with vocab extension, the
        # initialized embedding weight shape would be smaller than one in the state_dict.
        # So calling model embedding extension is required before load_state_dict.
        # If vocab and model embeddings are in sync, following would be just a no-op.
        model.extend_embedder_vocab()

        model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device))
        model.load_state_dict(model_state)

        # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
        # in sync with the weights
        if cuda_device >= 0:
            model.cuda(cuda_device)
        else:
            model.cpu()

        return model
1133748
def _load_pretrained_tok2vec(nlp, loc):
    
    with loc.open("rb") as file_:
        weights_data = file_.read()
    loaded = []
    for name, component in nlp.pipeline:
        if hasattr(component, "model") and hasattr(component.model, "tok2vec"):
            component.tok2vec.from_bytes(weights_data)
            loaded.append(name)
    return loaded
445943
def load_images(image_files, resize=True):
  """"""

  images = []
  for image_file in image_files:
    with file_io.FileIO(image_file, 'r') as ff:
      images.append(ff.read())
  if resize is False:
    return images

  # To resize, run a tf session so we can reuse 'decode_and_resize()'
  # which is used in prediction graph. This makes sure we don't lose
  # any quality in prediction, while decreasing the size of the images
  # submitted to the model over network.
  image_str_tensor = tf.placeholder(tf.string, shape=[None])
  image = tf.map_fn(resize_image, image_str_tensor, back_prop=False)
  feed_dict = collections.defaultdict(list)
  feed_dict[image_str_tensor.name] = images
  with tf.Session() as sess:
    images_resized = sess.run(image, feed_dict=feed_dict)
  return images_resized
369759
def load_dataset(self, dataset, verbose=True):
        
        assert self.get_dataset(dataset) is True, "Datasets aren't properly downloaded, " \
                                                  "rerun to try again or download datasets manually."

        if verbose:
            print("Loading %s" % dataset)

        dataset_path = self.base_dataset_path + dataset
        for path in tqdm(glob.glob(dataset_path + "/*.gnt")):
            for image, label in self.load_gnt_file(path):
                yield image, label
832946
def _data(self, cube, weighted, prune):
        """"""
        result = []
        for slice_ in cube.slices:
            if cube.has_mr:
                return self._mr_index(cube, weighted, prune)
            num = slice_.margin(axis=0, weighted=weighted, prune=prune)
            den = slice_.margin(weighted=weighted, prune=prune)
            margin = num / den
            proportions = slice_.proportions(axis=1, weighted=weighted, prune=prune)
            result.append(proportions / margin)

        if len(result) == 1 and cube.ndim < 3:
            result = result[0]
        else:
            if prune:
                mask = np.array([slice_.mask for slice_ in result])
                result = np.ma.masked_array(result, mask)
            else:
                result = np.array(result)

        return result
1273816
def _load(self, dataset_spec):
        """"""

        for idx, ds in enumerate(dataset_spec):
            self.append(ds, idx)
403568
def encode_dataset(dataset, vocabulary):
  
  def encode(features):
    return {k: vocabulary.encode_tf(v) for k, v in features.items()}
  return dataset.map(encode, num_parallel_calls=tf.data.experimental.AUTOTUNE)