Load filenames from the N_GEO_Ref attribute of a dataset's file.
323445
def _load_from_geo_ref(self, dsid):
        """"""
        file_handlers = self._get_file_handlers(dsid)
        if not file_handlers:
            return None

        fns = []
        for fh in file_handlers:
            base_dir = os.path.dirname(fh.filename)
            try:
                # get the filename and remove the creation time
                # which is often wrong
                fn = fh['/attr/N_GEO_Ref'][:46] + '*.h5'
                fns.extend(glob(os.path.join(base_dir, fn)))

                # usually is non-terrain corrected file, add the terrain
                # corrected file too
                if fn[:5] == 'GIMGO':
                    fn = 'GITCO' + fn[5:]
                elif fn[:5] == 'GMODO':
                    fn = 'GMTCO' + fn[5:]
                else:
                    continue
                fns.extend(glob(os.path.join(base_dir, fn)))
            except KeyError:
                LOG.debug("Could not load geo-reference information from {}".format(fh.filename))

        return fns
838467
def create_filehandlers(self, filenames, fh_kwargs=None):
        """"""
        filenames = list(OrderedDict.fromkeys(filenames))
        logger.debug("Assigning to %s: %s", self.info['name'], filenames)

        self.info.setdefault('filenames', []).extend(filenames)
        filename_set = set(filenames)
        created_fhs = {}
        # load files that we know about by creating the file handlers
        for filetype, filetype_info in self.sorted_filetype_items():
            filehandlers = self.new_filehandlers_for_filetype(filetype_info,
                                                              filename_set,
                                                              fh_kwargs=fh_kwargs)

            filename_set -= set([fhd.filename for fhd in filehandlers])
            if filehandlers:
                created_fhs[filetype] = filehandlers
                self.file_handlers[filetype] = sorted(
                    self.file_handlers.get(filetype, []) + filehandlers,
                    key=lambda fhd: (fhd.start_time, fhd.filename))

        # update existing dataset IDs with information from the file handler
        self.update_ds_ids_from_file_handlers()

        # load any additional dataset IDs determined dynamically from the file
        self.add_ds_ids_from_files()
        return created_fhs
426529
def _convert_dataset_files(value):
    """"""
    output = {}
    for k, v in value.items():
        inst = DatasetFile.from_jsonld(v)
        output[inst.path] = inst
    return output
419198
def _load_dataset_area(self, dsid, file_handlers, coords):
        """"""
        try:
            return self._load_area_def(dsid, file_handlers)
        except NotImplementedError:
            if any(x is None for x in coords):
                logger.warning(
                    "Failed to load coordinates for '{}'".format(dsid))
                return None

            area = self._make_area_from_coords(coords)
            if area is None:
                logger.debug("No coordinates found for %s", str(dsid))
            return area
954796
def load_files(filenames,multiproc=False,**kwargs):
    
    filenames = np.atleast_1d(filenames)
    logger.debug("Loading %s files..."%len(filenames))

    kwargs = [dict(filename=f,**kwargs) for f in filenames]

    if multiproc:
        from multiprocessing import Pool
        processes = multiproc if multiproc > 0 else None
        p = Pool(processes,maxtasksperchild=1)
        out = p.map(load_file,kwargs)
    else:
        out = [load_file(kw) for kw in kwargs]

    dtype = out[0].dtype
    for i,d in enumerate(out):
        if d.dtype != dtype: 
            # ADW: Not really safe...
            logger.warn("Casting input data to same type.")
            out[i] = d.astype(dtype,copy=False)

    logger.debug('Concatenating arrays...')
    return np.concatenate(out)
292960
def load_hdf_metadata(filename, groupname="data"):
    """""""
    with _h5py.File(filename, "r") as f:
        data = dict(f[groupname].attrs)
    return data
964624
def map_data(self, map_name):
    """"""
    with gfile.Open(os.path.join(self.data_dir, "Maps", map_name), "rb") as f:
      return f.read()
827975
def _read_projections(folder, indices):
    """"""
    datasets = []

    # Get the relevant file names
    file_names = sorted([f for f in os.listdir(folder) if f.endswith(".dcm")])

    if len(file_names) == 0:
        raise ValueError('No DICOM files found in {}'.format(folder))

    file_names = file_names[indices]

    data_array = None

    for i, file_name in enumerate(tqdm.tqdm(file_names,
                                            'Loading projection data')):
        # read the file
        dataset = dicom.read_file(folder + '/' + file_name)

        # Get some required data
        rows = dataset.NumberofDetectorRows
        cols = dataset.NumberofDetectorColumns
        hu_factor = dataset.HUCalibrationFactor
        rescale_intercept = dataset.RescaleIntercept
        rescale_slope = dataset.RescaleSlope

        # Load the array as bytes
        proj_array = np.array(np.frombuffer(dataset.PixelData, 'H'),
                              dtype='float32')
        proj_array = proj_array.reshape([rows, cols], order='F').T

        # Rescale array
        proj_array *= rescale_slope
        proj_array += rescale_intercept
        proj_array /= hu_factor

        # Store results
        if data_array is None:
            # We need to load the first dataset before we know the shape
            data_array = np.empty((len(file_names), cols, rows),
                                  dtype='float32')

        data_array[i] = proj_array[:, ::-1]
        datasets.append(dataset)

    return datasets, data_array
80335
def read(*filenames, **kwargs):
    
    encoding = kwargs.get('encoding', 'utf-8')
    sep = kwargs.get('sep', '\n')
    buf = []
    for filename in filenames:
        if path.splitext(filename)[1] == ".md":
            try:
                import pypandoc
                buf.append(pypandoc.convert_file(filename, 'rst'))
                continue
            except:
                with io.open(filename, encoding=encoding) as f:
                    buf.append(f.read())
        with io.open(filename, encoding=encoding) as f:
            buf.append(f.read())
    return sep.join(buf)
769205
def _read_name_file(self, filename):
        
        file_path = os.path.join(self._DATA_DIR, filename)
        with open(file_path) as f:
            names = json.load(f)

        return names
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
1361981
def load_metadata(self, data_dir, feature_name=None):
    """"""
    # Restore names if defined
    names_filepath = _get_names_filepath(data_dir, feature_name)
    if tf.io.gfile.exists(names_filepath):
      self.names = _load_names_from_file(names_filepath)
592197
def load(self, *filenames):
        """"""
        if len(filenames) == 1:
            if os.path.basename( filenames[0] ) == 'index.coldshot':
                return self.load_coldshot( os.path.dirname( filenames[0]) )
            elif os.path.isdir( filenames[0] ):
                return self.load_coldshot( filenames[0] )
        try:
            self.loader = pstatsloader.PStatsLoader(*filenames)
            self.ConfigureViewTypeChoices()
            self.SetModel( self.loader )
            self.viewType = self.loader.ROOTS[0]
            self.SetTitle(_("Run Snake Run: %(filenames)s")
                          % {'filenames': ', '.join(filenames)[:120]})
        except (IOError, OSError, ValueError,MemoryError), err:
            self.SetStatusText(
                _('Failure during load of %(filenames)s: %(err)s'
            ) % dict(
                filenames=" ".join([repr(x) for x in filenames]),
                err=err
            ))
731388
def _load_dataset_with_area(self, dsid, coords):
        """"""
        file_handlers = self._get_file_handlers(dsid)
        if not file_handlers:
            return

        area = self._load_dataset_area(dsid, file_handlers, coords)

        try:
            ds = self._load_dataset_data(file_handlers, dsid)
        except (KeyError, ValueError) as err:
            logger.exception("Could not load dataset '%s': %s", dsid, str(err))
            return None

        if area is not None:
            ds.attrs['area'] = area
            calc_coords = (('x' not in ds.coords) or('y' not in ds.coords)) and hasattr(area, 'get_proj_vectors_dask')
            if calc_coords and hasattr(area, 'get_proj_vectors'):
                ds['x'], ds['y'] = area.get_proj_vectors()
            elif calc_coords:
                # older pyresample with dask-only method
                ds['x'], ds['y'] = area.get_proj_vectors_dask(CHUNK_SIZE)
        return ds
701955
def load_from_filename(self, file_name, sep='\n'):
        """"""
        fp = open(file_name, 'rb')
        n = self.load_from_file(fp, sep)
        fp.close()
        return n
64723
def load_datafile(name, search_path, codecs=get_codecs(), **kwargs):
    
    return munge.load_datafile(name, search_path, codecs, **kwargs)
1023780
def read(*filenames, **kwargs):
    
    encoding = kwargs.get('encoding', 'utf-8')
    sep = kwargs.get('sep', '\n')
    buf = []
    for fl in filenames:
        with codecs.open(os.path.join(HERE, fl), 'rb', encoding) as f:
            buf.append(f.read())
    return sep.join(buf)