src_path : str or PathLike object
        A dataset path or URL. Will be opened in "r" mode.
184017
def _sentinel_stats(
    src_path, percentiles=(2, 98), histogram_bins=10, histogram_range=None
):
    

    with rasterio.open(src_path) as src:
        arr = src.read(indexes=[1], masked=True)
        arr[arr == 0] = np.ma.masked

    params = {}
    if histogram_bins:
        params.update(dict(bins=histogram_bins))
    if histogram_range:
        params.update(dict(range=histogram_range))

    return {1: utils._stats(arr, percentiles=percentiles, **params)}
699461
def dataset_path_iterator(file_path: str) -> Iterator[str]:
        
        logger.info("Reading CONLL sentences from dataset files at: %s", file_path)
        for root, _, files in list(os.walk(file_path)):
            for data_file in files:
                # These are a relic of the dataset pre-processing. Every
                # file will be duplicated - one file called filename.gold_skel
                # and one generated from the preprocessing called filename.gold_conll.
                if not data_file.endswith("gold_conll"):
                    continue

                yield os.path.join(root, data_file)
336889
def __remote_path_rewrite(self, dataset_path, dataset_path_type, name=None):
        
        path = str(dataset_path)  # Use false_path if needed.
        action = self.action_mapper.action(path, dataset_path_type)
        if action.staging_needed:
            if name is None:
                name = os.path.basename(path)
            remote_directory = self.__remote_directory(dataset_path_type)
            remote_path_rewrite = self.path_helper.remote_join(remote_directory, name)
        else:
            # Actions which don't require staging MUST define a path_rewrite
            # method.
            remote_path_rewrite = action.path_rewrite(self.path_helper)

        return remote_path_rewrite
1346987
private static Path getDirectory(FileSystem fs, Path path) {
    try {
      if (!fs.exists(path) || fs.isDirectory(path)) {
        return path;
      } else {
        return path.getParent();
      }

    } catch (IOException e) {
      throw new DatasetIOException("Cannot access path: " + path, e);
    }
  }
1307054
def pkgPath(root, path, rpath="/"):
    
    global data_files
    if not os.path.exists(path):
        return
    files = []
    for spath in os.listdir(path):
        # Ignore test directories
        if spath == 'test':
            continue
        subpath = os.path.join(path, spath)
        spath = os.path.join(rpath, spath)
        if os.path.isfile(subpath):
            files.append(subpath)
        if os.path.isdir(subpath):
            pkgPath(root, subpath, spath)
    data_files.append((root + rpath, files))
606381
def url2path(url, data=True, ext:str='.tgz'):
    ""
    name = url2name(url)
    return datapath4file(name, ext=ext, archive=False) if data else modelpath4file(name, ext=ext)
70751
def set_path(ctx, path_str, value, data):
    
    ctx.gbc.say('set_path:value:' + str(value) + ' at:' + path_str + ' in:',
                stuff=data, verbosity=1001)

    path = path_str.split('.')
    ctx.gbc.say('path:', stuff=path, verbosity=100)
    if len(path) > 1:
        destk = '.'.join(path[0:-1])
        lp = path[-1]
        ctx.gbc.say('destk:%s' % destk, verbosity=100)
        ctx.gbc.say('last:%s' % lp, verbosity=100)
        ctx.gbc.say('current keys:', stuff=data.keys(), verbosity=1001)
        if len(path) > 2:
            destk = unescape(ctx, destk)
        if destk not in data.keys():
            ctx.gbc.say('destk not in current keys:',
                        stuff=data.keys(), verbosity=1001)
            data[destk] = {}
            ctx.gbc.say('destk not added:', stuff=data, verbosity=1001)

        lp = unescape(ctx, lp)
        data[destk][lp] = value
        ctx.gbc.say('destk and val added:', stuff=data, verbosity=1001)
    else:
        path_str = unescape(ctx, path_str)
        data[path_str] = value
    ctx.gbc.say('set_path:res:', stuff=data, verbosity=1001)
    return data
176764
def _get_vqa_v2_image_raw_dataset(directory, image_root_url, image_urls):
  """"""
  for url in image_urls:
    filename = os.path.basename(url)
    download_url = os.path.join(image_root_url, url)
    path = generator_utils.maybe_download(directory, filename, download_url)
    unzip_dir = os.path.join(directory, filename.strip(".zip"))
    if not tf.gfile.Exists(unzip_dir):
      zipfile.ZipFile(path, "r").extractall(directory)
846904
def copy(src_uri, dest_base_uri, config_path=None, progressbar=None):
    
    dataset = DataSet.from_uri(src_uri)

    proto_dataset = _copy_create_proto_dataset(
        dataset,
        dest_base_uri,
        config_path,
        progressbar
    )
    _copy_content(dataset, proto_dataset, progressbar)
    proto_dataset.freeze(progressbar=progressbar)

    return proto_dataset.uri
448004
def download_from_url(src_url, dst_dir):
    
    dst_path = ""
    if "MD982181" not in src_url:
        dsn = input("Please enter the dataset name for this file (Name.Location.Year) : ")
        dst_path = os.path.join(dst_dir, dsn + ".lpd")
        try:
            print("downloading file from url...")
            urllib.request.urlretrieve(src_url, dst_path)
        except Exception as e:
            print("Error: unable to download from url: {}".format(e))
    else:
        print("Error: That file cannot be download due to server restrictions")
    return dst_path
761322
def get_dataset_path(*args):
    
    d = os.environ.get('TENSORPACK_DATASET', None)
    if d is None:
        d = os.path.join(os.path.expanduser('~'), 'tensorpack_data')
        if execute_only_once():
            logger.warn("Env var $TENSORPACK_DATASET not set, using {} for datasets.".format(d))
        if not os.path.isdir(d):
            mkdir_p(d)
            logger.info("Created the directory {}.".format(d))
    assert os.path.isdir(d), d
    return os.path.join(d, *args)
1321859
def open_resource(self, path: FilePath, mode: str='rb') -> IO[AnyStr]:
        
        if mode not in {'r', 'rb'}:
            raise ValueError('Files can only be opened for reading')
        return open(self.root_path / file_path_to_path(path), mode)
561476
def s3path(self, rel_path):
        """"""
        import urlparse

        path = self.path(rel_path, public_url=True)

        parts = list(urlparse.urlparse(path))

        parts[0] = 's3'
        parts[1] = self.bucket_name

        return urlparse.urlunparse(parts)
443483
def pretty_relpath(path, start):
    '''
    
    '''
    relpath = os.path.relpath(path, start)
    if relpath.startswith('..'):
        return path
    return relpath
567618
def list_gtfs(path: Path) -> DataFrame:
    
    path = Path(path)
    if not path.exists():
        raise ValueError(f"Path {path} does not exist")

    # Collect rows of DataFrame
    rows = []
    if path.is_file():
        # Zip file
        with zipfile.ZipFile(str(path)) as src:
            for x in src.infolist():
                if x.filename == "./":
                    continue
                d = {}
                d["file_name"] = x.filename
                d["file_size"] = x.file_size
                rows.append(d)
    else:
        # Directory
        for x in path.iterdir():
            d = {}
            d["file_name"] = x.name
            d["file_size"] = x.stat().st_size
            rows.append(d)

    return pd.DataFrame(rows)
1135533
def generate_dataset(self, cdl_path):
        '''
        
        '''
        if '.cdl' in cdl_path:  # it's possible the filename doesn't have the .cdl extension
            ds_str = cdl_path.replace('.cdl', '.nc')
        else:
            ds_str = cdl_path + '.nc'
        subprocess.call(['ncgen', '-o', ds_str, cdl_path])
        return ds_str
340027
def DataPath():
    
    return os.path.abspath('%s/data'%(os.path.dirname( \
        os.path.realpath(__file__))))