Returns path to where checksums are stored for a given dataset.
321468
def _get_path(dataset_name):
  """"""
  path = _checksum_paths().get(dataset_name, None)
  if path:
    return path
  msg = ('No checksums file could be find for dataset %s. Please create one in '
         'one of: %s') % (dataset_name, ', '.join(_CHECKSUM_DIRS))
  raise AssertionError(msg)
347165
def _get_sizes_checksums(checksums_path):
  """"""
  checksums = {}
  for line in _read_file(checksums_path).split('\n'):
    if not line:
      continue
    # URL might have spaces inside, but size and checksum will not.
    url, size, checksum = line.rsplit(' ', 2)
    checksums[url] = (int(size), checksum)
  return checksums
444803
function getChecksum(fs, path, callback) {
  fs.readFile(path, function(err, data) {
    if(!err) {
      callback(null, md5sum(data));
    } else if(err.code === 'ENOENT') {
      // File does not exist so the checksum is an empty string
      callback(null, "");
    } else {
      callback(err);
    }
  });
}
761322
def get_dataset_path(*args):
    
    d = os.environ.get('TENSORPACK_DATASET', None)
    if d is None:
        d = os.path.join(os.path.expanduser('~'), 'tensorpack_data')
        if execute_only_once():
            logger.warn("Env var $TENSORPACK_DATASET not set, using {} for datasets.".format(d))
        if not os.path.isdir(d):
            mkdir_p(d)
            logger.info("Created the directory {}.".format(d))
    assert os.path.isdir(d), d
    return os.path.join(d, *args)
501946
def _get_checksum(path):
    

    # md5 uses a 512-bit digest blocks, let's scale by defined block_size
    _md5 = hashlib.md5()
    chunk_size = 128 * _md5.block_size

    try:
        with open(path, 'rb') as _file:
            for chunk in iter(lambda: _file.read(chunk_size), ''):
                _md5.update(chunk)
        return _md5.hexdigest()

    except IOError:
        return None
767907
def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
                url=None, resume=True, verbose=1):
    
    warn(DEPRECATION_MSG)
    file_id, md5 = OSF_RESOURCES[dataset_name]
    if url is None:
        url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
    return fetch_file(dataset_name, url, data_dir, dataset_prefix=dataset_prefix,
                      filetype='tar', resume=resume, verbose=verbose, md5sum=md5)
944038
def path(self):
        """"""
        return "/projects/%s/datasets/%s/models/%s" % (
            self._proto.project_id,
            self._proto.dataset_id,
            self._proto.model_id,
        )
105451
def gcs_dataset_info_files(dataset_dir):
  """"""
  prefix = posixpath.join(GCS_DATASET_INFO_DIR, dataset_dir, "")
  # Filter for this dataset
  filenames = [el for el in gcs_files(prefix_filter=prefix)
               if el.startswith(prefix) and len(el) > len(prefix)]
  return filenames
508306
def check_missing_files(client):
    """"""
    missing = defaultdict(list)

    for path, dataset in client.datasets.items():
        for file in dataset.files:
            filepath = (path.parent / file)
            if not filepath.exists():
                missing[str(
                    path.parent.relative_to(client.renku_datasets_path)
                )].append(
                    os.path.normpath(str(filepath.relative_to(client.path)))
                )

    if not missing:
        return True

    click.secho(
        WARNING + 'There are missing files in datasets.'
        # '\n  (use "renku dataset clean <name>" to clean them)'
    )

    for dataset, files in missing.items():
        click.secho(
            '\n\t' + click.style(dataset, fg='yellow') + ':\n\t  ' +
            '\n\t  '.join(click.style(path, fg='red') for path in files)
        )

    return False
1281777
def checksum(self, path, hashtype='sha1'):
        """"""
        return self._handler.checksum(hashtype, posix_path(path))
1135533
def generate_dataset(self, cdl_path):
        '''
        
        '''
        if '.cdl' in cdl_path:  # it's possible the filename doesn't have the .cdl extension
            ds_str = cdl_path.replace('.cdl', '.nc')
        else:
            ds_str = cdl_path + '.nc'
        subprocess.call(['ncgen', '-o', ds_str, cdl_path])
        return ds_str
44966
def README(self):
        
        path = find_dataset_path(
            self.name, data_home=self.data_home, fname="README.md"
        )
        with open(path, 'r') as f:
            return f.read()
1129159
@Override
  public Set<Dataset> findDistinctDatasets() throws IOException {
    Set<Dataset> datasets = Sets.newHashSet();
    Path inputPath = new Path(this.inputDir);
    Path inputLatePath = new Path(inputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Path outputPath = new Path(this.destDir);
    Path outputLatePath = new Path(outputPath, MRCompactor.COMPACTION_LATE_DIR_SUFFIX);
    Dataset dataset =
        new Dataset.Builder().withPriority(this.getDatasetPriority(inputPath.getName()))
            .addInputPath(this.recompactDatasets ? outputPath : inputPath)
            .addInputLatePath(this.recompactDatasets ? outputLatePath : inputLatePath).withOutputPath(outputPath)
            .withOutputLatePath(outputLatePath).withOutputTmpPath(new Path(this.tmpOutputDir)).build();
    datasets.add(dataset);
    return datasets;
  }
1009956
def _download_datasets():
    """"""
    def filepath(*args):
        return abspath(join(dirname(__file__), '..', 'vega_datasets', *args))
    dataset_listing = {}
    for name in DATASETS_TO_DOWNLOAD:
        data = Dataset(name)
        url = data.url
        filename = filepath('_data', data.filename)
        print("retrieving data {0} -> {1}".format(url, filename))
        urlretrieve(url, filename)
        dataset_listing[name] = '_data/{0}'.format(data.filename)
    with open(filepath('local_datasets.json'), 'w') as f:
        json.dump(dataset_listing, f, indent=2, sort_keys=True)
84634
def _file_path(cls, checksum, download_dir):
        
        checksum = checksum.lower()
        file_prefix = '_'.join(['files'] + list(map(str, cls.DIRECTORY_NAME_LENGTHS)))
        path_pieces = [download_dir, '.hca', 'v2', file_prefix]
        checksum_index = 0
        assert(sum(cls.DIRECTORY_NAME_LENGTHS) <= len(checksum))
        for prefix_length in cls.DIRECTORY_NAME_LENGTHS:
            path_pieces.append(checksum[checksum_index:(checksum_index + prefix_length)])
            checksum_index += prefix_length
        path_pieces.append(checksum)
        return os.path.join(*path_pieces)
190839
def checksum(path):
    """"""
    hasher = hashlib.sha1()
    with open(path, 'rb') as stream:
        buf = stream.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = stream.read(BLOCKSIZE)
    return hasher.hexdigest()
1051677
def get_filehandle(self):
        """"""
        if os.path.exists(self.filename):
            self.filehandle = SD(self.filename, SDC.READ)
            logger.debug("Loading dataset {}".format(self.filename))
        else:
            raise IOError("Path {} does not exist.".format(self.filename))