Automatically determine datasets provided by this file
557200
def available_datasets(self):
        """"""
        sensor = self.get_sensor(self['/attr/sensor'])
        nadir_resolution = self.get_nadir_resolution(sensor)
        for var_name, val in self.file_content.items():
            if isinstance(val, SDS):
                ds_info = {
                    'file_type': self.filetype_info['file_type'],
                    'resolution': nadir_resolution,
                }
                if self._is_polar():
                    ds_info['coordinates'] = ['longitude', 'latitude']
                yield DatasetID(name=var_name, resolution=nadir_resolution), ds_info
701442
def identifiers(dataset_uri):
    """"""
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    for i in dataset.identifiers:
        click.secho(i)
159741
def __parseDatasets(self):
        """"""
        datasets = []
        if self.__dataItem.has_key('dataSets'):
            for dataset in self.__dataItem['dataSets']:
                datasets.append(DataSet(Sitools2Abstract.getBaseUrl(self), dataset))
        return datasets
938895
def add_ds_ids_from_files(self):
        """"""
        for file_handlers in self.file_handlers.values():
            try:
                fh = file_handlers[0]
                avail_ids = fh.available_datasets()
            except NotImplementedError:
                continue

            # dynamically discover other available datasets
            for ds_id, ds_info in avail_ids:
                # don't overwrite an existing dataset
                # especially from the yaml config
                coordinates = ds_info.get('coordinates')
                if isinstance(coordinates, list):
                    # xarray doesn't like concatenating attributes that are
                    # lists: https://github.com/pydata/xarray/issues/2060
                    ds_info['coordinates'] = tuple(ds_info['coordinates'])
                self.ids.setdefault(ds_id, ds_info)
539538
def main():
    """"""
    parser = argparse.ArgumentParser(description='A Python tool that automatically cleans data sets and readies them for analysis')

    parser.add_argument('INPUT_FILENAME', type=str, help='File name of the data file to clean')

    parser.add_argument('-cv', action='store', dest='CROSS_VAL_FILENAME', default=None,
                         type=str, help='File name for the validation data set if performing cross-validation')

    parser.add_argument('-o', action='store', dest='OUTPUT_FILENAME', default=None,
                        type=str, help='Data file to output the cleaned data set to')

    parser.add_argument('-cvo', action='store', dest='CV_OUTPUT_FILENAME', default=None,
                        type=str, help='Data file to output the cleaned cross-validation data set to')

    parser.add_argument('-is', action='store', dest='INPUT_SEPARATOR', default='\t',
                        type=str, help='Column separator for the input file(s) (default: \\t)')
                    
    parser.add_argument('-os', action='store', dest='OUTPUT_SEPARATOR', default='\t',
                        type=str, help='Column separator for the output file(s) (default: \\t)')

    parser.add_argument('--drop-nans', action='store_true', dest='DROP_NANS', default=False,
                        help='Drop all rows that have a NaN in any column (default: False)')
                        
    parser.add_argument('--ignore-update-check', action='store_true', dest='IGNORE_UPDATE_CHECK', default=False,
                        help='Do not check for the latest version of datacleaner (default: False)')

    parser.add_argument('--version', action='version', version='datacleaner v{version}'.format(version=__version__))

    args = parser.parse_args()

    input_data = pd.read_csv(args.INPUT_FILENAME, sep=args.INPUT_SEPARATOR)
    if args.CROSS_VAL_FILENAME is None:
        clean_data = autoclean(input_data, drop_nans=args.DROP_NANS, ignore_update_check=args.IGNORE_UPDATE_CHECK)
        if args.OUTPUT_FILENAME is None:
            print('Cleaned data set:')
            print(clean_data)
            print('')
            print('If you cannot view the entire data set, output it to a file instead. '
                  'Type datacleaner --help for more information.')
        else:
            clean_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
    else:
        if args.OUTPUT_FILENAME is not None and args.CV_OUTPUT_FILENAME is None:
            print('You must specify both output file names. Type datacleaner --help for more information.')
            return
    
        cross_val_data = pd.read_csv(args.CROSS_VAL_FILENAME, sep=args.INPUT_SEPARATOR)
        clean_training_data, clean_testing_data = autoclean_cv(input_data, cross_val_data,
                                                               drop_nans=args.DROP_NANS,
                                                               ignore_update_check=args.IGNORE_UPDATE_CHECK)

        if args.OUTPUT_FILENAME is None:
            print('Cleaned training data set:')
            print(clean_training_data)
            print('')
            print('Cleaned testing data set:')
            print(clean_testing_data)
            print('')
            print('If you cannot view the entire data set, output it to a file instead. '
                  'Type datacleaner --help for more information.')
        else:
            clean_training_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
            clean_testing_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
811353
def cli():
    """"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "dataset_uri",
        help="Dtool dataset URI"
    )
    parser.add_argument(
        "-q",
        "--quiet",
        action="store_true",
        help="Only return the http URI"
    )
    args = parser.parse_args()
    access_uri = publish(args.dataset_uri)

    if args.quiet:
        print(access_uri)
    else:
        print("Dataset accessible at: {}".format(access_uri))
105451
def gcs_dataset_info_files(dataset_dir):
  """"""
  prefix = posixpath.join(GCS_DATASET_INFO_DIR, dataset_dir, "")
  # Filter for this dataset
  filenames = [el for el in gcs_files(prefix_filter=prefix)
               if el.startswith(prefix) and len(el) > len(prefix)]
  return filenames
1267879
def parse_GDS(filepath):
    
    dataset_lines = []
    subsets = {}
    database = None
    dataset_name = None
    with utils.smart_open(filepath) as soft:
        groupper = groupby(soft, lambda x: x.startswith("^"))
        for is_new_entry, group in groupper:
            if is_new_entry:
                entry_type, entry_name = __parse_entry(next(group))
                logger.debug("%s: %s" % (entry_type.upper(), entry_name))
                if entry_type == "SUBSET":
                    is_data, data_group = next(groupper)
                    message = ("The key is not False, probably there is an "
                               "error in the SOFT file")
                    assert not is_data, message
                    subset_metadata = parse_metadata(data_group)
                    subsets[entry_name] = GDSSubset(name=entry_name,
                                                    metadata=subset_metadata)
                elif entry_type == "DATABASE":

                    is_data, data_group = next(groupper)
                    message = ("The key is not False, probably there is an "
                               "error in the SOFT file")
                    assert not is_data, message
                    database_metadata = parse_metadata(data_group)
                    database = GEODatabase(name=entry_name,
                                           metadata=database_metadata)
                elif entry_type == "DATASET":
                    is_data, data_group = next(groupper)
                    dataset_name = entry_name
                    for line in data_group:
                        dataset_lines.append(line.rstrip())
                else:
                    logger.error("Cannot recognize type %s" % entry_type)

    metadata = parse_metadata(dataset_lines)
    columns = parse_GDS_columns(dataset_lines, subsets)
    table = parse_table_data(dataset_lines)
    return GDS(name=dataset_name, metadata=metadata, columns=columns,
               table=table, subsets=subsets, database=database)
500031
def _set_data_filelist(self,
                           start_path,
                           attr_name,
                           conn,
                           file_exts=[],
                           dir_filter=set()):
        '''  '''

        def filter_path(filter_terms, dir_path):
            """ sees if any of the terms are present in the path if so returns
                True

            args:
                filter_terms(list): terms to check
                dir_path: the path of the directory
            """
            if filter_terms.intersection(set(dir_path.split(os.path.sep))):
                return True
            else:
                return False

        data_obj = {}
        files_dict = {}
        latest_mod = 0
        dir_filter = set(dir_filter)
        for root, dirnames, filenames in os.walk(start_path):
            if not dir_filter or filter_path(dir_filter, root):
                if file_exts:
                    filenames = [x for x in filenames
                                 if x.split('.')[-1].lower() in file_exts]
                files_dict[root] = filenames
                for def_file in filenames:
                    file_mod = os.path.getmtime(os.path.join(root,def_file))
                    if file_mod > latest_mod:
                        latest_mod = file_mod
        data_obj['latest_mod'] = latest_mod
        data_obj['files'] = files_dict
        json_mod = 0
        cache_path = os.path.join(CFG.CACHE_DATA_PATH, attr_name)
        if cache_path:
            for root, dirnames, filenames in os.walk(cache_path):
                for json_file in filenames:
                    file_mod = os.path.getmtime(os.path.join(root,json_file))
                    if file_mod > json_mod:
                        json_mod = file_mod
        data_obj['last_json_mod'] = json_mod
        data_obj['conn'] = conn
        data_obj['cache_path'] = cache_path
        self.datafile_obj[attr_name] = data_obj
185782
def data_available(dataset_name=None):
    """"""
    dr = data_resources[dataset_name]
    if 'dirs' in dr:
        for dirs, files in zip(dr['dirs'], dr['files']):
            for dir, file in zip(dirs, files):
                if not os.path.exists(os.path.join(data_path, dataset_name, dir, file)):
                    return False
    else:
        for file_list in dr['files']:
            for file in file_list:
                if not os.path.exists(os.path.join(data_path, dataset_name, file)):
                    return False
    return True
657498
def generate_matching_datasets(self, data_slug):
        """"""
        tag = Tag.objects.filter(slug=data_slug)
        try:
            return tag[0].dataset_set.all().order_by('-uploaded_by')
        except IndexError:
            return None
587463
def write_data_set(file_path, dataset, output_dir, color_array_info, new_name=None, compress=True):
    """"""
    fileName = new_name if new_name else os.path.basename(file_path)
    dataset_dir = os.path.join(output_dir, fileName)
    data_dir = os.path.join(dataset_dir, 'data')

    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    root = {}
    root['metadata'] = {}
    root['metadata']['name'] = fileName

    writer = writer_mapping[dataset.GetClassName()]
    if writer:
        writer(dataset_dir, data_dir, dataset, color_array_info, root, compress)
    else:
        print(dataObject.GetClassName(), 'is not supported')

    with open(os.path.join(dataset_dir, "index.json"), 'w') as f:
        f.write(json.dumps(root, indent=2))

    return dataset_dir
103838
def list_datasets():
    """"""
    lines = []
    for name, resource in itertools.chain(LOCAL_DATASETS.items(), REMOTE_DATASETS.items()):

        if isinstance(resource, LocalFileMetadata):
            location = "local: {}".format(resource.filename)
        elif isinstance(resource, RemoteFileMetadata):
            location = "remote: {}".format(resource.url)
        else:
            location = "unknown"
        lines.append("{}\n{}\n{}\n{}".format(name, "=" * len(name), resource.description, location))

    return "\n\n{}\n\n".format(10 * "-").join(lines)
1168471
def _default_output_dir():
  """"""
  try:
    dataset_name = gin.query_parameter("inputs.dataset_name")
  except ValueError:
    dataset_name = "random"
  dir_name = "{model_name}_{dataset_name}_{timestamp}".format(
      model_name=gin.query_parameter("train.model").configurable.name,
      dataset_name=dataset_name,
      timestamp=datetime.datetime.now().strftime("%Y%m%d_%H%M"),
  )
  dir_path = os.path.join("~", "trax", dir_name)
  print()
  trax.log("No --output_dir specified")
  return dir_path
816804
def display_dataset(self):
        """"""
        header = self.dataset.header

        self.parent.setWindowTitle(basename(self.filename))
        short_filename = short_strings(basename(self.filename))
        self.idx_filename.setText(short_filename)
        self.idx_s_freq.setText(str(header['s_freq']))
        self.idx_n_chan.setText(str(len(header['chan_name'])))
        start_time = header['start_time'].strftime('%b-%d %H:%M:%S')
        self.idx_start_time.setText(start_time)
        end_time = (header['start_time'] +
                    timedelta(seconds=header['n_samples'] / header['s_freq']))
        self.idx_end_time.setText(end_time.strftime('%b-%d %H:%M:%S'))
7835
def dataset_generator(filepath,
                      dataset,
                      chunk_size=1,
                      start_idx=None,
                      end_idx=None):
  """"""
  encoder = dna_encoder.DNAEncoder(chunk_size=chunk_size)
  with h5py.File(filepath, "r") as h5_file:
    # Get input keys from h5_file
    src_keys = [s % dataset for s in ["%s_in", "%s_na", "%s_out"]]
    src_values = [h5_file[k] for k in src_keys]
    inp_data, mask_data, out_data = src_values
    assert len(set([v.len() for v in src_values])) == 1

    if start_idx is None:
      start_idx = 0
    if end_idx is None:
      end_idx = inp_data.len()

    for i in range(start_idx, end_idx):
      if i % 100 == 0:
        print("Generating example %d for %s" % (i, dataset))
      inputs, mask, outputs = inp_data[i], mask_data[i], out_data[i]
      ex_dict = to_example_dict(encoder, inputs, mask, outputs)
      # Original data has one output for every 128 input bases. Ensure that the
      # ratio has been maintained given the chunk size and removing EOS.
      assert (len(ex_dict["inputs"]) - 1) == ((
          128 // chunk_size) * ex_dict["targets_shape"][0])
      yield ex_dict
1090144
def _checksum_paths():
  """"""
  dataset2path = {}
  for dir_path in _CHECKSUM_DIRS:
    for fname in _list_dir(dir_path):
      if not fname.endswith(_CHECKSUM_SUFFIX):
        continue
      fpath = os.path.join(dir_path, fname)
      dataset_name = fname[:-len(_CHECKSUM_SUFFIX)]
      dataset2path[dataset_name] = fpath
  return dataset2path