Retrieves the dataset dir on HDFS associated with the given {@link ConfigKeyPath} and the given version. This
directory contains the {@link #MAIN_CONF_FILE_NAME} and {@link #INCLUDES_CONF_FILE_NAME} file, as well as any child
datasets.
183387
private Path getDatasetDirForKey(ConfigKeyPath configKey, String version) throws VersionDoesNotExistException {
    String datasetFromConfigKey = getDatasetFromConfigKey(configKey);

    if (StringUtils.isBlank(datasetFromConfigKey)) {
      return getVersionRoot(version);
    }

    return new Path(getVersionRoot(version), datasetFromConfigKey);
  }
544979
@Override
  public String getStorageDir(DataSegment segment, boolean useUniquePath)
  {
    // This is only called by HdfsDataSegmentPusher.push(), which will always set useUniquePath to false since any
    // 'uniqueness' will be applied not to the directory but to the filename along with the shard number. This is done
    // to avoid performance issues due to excessive HDFS directories. Hence useUniquePath is ignored here and we
    // expect it to be false.
    Preconditions.checkArgument(
        !useUniquePath,
        "useUniquePath must be false for HdfsDataSegmentPusher.getStorageDir()"
    );

    return JOINER.join(
        segment.getDataSource(),
        StringUtils.format(
            "%s_%s",
            segment.getInterval().getStart().toString(ISODateTimeFormat.basicDateTime()),
            segment.getInterval().getEnd().toString(ISODateTimeFormat.basicDateTime())
        ),
        segment.getVersion().replace(':', '_')
    );
  }
711524
@Override
  public Collection<ConfigKeyPath> getChildren(ConfigKeyPath configKey) {
    try {
      return this.childrenMap.get(configKey, () -> this.fallback.getChildren(configKey));
    } catch (ExecutionException ee) {
      throw new RuntimeException(ee);
    }
  }
1051577
public static void ensureLocationExists(
      DatasetDescriptor descriptor, Configuration conf) {
    Preconditions.checkNotNull(descriptor.getLocation(),
        "Cannot get FileSystem for a descriptor with no location");

    Path dataPath = new Path(descriptor.getLocation().toString());
    FileSystem fs = null;

    try {
      fs = dataPath.getFileSystem(conf);
    } catch (IOException e) {
      throw new DatasetIOException(
          "Cannot get FileSystem for descriptor: " + descriptor, e);
    }

    try {
      if (!fs.exists(dataPath)) {
        fs.mkdirs(dataPath);
      }
    } catch (IOException e) {
      throw new DatasetIOException("Cannot access data location", e);
    }
  }
761322
def get_dataset_path(*args):
    
    d = os.environ.get('TENSORPACK_DATASET', None)
    if d is None:
        d = os.path.join(os.path.expanduser('~'), 'tensorpack_data')
        if execute_only_once():
            logger.warn("Env var $TENSORPACK_DATASET not set, using {} for datasets.".format(d))
        if not os.path.isdir(d):
            mkdir_p(d)
            logger.info("Created the directory {}.".format(d))
    assert os.path.isdir(d), d
    return os.path.join(d, *args)
319098
List<CopyableFile.Builder> getCopyableFilesFromPaths(Collection<FileStatus> paths,
      CopyConfiguration configuration, Optional<Partition> partition) throws IOException {
    List<CopyableFile.Builder> builders = Lists.newArrayList();
    List<SourceAndDestination> dataFiles = Lists.newArrayList();

    Configuration hadoopConfiguration = new Configuration();
    FileSystem actualSourceFs = null;
    String referenceScheme = null;
    String referenceAuthority = null;

    for (FileStatus status : paths) {
      dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true)));
    }

    for (SourceAndDestination sourceAndDestination : dataFiles) {

      URI uri = sourceAndDestination.getSource().getPath().toUri();
      if (actualSourceFs == null || !StringUtils.equals(referenceScheme, uri.getScheme())
          || !StringUtils.equals(referenceAuthority, uri.getAuthority())) {
        actualSourceFs = sourceAndDestination.getSource().getPath().getFileSystem(hadoopConfiguration);
        referenceScheme = uri.getScheme();
        referenceAuthority = uri.getAuthority();
      }

      if (!this.dataset.getTableRootPath().isPresent()) {
        // The logic for computing ancestor owner and permissions for hive copies depends on tables having a non-glob
        // location. Currently, this restriction is also imposed by Hive, so this is not a problem. If this ever changes
        // on the Hive side, and we try to copy a table with a glob location, this logic will have to change.
        throw new IOException(String.format("Table %s does not have a concrete table root path.",
            this.dataset.getTable().getCompleteName()));
      }
      List<OwnerAndPermission> ancestorOwnerAndPermission =
          CopyableFile.resolveReplicatedOwnerAndPermissionsRecursively(actualSourceFs,
              sourceAndDestination.getSource().getPath().getParent(), this.dataset.getTableRootPath().get().getParent(), configuration);

      builders.add(CopyableFile.fromOriginAndDestination(actualSourceFs, sourceAndDestination.getSource(),
          sourceAndDestination.getDestination(), configuration).
          ancestorsOwnerAndPermission(ancestorOwnerAndPermission));
    }

    return builders;
  }
629924
@Override
  public Configuration getMainConfiguration() {

    return Tang.Factory.getTang().newConfigurationBuilder()
            .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
            .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
            .build();
  }
687165
@Override
  public Config getOwnConfig(final ConfigKeyPath configKey) {
    try {
      return this.ownConfigCache.get(configKey, new Callable<Config>() {
        @Override
        public Config call()  {
          return InMemoryValueInspector.this.valueFallback.getOwnConfig(configKey);
        }
      });
    } catch (ExecutionException e) {
      // should NOT come here
      throw new RuntimeException("Can not getOwnConfig for " + configKey);
    }
  }
426390
def get_legacy_storage_path(self):
        
        config_dir = os.path.dirname(
            self.py3_wrapper.config.get("i3status_config_path", "/tmp")
        )
        storage_path = os.path.join(config_dir, "py3status.data")
        if os.path.exists(storage_path):
            return storage_path
        else:
            return None
1183932
def getConfigDirectory():
		
		if platform.system() == 'Windows':
			return os.path.join(os.environ['APPDATA'], 'ue4cli')
		else:
			return os.path.join(os.environ['HOME'], '.config', 'ue4cli')
213182
protected Config getConfig (String packagePath)
    {
        Config c = _configs.get(packagePath);
        if (c == null) {
            Properties props = new Properties();
            String propFilename = packagePath + Sounds.PROP_NAME + ".properties";
            try {
                props = ConfigUtil.loadInheritedProperties(propFilename, _rmgr.getClassLoader());
            } catch (IOException ioe) {
                log.warning("Failed to load sound properties", "filename", propFilename, ioe);
            }
            c = new Config(props);
            _configs.put(packagePath, c);
        }
        return c;
    }
857722
def _get_learning_data_path(self):
        
        for res in self.dsDoc['dataResources']:
            resPath = res['resPath']
            resType = res['resType']

            dirname = os.path.basename(os.path.normpath(os.path.dirname(resPath)))

            if resType == 'table' and dirname == 'tables':
                if 'learningData.csv' in res['resPath']:
                    return os.path.join(self.dsHome, resPath)

        # if the for loop is over and learningDoc is not found, then return None
        raise RuntimeError('could not find learningData file the dataset')
1213678
@Nullable
  static SourceFile getRelativePath(String baseFilePath, String relativePath) {
    return SourceFile.fromPath(
        FileSystems.getDefault().getPath(baseFilePath).resolveSibling(relativePath).normalize(),
        UTF_8);
  }
44245
protected String getDirectory() {
        //get the default filepath of the permanet storage file
        String directory = storageConfiguration.getDirectory();

        if(directory == null || directory.isEmpty()) {
            directory = System.getProperty("java.io.tmpdir"); //write them to the tmp directory
        }

        return directory;
    }
525183
def get_distribution_dir(catalog_id, dataset_id, distribution_id,
                         catalogs_dir=CATALOGS_DIR, use_short_path=False):
    """"""
    if use_short_path:
        catalog_path = os.path.join(catalogs_dir, "catalog", catalog_id)
        distribution_dir = os.path.join(catalog_path, dataset_id)
    else:
        catalog_path = os.path.join(catalogs_dir, "catalog", catalog_id)
        dataset_path = os.path.join(catalog_path, "dataset", dataset_id)
        distribution_dir = os.path.join(
            dataset_path, "distribution", distribution_id)

    return os.path.abspath(distribution_dir)
841555
private boolean scanHadoopConfigFiles(final ServerInformationCatalog serverInformationCatalog,
            final String selectedServer) {

        final HadoopClusterInformation clusterInformation;
        if (selectedServer != null) {
            clusterInformation = (HadoopClusterInformation) serverInformationCatalog.getServer(selectedServer);
        } else {
            clusterInformation = (HadoopClusterInformation) serverInformationCatalog
                    .getServer(HadoopResource.DEFAULT_CLUSTERREFERENCE);
        }

        if (clusterInformation == null) {
            return false;
        }

        final Configuration configuration = HdfsUtils.getHadoopConfigurationWithTimeout(clusterInformation);

        _currentDirectory = new Path("/");

        try {
            _fileSystem = FileSystem.newInstance(configuration);
        } catch (final IOException e) {
            throw new IllegalArgumentException("Illegal URI when showing HDFS chooser", e);
        }
        final HdfsDirectoryModel model = (HdfsDirectoryModel) _fileList.getModel();
        model.updateFileList();
        return model._files.length > 0;
    }
931086
def dataset_path(cache=None, cachefile="~/.io3d_cache.yaml", get_root=False):
    
    local_data_dir = local_dir

    if cachefile is not None:
        cache = cachef.CacheFile(cachefile)
        # cache.update('local_dataset_dir', head)
    if cache is not None:
        local_data_dir = cache.get_or_save_default("local_dataset_dir", local_dir)

    if get_root:
        local_data_dir
    else:
        logger.warning("Parameter")
        local_data_dir = op.join(local_data_dir, "medical", "orig")

    return op.expanduser(local_data_dir)