Due to https://issues.apache.org/jira/browse/HDFS-13 ":" are not allowed in
path names. So we format paths differently for HDFS.
969372
private Path getTaskLogFileFromId(String taskId)
  {
    return new Path(mergePaths(config.getDirectory(), taskId.replace(':', '_')));
  }
1039937
@Override
  public int run(String[] args) throws Exception {

    myConf = getConf();

    // Presume this is all HDFS paths, even when access as file://
    hdfs = FileSystem.get(myConf);

    // Grab input args and allow for -Dxyz style arguments
    String[] otherArgs = new GenericOptionsParser(myConf, args)
        .getRemainingArgs();

    // Grab the arguments we're looking for.
    CommandLine commandLine = parseArgs(otherArgs);

    // Grab the input path argument
    input = commandLine.getOptionValue("i");
    LOG.info("input=" + input);

    // Grab the input path argument
    String output = commandLine.getOptionValue("o");
    LOG.info("output=" + output);

    skipExisting = commandLine.hasOption("s");
    LOG.info("skipExisting=" + skipExisting);

    moveFiles = commandLine.hasOption("m");
    LOG.info("moveFiles=" + moveFiles);

    if (skipExisting && moveFiles) {
      throw new IllegalArgumentException(
          "Cannot use both options skipExisting and move simultaneously.");
    }

    if (commandLine.hasOption("x")) {
      try {
        maXretention = Integer.parseInt(commandLine.getOptionValue("x"));
      } catch (NumberFormatException nfe) {
        throw new IllegalArgumentException(
            "maXretention option -x is is not a valid number: "
                + commandLine.getOptionValue("x"), nfe);
      }
      // Additional check
      if (maXretention < 0) {
        throw new IllegalArgumentException(
            "Cannot retain less than 0 files. Specified maXretention option -x is: "
                + commandLine.getOptionValue("x"));
      }
      LOG.info("maXretention=" + maXretention);
      if (moveFiles) {
        throw new IllegalArgumentException(
            "Cannot use both options maXretention and move simultaneously.");
      }
    } else {
      maXretention = Integer.MAX_VALUE;
    }

    outputPath = new Path(output);
    FileStatus outputFileStatus = hdfs.getFileStatus(outputPath);

    if (!outputFileStatus.isDir()) {
      throw new IOException("Output is not a directory"
          + outputFileStatus.getPath().getName());
    }

    Path inputPath = new Path(input);
    URI inputURI = inputPath.toUri();
    String inputScheme = inputURI.getScheme();

    LOG.info("input scheme is: " + inputScheme);

    // If input directory is HDFS, then process as such. Assume not scheme is
    // HDFS
    if ((inputScheme == null)
        || (hdfs.getUri().getScheme().equals(inputScheme))) {
      processHDFSSources(inputPath);
    } else if (inputScheme.equals("file")) {
      if (moveFiles) {
        throw new IllegalArgumentException(
            "Cannot move files that are not already in hdfs. Input is not HDFS: "
                + input);
      }
      processPlainFileSources(inputURI);
    } else {
      throw new IllegalArgumentException(
          "Cannot process files from this URI scheme: " + inputScheme);
    }

    Statistics statistics = FileSystem.getStatistics(outputPath.toUri()
        .getScheme(), hdfs.getClass());
    if (statistics != null) {
      LOG.info("HDFS bytes read: " + statistics.getBytesRead());
      LOG.info("HDFS bytes written: " + statistics.getBytesWritten());
      LOG.info("HDFS read ops: " + statistics.getReadOps());
      System.out
          .println("HDFS large read ops: " + statistics.getLargeReadOps());
      LOG.info("HDFS write ops: " + statistics.getWriteOps());
    }

    return 0;
  }
717048
private static String fixName(String name) {
    // convert old-format name to new-format name
    if (name.equals("local")) {         // "local" is now "file:///".
      LOG.warn("\"local\" is a deprecated filesystem name."
               +" Use \"file:///\" instead.");
      name = "file:///";
    } else if (name.indexOf('/')==-1) {   // unqualified is "hdfs://"
      LOG.warn("\""+name+"\" is a deprecated filesystem name."
               +" Use \"hdfs://"+name+"/\" instead.");
      name = "hdfs://"+name;
    }
    return name;
  }
738979
@Override
  public void setWorkingDirectory(Path hadoopPath) {
    long startTime = System.nanoTime();
    Preconditions.checkArgument(hadoopPath != null, "hadoopPath must not be null");

    logger.atFine().log("GHFS.setWorkingDirectory: %s", hadoopPath);
    URI gcsPath = FileInfo.convertToDirectoryPath(pathCodec, getGcsPath(hadoopPath));
    Path newPath = getHadoopPath(gcsPath);

    // Ideally we should check (as we did earlier) if the given path really points to an existing
    // directory. However, it takes considerable amount of time for that check which hurts perf.
    // Given that HDFS code does not do such checks either, we choose to not do them in favor of
    // better performance.

    workingDirectory = newPath;
    logger.atFine().log("GHFS.setWorkingDirectory: => %s", workingDirectory);

    long duration = System.nanoTime() - startTime;
    increment(Counter.SET_WD);
    increment(Counter.SET_WD_TIME, duration);
  }
112482
public static void copyToHDFS(String localFile, String hdfsFile) throws IOException {

    String hadoopConfPath; 

    if (Environment.getProperty("hadoop.conf.path") == null)
      hadoopConfPath = "../../../conf";
    else
      hadoopConfPath = Environment.getProperty("hadoop.conf.path");

    // Read the configuration for the Hadoop environment
    Configuration hadoopConf = new Configuration();
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-default.xml"));
    hadoopConf.addResource(new Path(hadoopConfPath + "/hadoop-site.xml"));

    // System.out.println(hadoopConf.get("hadoop.tmp.dir"));
    // System.out.println(hadoopConf.get("fs.default.name"));
    FileSystem fs = FileSystem.get(hadoopConf);

    // HadoopDFS deals with Path
    Path inFile = new Path("file://" + localFile);
    Path outFile = new Path(hadoopConf.get("fs.default.name") + hdfsFile);

     // Read from and write to new file
    Environment.logInfo("Uploading to HDFS (file " + outFile + ") ...");
    fs.copyFromLocalFile(false, inFile, outFile);
  }
457300
private void validatePathname(String pathname) {
    File file = new File(storageDir, pathname);

    String canonicalPath;
    try {
      canonicalPath = file.getCanonicalPath();
    } catch (IOException e) {
      throw new UncheckedIOException(e);
    }
    if (!canonicalPath.startsWith(storageDir)) {
      throw new UncheckedIOException(new IOException("File path traversal not allowed"));
    }
  }
796575
private boolean rename(String src, String dst) throws IOException {
    IOException te = null;
    FileSystem hdfs = getFs();
    RetryPolicy retryPolicy = new CountingRetry(MAX_TRY);
    while (retryPolicy.attempt()) {
      try {
        return hdfs.rename(new Path(src), new Path(dst));
      } catch (IOException e) {
        LOG.warn("{} try to rename {} to {} : {}", retryPolicy.getAttemptCount(), src, dst,
            e.getMessage());
        te = e;
      }
    }
    throw te;
  }
1116563
@VisibleForTesting
  boolean couldUseFlatGlob(Path fixedPath) {
    // Only works for filesystems where the base Hadoop Path scheme matches the underlying URI
    // scheme for GCS.
    if (!getUri().getScheme().equals(GoogleCloudStorageFileSystem.SCHEME)) {
      logger.atFine().log(
          "Flat glob is on, but doesn't work for scheme '%s'; using default behavior.",
          getUri().getScheme());
      return false;
    }

    // The full pattern should have a wildcard, otherwise there's no point doing the flat glob.
    GlobPattern fullPattern = new GlobPattern(fixedPath.toString());
    if (!fullPattern.hasWildcard()) {
      logger.atFine().log(
          "Flat glob is on, but Path '%s' has no wildcard; using default behavior.", fixedPath);
      return false;
    }

    // To use a flat glob, there must be an authority defined.
    if (Strings.isNullOrEmpty(fixedPath.toUri().getAuthority())) {
      logger.atInfo().log(
          "Flat glob is on, but Path '%s' has a empty authority, using default behavior.",
          fixedPath);
      return false;
    }

    // And the authority must not contain a wildcard.
    GlobPattern authorityPattern = new GlobPattern(fixedPath.toUri().getAuthority());
    if (authorityPattern.hasWildcard()) {
      logger.atInfo().log(
          "Flat glob is on, but Path '%s' has a wildcard authority, using default behavior.",
          fixedPath);
      return false;
    }

    return true;
  }
800475
def _render_filenames(filenames, zip_file, saltenv, template):
    '''
    
    '''
    if not template:
        return (filenames, zip_file)

    # render the path as a template using path_template_engine as the engine
    if template not in salt.utils.templates.TEMPLATE_REGISTRY:
        raise CommandExecutionError(
            'Attempted to render file paths with unavailable engine '
            '{0}'.format(template)
        )

    kwargs = {}
    kwargs['salt'] = __salt__
    kwargs['pillar'] = __pillar__
    kwargs['grains'] = __grains__
    kwargs['opts'] = __opts__
    kwargs['saltenv'] = saltenv

    def _render(contents):
        '''
        Render :param:`contents` into a literal pathname by writing it to a
        temp file, rendering that file, and returning the result.
        '''
        # write out path to temp file
        tmp_path_fn = salt.utils.files.mkstemp()
        with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_:
            fp_.write(salt.utils.stringutils.to_str(contents))
        data = salt.utils.templates.TEMPLATE_REGISTRY[template](
            tmp_path_fn,
            to_str=True,
            **kwargs
        )
        salt.utils.files.safe_rm(tmp_path_fn)
        if not data['result']:
            # Failed to render the template
            raise CommandExecutionError(
                'Failed to render file path with error: {0}'.format(
                    data['data']
                )
            )
        else:
            return data['data']

    filenames = _render(filenames)
    zip_file = _render(zip_file)
    return (filenames, zip_file)
41072
private void initFileDirTables(Path path) throws IOException {
    FileStatus[] stats = fs.listStatus(path);
    if (stats != null) {
      for (FileStatus stat : stats) {
        if (stat.isDir()) {
          dirs.add(stat.getPath().toString());
          initFileDirTables(stat.getPath());
        } else {
          Path filePath = stat.getPath();
          if (filePath.getName().startsWith(StructureGenerator.FILE_NAME_PREFIX)) {
            files.add(filePath.toString());
          }
        }
      }
    }
  }
319098
List<CopyableFile.Builder> getCopyableFilesFromPaths(Collection<FileStatus> paths,
      CopyConfiguration configuration, Optional<Partition> partition) throws IOException {
    List<CopyableFile.Builder> builders = Lists.newArrayList();
    List<SourceAndDestination> dataFiles = Lists.newArrayList();

    Configuration hadoopConfiguration = new Configuration();
    FileSystem actualSourceFs = null;
    String referenceScheme = null;
    String referenceAuthority = null;

    for (FileStatus status : paths) {
      dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true)));
    }

    for (SourceAndDestination sourceAndDestination : dataFiles) {

      URI uri = sourceAndDestination.getSource().getPath().toUri();
      if (actualSourceFs == null || !StringUtils.equals(referenceScheme, uri.getScheme())
          || !StringUtils.equals(referenceAuthority, uri.getAuthority())) {
        actualSourceFs = sourceAndDestination.getSource().getPath().getFileSystem(hadoopConfiguration);
        referenceScheme = uri.getScheme();
        referenceAuthority = uri.getAuthority();
      }

      if (!this.dataset.getTableRootPath().isPresent()) {
        // The logic for computing ancestor owner and permissions for hive copies depends on tables having a non-glob
        // location. Currently, this restriction is also imposed by Hive, so this is not a problem. If this ever changes
        // on the Hive side, and we try to copy a table with a glob location, this logic will have to change.
        throw new IOException(String.format("Table %s does not have a concrete table root path.",
            this.dataset.getTable().getCompleteName()));
      }
      List<OwnerAndPermission> ancestorOwnerAndPermission =
          CopyableFile.resolveReplicatedOwnerAndPermissionsRecursively(actualSourceFs,
              sourceAndDestination.getSource().getPath().getParent(), this.dataset.getTableRootPath().get().getParent(), configuration);

      builders.add(CopyableFile.fromOriginAndDestination(actualSourceFs, sourceAndDestination.getSource(),
          sourceAndDestination.getDestination(), configuration).
          ancestorsOwnerAndPermission(ancestorOwnerAndPermission));
    }

    return builders;
  }
878661
private static void checkSrcPath(Configuration conf, List<Path> srcPaths
      ) throws IOException {
    List<IOException> rslt = new ArrayList<IOException>();
    List<Path> unglobbed = new LinkedList<Path>();
    for (Path p : srcPaths) {
      FileSystem fs = p.getFileSystem(conf);
      FileStatus[] inputs = fs.globStatus(p);

      if(inputs != null && inputs.length > 0) {
        for (FileStatus onePath: inputs) {
          unglobbed.add(onePath.getPath());
        }
      } else {
        rslt.add(new IOException("Input source " + p + " does not exist."));
      }
    }
    if (!rslt.isEmpty()) {
      throw new InvalidInputException(rslt);
    }
    srcPaths.clear();
    srcPaths.addAll(unglobbed);
  }
1045317
private static void traverseDirs(List<FileStatus> fileStatusesList, FileSystem hdfs,
      Path inputPath, JobFileModifiedRangePathFilter jobFileModifiedRangePathFilter)
          throws IOException
  {
    // get all the files and dirs in the current dir
    FileStatus allFiles[] = hdfs.listStatus(inputPath);
    for (FileStatus aFile: allFiles) {
      if (aFile.isDir()) {
        //recurse here
        traverseDirs(fileStatusesList, hdfs, aFile.getPath(), jobFileModifiedRangePathFilter);
      }
      else {
        // check if the pathFilter is accepted for this file
        if (jobFileModifiedRangePathFilter.accept(aFile.getPath())) {
          fileStatusesList.add(aFile);
        }
      }
    }
  }
988258
def not_found(res, path)
      path = ::Rack::Utils.escape_html(path)
      res.status = 404
      res.write "<html><head></head><body><h1>File Not Found</h1><p>#{path}</p></body></html>"
      res.finish
    end
1118755
def rm(hdfs_path, recursive=True, user=None):
    
    host, port, path_ = path.split(hdfs_path, user)
    fs = hdfs(host, port, user)
    retval = fs.delete(path_, recursive=recursive)
    fs.close()
    return retval
1188007
def ensure_storage_format(root_dir):
    
    if not os.path.isdir(os.path.join(root_dir, 'blobs')):
        print('"blobs/" directory not found')
        sys.exit(1)

    if not os.path.isdir(os.path.join(root_dir, 'links')):
        print('"links/" directory not found')
        sys.exit(1)

    if not os.path.isdir(os.path.join(root_dir, 'db')):
        print('"db/" directory not found')
        sys.exit(1)
1123298
private static final HadoopPath toHadoopPath(Path path) {
    if (path == null) {
      throw new NullPointerException();
    }
    if (!(path instanceof HadoopPath)) {
      throw new ProviderMismatchException();
    }
    return (HadoopPath) path;
  }