Iterates through files in the directory dir, and counts the number
of usages of datanodes for the files in the directory dir.
Just iterates through threshold number of blocks.
424609
private HashMap<DatanodeDescriptor, Integer> directoryDataNodeUsage(INodeDirectory dir,
      int threshold) {
    HashMap<DatanodeDescriptor, Integer> dataNodeUsage = new HashMap<DatanodeDescriptor, Integer>();

    List<INode> children;
    nameSystem.readLock();
    try {
      if (dir.getChildrenRaw() == null) {
        return dataNodeUsage;
      }
      children = new ArrayList<INode>(dir.getChildrenRaw());
      Collections.shuffle(children);

      for (INode node : children) {
        if (!(node instanceof INodeFile)) { // The condition is always false.
          continue;
        }

        INodeFile file = (INodeFile) node;
        BlockInfo[] blocks = file.getBlocks();

        for (BlockInfo block : blocks) {
          if (threshold == 0) {
            return dataNodeUsage;
          }

          int replication = block.numNodes();
          for (int i = 0; i < replication; i++) {
            DatanodeDescriptor datanode = block.getDatanode(i);
            Integer currentUsage = dataNodeUsage.get(datanode);
            dataNodeUsage.put(datanode, currentUsage == null ? 1 : currentUsage + 1);
          }
          threshold--;
        }
      }
    } finally {
      nameSystem.readUnlock();
    }
    return dataNodeUsage;
  }
1214659
def DirContains(self,f) :
        
        def match(fsNode) :
            if not fsNode.isdir() : return False 
            for c in fsNode.children() :
                if f(c) : return True
            return False
        return self.make_return(match)
19007
def count_files(admin_sets)
        file_counts = Hash.new(0)
        admin_sets.each do |admin_set|
          query = "{!join from=file_set_ids_ssim to=id}isPartOf_ssim:#{admin_set.id}"
          file_results = ActiveFedora::SolrService.instance.conn.get(
            ActiveFedora::SolrService.select_path,
            params: { fq: [query, "has_model_ssim:FileSet"],
                      rows: 0 }
          )
          file_counts[admin_set.id] = file_results['response']['numFound']
        end
        file_counts
      end
657719
def diskusage(path):
    '''
    
    '''

    total_size = 0
    seen = set()
    if os.path.isfile(path):
        stat_structure = os.stat(path)
        ret = stat_structure.st_size
        return ret

    for dirpath, dirnames, filenames in salt.utils.path.os_walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)

            try:
                stat_structure = os.stat(fp)
            except OSError:
                continue

            if stat_structure.st_ino in seen:
                continue

            seen.add(stat_structure.st_ino)

            total_size += stat_structure.st_size

    ret = total_size
    return ret
561229
def list_files(self, node=None):
        

        return [file_node.path for file_node in self.list_file_nodes(node) if file_node.path]
730403
def recursive_file_count(files, item=None, checksum=False):
    """"""
    if not isinstance(files, (list, set)):
        files = [files]
    total_files = 0
    if checksum is True:
        md5s = [f.get('md5') for f in item.files]
    else:
        md5s = list()
    if isinstance(files, dict):
        # make sure to use local filenames.
        _files = files.values()
    else:
        if isinstance(files[0], tuple):
            _files = dict(files).values()
        else:
            _files = files
    for f in _files:
        try:
            is_dir = os.path.isdir(f)
        except TypeError:
            try:
                f = f[0]
                is_dir = os.path.isdir(f)
            except (AttributeError, TypeError):
                is_dir = False
        if is_dir:
            for x, _ in iter_directory(f):
                lmd5 = get_md5(open(x, 'rb'))
                if lmd5 in md5s:
                    continue
                else:
                    total_files += 1
        else:
            try:
                lmd5 = get_md5(open(f, 'rb'))
            except TypeError:
                # Support file-like objects.
                lmd5 = get_md5(f)
            if lmd5 in md5s:
                continue
            else:
                total_files += 1
    return total_files
305357
@Override
    public void scanDirectoryNoResults(
            File directory, FileFilter fileFilter, Function<Resource, ?> fileProcessor) {
        this.scanDirectory(directory, null, fileFilter, fileProcessor);
    }
749302
def iter_directory(directory):
    """"""
    for path, dir, files in os.walk(directory):
        for f in files:
            filepath = os.path.join(path, f)
            key = os.path.relpath(filepath, directory)
            yield (filepath, key)
337360
def collect_read_counts(data, work_dir):
    
    out_file = os.path.join(work_dir, "%s-target-coverage.hdf5" % dd.get_sample_name(data))
    if not utils.file_exists(out_file):
        with file_transaction(data, out_file) as tx_out_file:
            params = ["-T", "CollectReadCounts", "-I", dd.get_align_bam(data),
                      "-L", tz.get_in(["regions", "bins", "target"], data),
                      "--interval-merging-rule", "OVERLAPPING_ONLY",
                      "-O", tx_out_file, "--format", "HDF5"]
            _run_with_memory_scaling(params, tx_out_file, data)
    return out_file
54791
private static void usage() {
    new HelpFormatter().printHelp(String.format(
        "java -cp %s %s -type <[READ, WRITE]> -fileSize <fileSize> -iterations <iterations> "
            + "-concurrency <concurrency>",
        RuntimeConstants.ALLUXIO_JAR, MiniBenchmark.class.getCanonicalName()),
        "run a mini benchmark to write or read a file",
        OPTIONS, "", true);
  }
5702
function (options, iterationInfo) {
        var files;
        var modules = {};

        // check if the counter has reached the desired depth indicated in options.depth
        if (typeof options.depth !== 'undefined' &&
            iterationInfo.currentDepth >= options.depth) {
            return;
        }

        // remember the starting directory
        try {
            files = fs.readdirSync(iterationInfo.dirName);
        } catch (e) {
            if (options.optional)
                return {};
            else
                throw new Error('Directory not found: ' + iterationInfo.dirName);
        }

        // iterate through files in the current directory
        files.forEach(function (fileName) {
            iterationInfo.fileName = fileName;
            //get absolute path
            iterationInfo.absolutePath = rtrim(iterationInfo.dirName, '/');
            iterationInfo.absoluteFullPath = iterationInfo.absolutePath + '/' + fileName;
            //get relative path
            iterationInfo.relativePath = iterationInfo.absolutePath.replace(iterationInfo.initialDirName, '');
            iterationInfo.relativeFullPath = iterationInfo.absoluteFullPath.replace(iterationInfo.initialDirName, '');
            // make sure a slash exists on the left side of relative paths
            iterationInfo.relativePath = '/' + ltrim(iterationInfo.relativePath, '/');
            iterationInfo.relativeFullPath = '/' + ltrim(iterationInfo.relativeFullPath, '/');

            if (fs.statSync(iterationInfo.absoluteFullPath).isDirectory()) {
                processDirectory(options, iterationInfo, modules)
            }
            else {
                processFile(options, iterationInfo, modules);
            }
        });

        return modules;
    }
216537
private RemoteIterator<LocatedFileStatus> iteratorListing(final String src)
  throws IOException {
    return new RemoteIterator<LocatedFileStatus>() {
      private LocatedDirectoryListing thisListing;
      private int i;

      { // initializer
        // fetch the first batch of entries in the directory

        thisListing = namenode.getLocatedPartialListing(
            src, HdfsFileStatus.EMPTY_NAME);
        if (thisListing == null) { // the directory does not exist
          throw new FileNotFoundException("File " + src + " does not exist.");
        }
      }

      @Override
      public boolean hasNext() throws IOException {
        if (i>=thisListing.getPartialListing().length
            && thisListing.hasMore()) {
          // current listing is exhausted & fetch a new listing
          thisListing = namenode.getLocatedPartialListing(
              src, thisListing.getLastName());
          if (thisListing == null) {
            throw new FileNotFoundException("File " + src + " does not exist.");
          }
          i = 0;
        }
        return i < thisListing.getPartialListing().length;
      }

      @Override
      public LocatedFileStatus next() throws IOException {
        if (!hasNext()) {
          throw new java.util.NoSuchElementException("No more entry in " + src);
        }
        return HdfsFileStatus.toLocatedFileStatus(thisListing.getPartialListing()[i],
            thisListing.getBlockLocations()[i++], src);
      }
    };

  }
295798
def run(self, directories=None):
        
        if directories is None:
            directories = [os.getcwd()]

        total_successes, total_errors, total_skipped = 0, 0, 0
        for directory in directories:
            successes, errors, skips = self.execute(self.find_specs(directory))
            total_successes += successes
            total_errors += errors
            total_skipped += skips


        self.formatter.finalize()
        return total_successes, total_errors, total_skipped
1192839
function traverse (root, dirs, collect, next) {
  const rootPath = path.join (root, ...dirs);
  fs.lstat (rootPath, (err0) => {
    if (err0) {
      next (err0);
    } else {
      fs.readdir (rootPath, (_, files) => {
        let pending = files.length;
        for (let file of files) {
          const filePath = path.join (root, ...dirs, file);
          fs.lstat (filePath, (err1, stats) => {
            if (err1) {
              next (err1);
            } else {
              if (stats.isDirectory ()) {
                traverse (root, [...dirs, file], collect, err2 => {
                  if (err2) {
                    next (err2);
                  } else {
                    if (--pending === 0) {
                      next ();
                    }
                  }
                });
              } else {
                if (stats.isFile ()) {
                  collect (dirs, file, files);
                  if (--pending === 0) {
                    next ();
                  }
                }
              }
            }
          });
        }
      });
    }
  });
}
660076
private boolean directoryContains(File directory, IGroupMember member) {
        boolean found = false;
        if (member.isGroup()) {
            File memberFile = getFile((IEntityGroup) member);
            File[] files = directory.listFiles();
            for (int i = 0; i < files.length & !found; i++) {
                found = files[i].equals(memberFile);
            }
        }
        return found;
    }
774691
private static int countWildcardPhraseFrequencies(File dir, String pattern)
        throws Exception {
        
        File[] files = dir.listFiles();

        int total = 0;

        for (int i = 0; i < files.length; i++) {
            File f = files[i];
            if (f.isDirectory()) {
                total += countWildcardPhraseFrequencies(f, pattern);
            } else if (f.getName().endsWith(".txt")) {
                Scanner sc = new Scanner(f);
                while (sc.hasNext()) {
                    String line = sc.nextLine();
                    if (line.matches(pattern)) {
                        total++;
                    }
                }
            }
        }
        return total;
    }
216436
function listFiles(directory) {
    return fs.lstatAsync(directory)
        .then(lstat => lstat.isDirectory() ? fs.readdirAsync(directory) : [])
        .catchReturn([]);
}