Returns a fingerprint of the given file directories and all their sub contents.

    This assumes that the file directories are of reasonable size
    to cause memory or performance issues.
483288
def _fingerprint_dirs(self, dirpaths, topdown=True, onerror=None, followlinks=False):
    
    # Note that we don't sort the dirpaths, as their order may have meaning.
    filepaths = []
    for dirpath in dirpaths:
      dirs = os.walk(dirpath, topdown=topdown, onerror=onerror,
                     followlinks=followlinks)
      sorted_dirs = sorted(dirs, key=lambda d: d[0])
      filepaths.extend([os.path.join(dirpath, filename)
                   for dirpath, dirnames, filenames in sorted_dirs
                   for filename in sorted(filenames)])
    return self._fingerprint_files(filepaths)
1249798
def lookupSpatialReferenceID(cls, directory, filename):
        

        path = os.path.join(directory, filename)

        with open(path, 'r') as f:
            srid = lookupSpatialReferenceID(f.read())

        return srid
275316
def calc_dirs_with_files(sources)
      filemap = {}
      sources.keys.sort.reverse.each do |o|
        d = File.dirname(o)
        if filemap.include?(d)
          filemap[d] << o
        else
          filemap[d] = [o]
        end
      end
      return filemap
    end
95522
def get_hash_of_dirs(directory):
    
    import hashlib
    sha = hashlib.sha512()
    if not os.path.exists(directory):
        return -1

    for root, _, files in os.walk(directory):
        for name in files:
            filepath = local.path(root) / name
            if filepath.exists():
                with open(filepath, 'rb') as next_file:
                    for line in next_file:
                        sha.update(line)
    return sha.hexdigest()
585871
def files
    @files ||= lambda {
      sorted_relevant_files = []

      file_globs.each do |glob|
        current_glob_files  = Pathname.glob(glob)
        relevant_glob_files = relevant_files & current_glob_files

        relevant_glob_files.map! do |file|
          File.new(path:            file,
                   namespaces:      namespaces,
                   decryption_keys: decryption_keys,
                   encryption_keys: encryption_keys)
        end

        sorted_relevant_files += relevant_glob_files
      end

      sorted_relevant_files.uniq
    }.call
  end
751261
def get_content_hash(self):
        
        if not self.rexists():
            return SCons.Util.MD5signature('')
        fname = self.rfile().get_abspath()
        try:
            cs = SCons.Util.MD5filesignature(fname,
                chunksize=SCons.Node.FS.File.md5_chunksize*1024)
        except EnvironmentError as e:
            if not e.filename:
                e.filename = fname
            raise
        return cs
832734
def get_names(file_dir, files):
    
    # for each user, check if they have files available
    # return user name list
    total_list = []
    name_list = []
    get_sub = False
    for path, subdir, dir_files in os.walk(file_dir):
        if not get_sub:
            total_list = subdir[:]
            get_sub = True
        else:
            break
    for user in total_list:
        has_file = True
        for f in files:
            file_path = file_dir + user + "/" + f + ".txt"
            if not os.path.exists(file_path):
                has_file = False
                break
        if has_file:
            name_list.append(user)
    if len(name_list) == 0:
        print("********Error: Cannot find any user who completes the files*************", file=ERROR_LOG)
    return name_list
464209
def mattching_dir_files(dir1, dir2)
      dir1_files = dir_files(dir1).map {|f| f.sub("#{dir1}/",'') }
      dir2_files = dir_files(dir2).map {|f| f.sub("#{dir2}/",'') }

      matches = dir1_files & dir2_files

      unique_files(dir1, dir2_files, matches)
      unique_files(dir2, dir1_files, matches)

      matches
    end
305407
def nuke!
      Dir["#{base_dir}/**/*"]
        .select { |path| path =~ FINGERPRINT_REGEX }
        .each { |file| FileUtils.rm(file) }

      FileUtils.rm(manifest_path)
    end
103595
def glob(self):
        """"""

        # Cached symlinks
        self.symlinks = {}

        if self.is_bytes:
            curdir = os.fsencode(os.curdir)
        else:
            curdir = os.curdir

        for pattern in self.pattern:
            # If the pattern ends with `/` we return the files ending with `/`.
            dir_only = pattern[-1].dir_only if pattern else False

            if pattern:
                if not pattern[0].is_magic:
                    # Path starts with normal plain text
                    # Lets verify the case of the starting directory (if possible)
                    this = pattern[0]

                    curdir = this[0]

                    if not os.path.lexists(curdir):
                        return

                    # Make sure case matches, but running case insensitive
                    # on a case sensitive file system may return more than
                    # one starting location.
                    results = [curdir] if this.is_drive else self._get_starting_paths(curdir)
                    if not results:
                        if not dir_only:
                            # There is no directory with this name,
                            # but we have a file and no directory restriction
                            yield curdir
                        return

                    if this.dir_only:
                        # Glob these directories if they exists
                        for start in results:
                            if os.path.isdir(start):
                                rest = pattern[1:]
                                if rest:
                                    this = rest.pop(0)
                                    for match in self._glob(curdir, this, rest):
                                        if not self._is_excluded(match, dir_only):
                                            yield os.path.join(match, self.empty) if dir_only else match
                                elif not self._is_excluded(curdir, dir_only):
                                    yield os.path.join(curdir, self.empty) if dir_only else curdir
                    else:
                        # Return the file(s) and finish.
                        for start in results:
                            if os.path.lexists(start) and not self._is_excluded(start, dir_only):
                                yield os.path.join(start, self.empty) if dir_only else start
                else:
                    # Path starts with a magic pattern, let's get globbing
                    rest = pattern[:]
                    this = rest.pop(0)
                    for match in self._glob(curdir if not curdir == self.current else self.empty, this, rest):
                        if not self._is_excluded(match, dir_only):
                            yield os.path.join(match, self.empty) if dir_only else match
1084116
def file_tree(start_path):
    
    nested_dirs = {}
    root_dir = start_path.rstrip(os.sep)
    start = root_dir.rfind(os.sep) + 1
    for path, dirs, files in os.walk(root_dir):
        folders = path[start:].split(os.sep)
        subdir = dict.fromkeys(files)
        parent = reduce(dict.get, folders[:-1], nested_dirs)
        parent[folders[-1]] = subdir
    return nested_dirs
284789
private static @Nonnull File getFingerprintFile(@Nonnull byte[] md5sum) {
        assert md5sum.length==16;
        return new File( Jenkins.getInstance().getRootDir(),
            "fingerprints/"+ Util.toHexString(md5sum,0,1)+'/'+Util.toHexString(md5sum,1,1)+'/'+Util.toHexString(md5sum,2,md5sum.length-2)+".xml");
    }
544426
def globs_in_single_dir(cls, spec_dir_path, address_mapper):
    """"""
    return [os.path.join(spec_dir_path, pat) for pat in address_mapper.build_patterns]
503273
def identify_datafiles(root,
                       extensions_to_ignore=None,
                       directories_to_ignore=None,
                       files_to_ignore=None):
    

    for dirpath, dirnames, filenames in walk(root):

        for ignore in directories_to_ignore:
            if ignore in dirnames:
                dirnames.remove(ignore)  # don't visit ignored directories


        for filename in filenames:


            # Ignore extensions that need no IP check
            ignore = False
            for ext in extensions_to_ignore:
                if filename.endswith(ext):
                    ignore = True

            if filename in files_to_ignore:
                ignore = True

            if ignore is False:
                yield dirpath, filename
581090
def get_fingerprint_from_public_key(pubkey):
    

    # TODO: why is there a tmpdir?
    with tempdir() as workdir:
        key = os.path.join(workdir, 'key.pub')
        with open(key, 'w') as fd:
            fd.write(pubkey)

        cmd = [
            'ssh-keygen',
            '-l',
            '-f', key,
        ]

        p = Subprocess(cmd)
        output = p.stdout.strip()
        bits, fingerprint, _ = output.split(' ', 2)
        return fingerprint
74699
def fingerprints(data):
    

    Hashes = namedtuple('Hashes', "md5 sha1 sha256 sha512")

    if six.PY2:
        if not isinstance(data, str):
            data = data.encode("utf-8")
    elif six.PY3:
        if not isinstance(data, bytes):
            data = data.encode("utf-8")

    # md5
    md5 = hashlib.md5()
    md5.update(data)
    md5 = md5.hexdigest()

    # sha1
    sha1 = hashlib.sha1()
    sha1.update(data)
    sha1 = sha1.hexdigest()

    # sha256
    sha256 = hashlib.sha256()
    sha256.update(data)
    sha256 = sha256.hexdigest()

    # sha512
    sha512 = hashlib.sha512()
    sha512.update(data)
    sha512 = sha512.hexdigest()

    return Hashes(md5, sha1, sha256, sha512)
1131650
def fingerprint(self, option_type, option_val):
    
    if option_val is None:
      return None

    # Wrapping all other values in a list here allows us to easily handle single-valued and
    # list-valued options uniformly. For non-list-valued options, this will be a singleton list
    # (with the exception of dict, which is not modified). This dict exception works because we do
    # not currently have any "list of dict" type, so there is no ambiguity.
    if not isinstance(option_val, (list, tuple, dict)):
      option_val = [option_val]

    if option_type == target_option:
      return self._fingerprint_target_specs(option_val)
    elif option_type == dir_option:
      return self._fingerprint_dirs(option_val)
    elif option_type == file_option:
      return self._fingerprint_files(option_val)
    elif option_type == dict_with_files_option:
      return self._fingerprint_dict_with_files(option_val)
    else:
      return self._fingerprint_primitives(option_val)