Compare potentially partial criteria against built filesystems entry dictionary
964199
def match(self, fsys_view):
        '''
        
        '''
        evalue_dict = fsys_view[1]
        for key, value in six.viewitems(self.criteria):
            if key in evalue_dict:
                if evalue_dict[key] != value:
                    return False
            else:
                return False
        return True
1245565
def _validate_completeness(self):
        
        errors = list()

        # First we'll make sure there's no mismatch between the filesystem
        # and the list of files in the manifest(s)
        only_in_manifests, only_on_fs, only_in_fetch = self.compare_manifests_with_fs_and_fetch()
        for path in only_in_manifests:
            e = FileMissing(path)
            LOGGER.warning(force_unicode(e))
            errors.append(e)
        for path in only_on_fs:
            e = UnexpectedFile(path)
            LOGGER.warning(force_unicode(e))
            errors.append(e)
        for path in only_in_fetch:
            e = UnexpectedRemoteFile(path)
            # this is non-fatal according to spec but the warning is still reasonable
            LOGGER.warning(force_unicode(e))

        if errors:
            raise BagValidationError(_("Bag validation failed"), errors)
45851
def IsFileStatEntry(self, original_result):
    """"""
    return (original_result.pathspec.pathtype in [
        rdf_paths.PathSpec.PathType.OS, rdf_paths.PathSpec.PathType.TSK
    ])
1176558
def validate(self, signature):
        ''''''
        if not signature:
            return 'Empty signature'
        sig_files = self.input_files._targets + self.output_files._targets + \
            self.dependent_files._targets
        for x in sig_files:
            if not x.target_exists('any'):
                return f'Missing target {x}'
        #

        files_checked = {x.target_name(): False for x in sig_files}
        res = {'input': [], 'output': [], 'depends': [], 'vars': {}}
        cur_type = 'input'
        # old signature
        if 'init_context' in signature:
            for key, value in signature['init_context'].items():
                if key not in env.sos_dict:
                    return f'Variable {key} not in running environment'
                try:
                    if env.sos_dict[key] != value:
                        return f'Context variable {key} value mismatch: {short_repr(value)} saved, {short_repr(env.sos_dict[key])} current'
                except Exception as e:
                    env.logger.debug(
                        f"Variable {key} of type {type(value).__name__} cannot be compared: {e}"
                    )
        elif 'init_context_sig' in signature:
            for key, value in signature['init_context_sig'].items():
                if key not in env.sos_dict:
                    return f'Variable {key} not in running environment'
                try:
                    if objectMD5(env.sos_dict[key]) != value:
                        return f'ID of context variable {key} ({objectMD5(env.sos_dict[key])}) mismatch: {short_repr(env.sos_dict[key])} does not match id {value}'
                except Exception as e:
                    env.logger.debug(f"Variable {key} cannot be compared: {e}")

        res['vars'].update(signature['end_context'])
        #
        for cur_type in ['input', 'output', 'depends']:
            for f, m in signature[cur_type].items():
                try:
                    if '(' in f and ')' in f:
                        # this part is hard, because this can be a customized target.
                        target_type = f.split('(')[0]
                        target_class = None
                        if target_type in globals():
                            target_class = eval(target_type)
                        else:
                            # check registry
                            for entrypoint in pkg_resources.iter_entry_points(
                                    group='sos_targets'):
                                if entrypoint.name.strip() == target_type:
                                    target_class = entrypoint.load()
                                    break
                        if target_class is None:
                            raise ValueError(
                                f'Failed to identify target class {target_type}'
                            )
                        # parameter of class?
                        freal = eval(f, {target_type: target_class})
                    else:
                        freal = file_target(f)
                    if not freal.validate(m):
                        return f'Target {f} does not exist or does not match saved signature {m}'
                    res[cur_type].append(freal.target_name(
                    ) if isinstance(freal, file_target) else freal)
                    files_checked[freal.target_name()] = True
                except Exception as e:
                    env.logger.debug(f'Wrong md5 in signature: {e}')
        #
        if not all(files_checked.values()):
            return f'No MD5 signature for {", ".join(x for x,y in files_checked.items() if not y)}'
        if 'input_obj' in signature:
            # for new style signature, the entire objects are kept
            res['input'] = signature['input_obj']
            res['depends'] = signature['depends_obj']
            res['output'] = signature['output_obj']
        return res
1340948
def match_path(entry, opts):
    """"""
    if entry.name in ALWAYS_OMIT:
        return False
    # TODO: currently we use fnmatch syntax and match against names.
    # We also might allow glob syntax and match against the whole relative path instead
    # path = entry.get_rel_path()
    path = entry.name
    ok = True
    match = opts.get("match")
    exclude = opts.get("exclude")
    if entry.is_file() and match:
        assert type(match) is list
        ok = False
        for pat in match:
            if fnmatch.fnmatch(path, pat):
                ok = True
                break
    if ok and exclude:
        assert type(exclude) is list
        for pat in exclude:
            if fnmatch.fnmatch(path, pat):
                ok = False
                break
    # write("match", ok, entry)
    return ok
122729
def get_file_systems(self):
        
        result = {}
        if os.access('/proc/mounts', os.R_OK):
            file = open('/proc/mounts')
            for line in file:
                try:
                    mount = line.split()
                    device = mount[0]
                    mount_point = mount[1]
                    fs_type = mount[2]
                except (IndexError, ValueError):
                    continue

                # Skip the filesystem if it is not in the list of valid
                # filesystems
                if fs_type not in self.filesystems:
                    self.log.debug("Ignoring %s since it is of type %s " +
                                   " which is not in the list of filesystems.",
                                   mount_point, fs_type)
                    continue

                # Process the filters
                if self.exclude_reg.search(mount_point):
                    self.log.debug("Ignoring %s since it is in the " +
                                   "exclude_filter list.", mount_point)
                    continue

                if ((('/' in device or device == 'tmpfs') and
                     mount_point.startswith('/'))):
                    try:
                        stat = os.stat(mount_point)
                    except OSError:
                        self.log.debug("Path %s is not mounted - skipping.",
                                       mount_point)
                        continue

                    if stat.st_dev in result:
                        continue

                    result[stat.st_dev] = {
                        'device': os.path.realpath(device),
                        'mount_point': mount_point,
                        'fs_type': fs_type
                    }

            file.close()

        else:
            if not psutil:
                self.log.error('Unable to import psutil')
                return None

            partitions = psutil.disk_partitions(False)
            for partition in partitions:
                result[len(result)] = {
                    'device': os.path.realpath(partition.device),
                    'mount_point': partition.mountpoint,
                    'fs_type': partition.fstype
                }
            pass

        return result
60302
def _mkstemp_inner(dir, pre, suf, flags):
    """"""

    names = _get_candidate_names()

    for seq in range(TMP_MAX):
        name = next(names)
        file = _os.path.join(dir, pre + name + suf)
        try:
            fd = _os.open(file, flags, 0o600)
            return (fd, _os.path.abspath(file))
        except FileExistsError:
            continue    # try again
        except PermissionError:
            # This exception is thrown when a directory with the chosen name
            # already exists on windows.
            if _os.name == 'nt':
                continue
            else:
                raise

    raise FileExistsError(_errno.EEXIST,
                          "No usable temporary file name found")
1192391
def _should_skip_entry(self, entry):
        
        # Don't replicate entries resulting from chunk moves
        if entry.get("fromMigrate"):
            return True, False

        # Ignore no-ops
        if entry["op"] == "n":
            return True, False
        ns = entry["ns"]

        if "." not in ns:
            return True, False
        coll = ns.split(".", 1)[1]

        # Ignore system collections
        if coll.startswith("system."):
            return True, False

        # Ignore GridFS chunks
        if coll.endswith(".chunks"):
            return True, False

        is_gridfs_file = False
        if coll.endswith(".files"):
            ns = ns[: -len(".files")]
            if self.namespace_config.gridfs_namespace(ns):
                is_gridfs_file = True
            else:
                return True, False

        # Commands should not be ignored, filtered, or renamed. Renaming is
        # handled by the DocManagers via the CommandHelper class.
        if coll == "$cmd":
            return False, False

        # Rename or filter out namespaces that are ignored keeping
        # included gridfs namespaces.
        namespace = self.namespace_config.lookup(ns)
        if namespace is None:
            LOG.debug(
                "OplogThread: Skipping oplog entry: "
                "'%s' is not in the namespace configuration." % (ns,)
            )
            return True, False

        # Update the namespace.
        entry["ns"] = namespace.dest_name

        # Take fields out of the oplog entry that shouldn't be replicated.
        # This may nullify the document if there's nothing to do.
        if not self.filter_oplog_entry(
            entry,
            include_fields=namespace.include_fields,
            exclude_fields=namespace.exclude_fields,
        ):
            return True, False
        return False, is_gridfs_file
363966
def FileEntryExistsByPathSpec(self, path_spec):
    
    # Opening a file by MFT entry is faster than opening a file by location.
    # However we need the index of the corresponding $FILE_NAME MFT attribute.
    fsntfs_file_entry = None
    location = getattr(path_spec, 'location', None)
    mft_attribute = getattr(path_spec, 'mft_attribute', None)
    mft_entry = getattr(path_spec, 'mft_entry', None)

    try:
      if mft_attribute is not None and mft_entry is not None:
        fsntfs_file_entry = self._fsntfs_volume.get_file_entry(mft_entry)
      elif location is not None:
        fsntfs_file_entry = self._fsntfs_volume.get_file_entry_by_path(location)

    except IOError as exception:
      raise errors.BackEndError(exception)

    return fsntfs_file_entry is not None
1338993
def _get_filesystem_types(args, sample_file):
    
    out = set([])
    ext = "" if args.no_container else "_container"
    with open(sample_file) as in_handle:
        for f in _get_file_paths(json.load(in_handle)):
            if f.startswith("gs:"):
                out.add("gcp%s" % ext)
            elif f.startswith("s3:"):
                out.add("s3%s" % ext)
            elif f.startswith(("https:", "http:")):
                out.add("http%s" % ext)
            else:
                out.add("local%s" % ext)
    return out
150227
function compareVersions(installed, required) {

	var a = installed.split('.');
	var b = required.split('.');

	for (var i = 0; i < a.length; ++i) {
		a[i] = Number(a[i]);
	}
	for (var i = 0; i < b.length; ++i) {
		b[i] = Number(b[i]);
	}
	if (a.length == 2) {
		a[2] = 0;
	}

	if (a[0] > b[0]) return true;
	if (a[0] < b[0]) return false;

	if (a[1] > b[1]) return true;
	if (a[1] < b[1]) return false;

	if (a[2] > b[2]) return true;
	if (a[2] < b[2]) return false;

	return true;
}
662232
def _list_existing(filesystem, glob, paths):
    
    globs = _constrain_glob(glob, paths)
    time_start = time.time()
    listing = []
    for g in sorted(globs):
        logger.debug('Listing %s', g)
        if filesystem.exists(g):
            listing.extend(filesystem.listdir(g))
    logger.debug('%d %s listings took %f s to return %d items',
                 len(globs), filesystem.__class__.__name__, time.time() - time_start, len(listing))
    return set(listing)
746733
def _check_avail(cmd):
    '''
    
    '''
    if isinstance(cmd, list):
        cmd = ' '.join([six.text_type(x) if not isinstance(x, six.string_types) else x
                        for x in cmd])
    bret = True
    wret = False
    if __salt__['config.get']('cmd_blacklist_glob'):
        blist = __salt__['config.get']('cmd_blacklist_glob', [])
        for comp in blist:
            if fnmatch.fnmatch(cmd, comp):
                # BAD! you are blacklisted
                bret = False
    if __salt__['config.get']('cmd_whitelist_glob', []):
        blist = __salt__['config.get']('cmd_whitelist_glob', [])
        for comp in blist:
            if fnmatch.fnmatch(cmd, comp):
                # GOOD! You are whitelisted
                wret = True
                break
    else:
        # If no whitelist set then alls good!
        wret = True
    return bret and wret
272837
def ls(system, user, local, include_missing):
    """"""

    # default action is to list *all* auto-detected files
    if not (system or user or local):
        system = user = local = True

    for path in get_configfile_paths(system=system, user=user, local=local,
                                     only_existing=not include_missing):
        click.echo(path)
59722
def show_matching_entry(program, arguments, use_clipboard=True, quiet=False, filters=()):
    """"""
    entry = program.select_entry(*arguments)
    if not quiet:
        formatted_entry = entry.format_text(include_password=not use_clipboard, filters=filters)
        if formatted_entry and not formatted_entry.isspace():
            output(formatted_entry)
    if use_clipboard:
        entry.copy_password()
287011
def _where(filename, dirs=[], env="PATH"):
    """"""
    if not isinstance(dirs, list):
        dirs = [dirs]
    if glob(filename):
        return filename
    paths = [os.curdir] + os.environ[env].split(os.path.pathsep) + dirs
    for path in paths:
        for match in glob(os.path.join(path, filename)):
            if match:
                return os.path.normpath(match)
    raise IOError("File not found: %s" % filename)
1055082
def _scan_payload(self):
        '''
        
        '''
        # Get ignored points
        allowed = list()
        for allowed_dir in self.db.get(AllowedDir):
            if os.path.exists(allowed_dir.path):
                allowed.append(allowed_dir.path)

        ignored = list()
        if not allowed:
            for ignored_dir in self.db.get(IgnoredDir):
                if os.path.exists(ignored_dir.path):
                    ignored.append(ignored_dir.path)

        all_files = list()
        all_dirs = list()
        all_links = list()
        for entry_path in [pth for pth in (allowed or os.listdir("/")) if pth]:
            if entry_path[0] != "/":
                entry_path = "/{0}".format(entry_path)
            if entry_path in ignored or os.path.islink(entry_path):
                continue
            e_files, e_dirs, e_links = self._get_all_files(entry_path, *ignored)
            all_files.extend(e_files)
            all_dirs.extend(e_dirs)
            all_links.extend(e_links)

        return self._get_unmanaged_files(self._get_managed_files(), (all_files, all_dirs, all_links,))