Return which file(s) to use according to dbname, var, etc
437995
def dbsource(dbname, var, resolution=None, tscale=None):
    
    db_cfg = {}
    cfg_dir = 'datasource'
    cfg_files = pkg_resources.resource_listdir('oceansdb', cfg_dir)
    cfg_files = [f for f in cfg_files if f[-5:] == '.json']
    for src_cfg in cfg_files:
        text = pkg_resources.resource_string(
                'oceansdb', os.path.join(cfg_dir, src_cfg))
        text = text.decode('UTF-8', 'replace')
        cfg = json.loads(text)
        for c in cfg:
            assert c not in db_cfg, "Trying to overwrite %s"
            db_cfg[c] = cfg[c]

    dbpath = oceansdb_dir()
    datafiles = []
    cfg = db_cfg[dbname]

    if (resolution is None):
        resolution = cfg['vars'][var]['default_resolution']

    if (tscale is None):
        tscale = cfg['vars'][var][resolution]["default_tscale"]

    for c in cfg['vars'][var][resolution][tscale]:
        download_file(outputdir=dbpath, **c)

        if 'filename' in c:
            filename = os.path.join(dbpath, c['filename'])
        else:
            filename = os.path.join(dbpath,
                    os.path.basename(urlparse(c['url']).path))

        if 'varnames' in cfg['vars'][var][resolution]:
            datafiles.append(Dataset_flex(filename,
                aliases=cfg['vars'][var][resolution]['varnames']))
        else:
            datafiles.append(Dataset_flex(filename))

    return datafiles
1069941
def db_file(self, value):
        
        assert not os.path.isfile(value), "%s already exists" % value
        self._db_file = value
912929
def get_installed_use(cp, use="USE"):
    '''
    
    '''
    portage = _get_portage()
    cpv = _get_cpv(cp)
    return portage.db[portage.root]["vartree"].dbapi.aux_get(cpv, [use])[0].split()
714341
def get_filesnames(self):
        """"""
        self._update()
        projects = set(list(self._tasks.keys()) + self._task_projects + ['all_projects', 'unaffiliated'])
        return [join(self._data_location, p.split()[0]) for p in projects]
1369710
def getScriptNames(self):
		""""""

		if self._db == '':
			raise FMError, 'No database was selected'

		request = []
		request.append(uu({'-db': self._db }))
		request.append(uu({'-scriptnames': '' }))

		result = self._doRequest(request)
		result = FMResultset.FMResultset(result)

		scriptNames = []
		for scriptName in result.resultset:
			scriptNames.append(string.lower(scriptName['SCRIPT_NAME']))

		return scriptNames
568038
def bsfile(self, path):
        """"""
        from sqlalchemy.orm.exc import NoResultFound
        from ambry.orm.exc import NotFoundError

        try:

            f =  object_session(self)\
                .query(File)\
                .filter(File.d_vid == self.vid)\
                .filter(File.major_type == File.MAJOR_TYPE.BUILDSOURCE)\
                .filter(File.path == path)\
                .one()

            return f

        except NoResultFound:
            raise NotFoundError("Failed to find file for path '{}' ".format(path))
529687
def _get_db_names(self, dbs, strict=True):
        
        dbs = utils.coerce_to_list(dbs)
        db_names = [utils.get_name(db) for db in dbs]
        if strict:
            good_dbs = self.instance.list_databases()
            good_names = [utils.get_name(good_db) for good_db in good_dbs]
            bad_names = [db_name for db_name in db_names
                    if db_name not in good_names]
            if bad_names:
                bad = ", ".join(bad_names)
                raise exc.NoSuchDatabase("The following database(s) were not "
                        "found: %s" % bad)
        return db_names
23615
def get_local_filepath(filename):
    
    procs_dir = os.path.normpath(os.path.join(
        __file__,
        '../../',
        'external/postgresql/raw_sql/procs'
    ))
    return os.path.join(procs_dir, filename)
628341
def build_db_cmd(self, fname):
        """"""
        return self.funcs.db_func(fname, self.outdir, self.exes.format_exe)[0]
903348
function scripts(action) {
  var fullpath = path.resolve(__dirname, '..', 'sql', action);
  return fs.readdirSync(fullpath)
    .filter(function(file) {
      return path.extname(file) === '.sql';
    })
    .map(function(file) {
      return [fs.readFileSync(path.join(fullpath, file)).toString(), file];
    });
}
920506
def _get_result_paths(self, data):
        
        # access data through self.Parameters so we know it's been cast
        # to a FilePath
        wd = self.WorkingDir
        db_name = self.Parameters['-n'].Value
        log_name = self.Parameters['-l'].Value
        result = {}
        result['log'] = ResultPath(Path=wd + log_name, IsWritten=True)
        if self.Parameters['-p'].Value == 'F':
            extensions = ['nhr', 'nin', 'nsq', 'nsd', 'nsi']
        else:
            extensions = ['phr', 'pin', 'psq', 'psd', 'psi']
        for extension in extensions:
            for file_path in glob(wd + (db_name + '*' + extension)):
                # this will match e.g. nr.01.psd and nr.psd
                key = file_path.split(db_name + '.')[1]
                result_path = ResultPath(Path=file_path, IsWritten=True)
                result[key] = result_path
        return result
779477
def filenames(self):
        
        fns = set()
        for fn2var in self._type2filename2variable.values():
            fns.update(fn2var.keys())
        return sorted(fns)
1314830
def _get_collections(self, db_names=None):
        
        if db_names:
            db_list = db_names
        else:
            db_list = self.client.list_database_names()

        for db_name in db_list:
            db = self.client.get_database(db_name)
            for col_name in db.list_collection_names():
                size = db[col_name].g
                yield f"{db_name}.{col_name}"
1226279
def db(self):
        """"""
        if self._db is None:
            if self.tcex.default_args.tc_playbook_db_type == 'Redis':
                from .tcex_redis import TcExRedis

                self._db = TcExRedis(
                    self.tcex.default_args.tc_playbook_db_path,
                    self.tcex.default_args.tc_playbook_db_port,
                    self.tcex.default_args.tc_playbook_db_context,
                )
            elif self.tcex.default_args.tc_playbook_db_type == 'TCKeyValueAPI':
                from .tcex_key_value import TcExKeyValue

                self._db = TcExKeyValue(self.tcex)
            else:
                err = u'Invalid DB Type: ({})'.format(self.tcex.default_args.tc_playbook_db_type)
                raise RuntimeError(err)
        return self._db
909377
function getFileDetails(db, fileSelectionCriteria, fileOptions, cb) {

  defaultLogger.debug("In getFileDetails ");
  var selectionQuery = undefined;
  if (fileSelectionCriteria.groupId) {

    selectionQuery= {"metadata.groupId": fileSelectionCriteria.groupId};


  } else if (fileSelectionCriteria.hash) {
    selectionQuery= {"md5": fileSelectionCriteria.hash};
  }

  var gridStore = new GridStore(db, null, "r", {"root": constants.ROOT_COLLECTION});
  var fileOfInterest = undefined;

  gridStore.collection(function(err, collection) {
    collection.find(selectionQuery, { "sort": {"metadata.version":-1}}, function(err, files) {
      if (err) {
        defaultLogger.error(err); return cb(err);
      }

      files.toArray(function(err, filesArray) {
        if (err) {
          defaultLogger.error(err); return cb(err);
        }
        if (!(filesArray && Array.isArray(filesArray) && filesArray.length > 0)) {
          return cb(new Error("No files exist for groupId " + fileSelectionCriteria.groupId));
        }

        //If the file details are needed for all files in the groupId, then an array containing all of the file details is returned.
        if (fileOptions.allMatchingFiles == true) { // eslint-disable-line eqeqeq
          fileOfInterest = filesArray;
        } else {// Just want the details of a single file.
          //If there is no version, get the latest version

          //If no version is found or we have a hash query, just want the first entry of the array.
          if (!fileSelectionCriteria.version || fileSelectionCriteria.hash) {
            fileOfInterest = filesArray[0];
          } else {
            fileOfInterest = filesArray.filter(function(file) {
              return file.metadata.version === fileSelectionCriteria.version;
            });

            if (fileOfInterest.length === 1) {
              fileOfInterest = fileOfInterest[0];
            } else {
              return cb(new Error("Unexpected number of files returned for groupId " + fileSelectionCriteria.groupId, fileOfInterest.length));
            }
          }
        }

        if (!Array.isArray(fileOfInterest)) {
          //Actually want the thumbnail for the file, return the details for that file instead.
          if (fileOptions.thumbnail) {
            if (fileOfInterest.metadata.thumbnail) {
              getThumbnailFileDetails(db, fileOfInterest.metadata.thumbnail, function(err, thumbFileDetails) {
                if (err) {
                  defaultLogger.error(err); return cb(err);
                }

                fileOfInterest = thumbFileDetails;

                gridStore.close(function(err) {
                  return cb(err, fileOfInterest);
                });
              });
            } else {
              return cb(new Error("Thumbnail for file " + JSON.stringify(fileSelectionCriteria) + " does not exist."));
            }
          } else {
            gridStore.close(function(err) {
              return cb(err, fileOfInterest);
            });
          }
        } else {
          gridStore.close(function(err) {
            return cb(err, fileOfInterest);
          });
        }
      });
    });
  });
}
266320
def default_database(ctx: click.Context, _param: Parameter, value: Optional[str]):
    """"""
    if value:
        return value

    if ctx.params["repository"]:
        return os.path.join(ctx.params["repository"], DB.DEFAULT_DB_FILE)

    raise click.BadParameter("Could not guess a database location")
949810
def files_type(request, fstype=None):
    """"""
    try:
        host = Host.objects.get(code=fstype)
    except Host.DoesNotExist:
        messages.error(request, "Could not find host in database.")
        return redirect("files")

    if host.available_to_all:
        pass
    elif not host.visible_to(request.user):
        messages.error(request, "You don't have permission to access this host.")
        return redirect("files")

    authinfo = get_authinfo(request)

    if not authinfo:
        return redirect("{}?next={}".format(reverse("files_auth"), request.get_full_path()))

    try:
        sftp = create_session(host.address, authinfo["username"], authinfo["password"])
    except exceptions as e:
        messages.error(request, e)
        error_msg = str(e).lower()
        if "authentication" in error_msg:
            return redirect("files_auth")
        return redirect("files")
    finally:
        # Delete the stored credentials, so they aren't mistakenly used or
        # accessed later.
        del authinfo

    if host.directory:
        host_dir = host.directory
        if "{}" in host_dir:
            host_dir = host_dir.format(request.user.username)
        if "{win}" in host_dir:
            host_dir = windows_dir_format(host_dir, request.user)
            try:
                sftp.chdir(host_dir)
            except exceptions as e:
                if "NoSuchFile" in "{}".format(e):
                    host_dir = "/"
                    try:
                        sftp.chdir(host_dir)
                    except exceptions as e2:
                        messages.error(request, e)
                        messages.error(request, "Root directory: {}".format(e2))
                        return redirect("files")
                    else:
                        messages.error(request, "Unable to access home folder -- showing root directory instead.")
                else:
                    messages.error(request, e)
                    return redirect("files")
        else:
            try:
                sftp.chdir(host_dir)
            except exceptions as e:
                messages.error(request, e)
                return redirect("files")

    default_dir = normpath(sftp.pwd)

    def can_access_path(fsdir):
        return normpath(fsdir).startswith(default_dir)

    if "file" in request.GET:
        # Download file
        filepath = request.GET.get("file")
        filepath = normpath(filepath)
        filebase = os.path.basename(filepath)
        filebase_escaped = re.sub(r"[^\w\s\-\.]", "", filebase).strip()
        if can_access_path(filepath):
            try:
                fstat = sftp.stat(filepath)
            except exceptions as e:
                messages.error(request, "Unable to access {}: {}".format(filebase, e))
                return redirect("/files/{}?dir={}".format(fstype, os.path.dirname(filepath)))

            if fstat.st_size > settings.FILES_MAX_DOWNLOAD_SIZE:
                messages.error(request, "Too large to download (>200MB)")
                return redirect("/files/{}?dir={}".format(fstype, os.path.dirname(filepath)))

            tmpfile = tempfile.TemporaryFile(prefix="ion_filecenter_{}_{}".format(request.user.username, filebase_escaped))
            logger.debug(tmpfile)

            try:
                sftp.getfo(filepath, tmpfile)
            except exceptions as e:
                messages.error(request, e)
                return redirect("/files/{}?dir={}".format(fstype, os.path.dirname(filepath)))

            content_len = tmpfile.tell()
            tmpfile.seek(0)
            chunk_size = 8192
            response = StreamingHttpResponse(FileWrapper(tmpfile, chunk_size), content_type="application/octet-stream")
            response['Content-Length'] = content_len
            response["Content-Disposition"] = "attachment; filename={}".format(filebase_escaped)
            return response

    fsdir = request.GET.get("dir")
    if fsdir:
        fsdir = normpath(fsdir)
        if can_access_path(fsdir):
            try:
                sftp.chdir(fsdir)
            except exceptions as e:
                messages.error(request, e)
                return redirect("files")
        else:
            messages.error(request, "Access to the path you provided is restricted.")
            return redirect("/files/{}/?dir={}".format(fstype, default_dir))

    if "zip" in request.GET:
        dirbase_escaped = os.path.basename(fsdir)
        dirbase_escaped = slugify(dirbase_escaped)
        tmpfile = tempfile.TemporaryFile(prefix="ion_filecenter_{}_{}".format(request.user.username, dirbase_escaped))

        with tempfile.TemporaryDirectory(prefix="ion_filecenter_{}_{}_zip".format(request.user.username, dirbase_escaped)) as tmpdir:
            remote_directories = [fsdir]
            totalsize = 0
            while remote_directories:
                rd = remote_directories.pop()
                try:
                    remotelist = sftp.listdir(rd)
                except PermissionError as e:
                    logger.debug("Exception %s on %s" % (e, rd))
                    continue
                for item in remotelist:
                    itempath = os.path.join(rd, item)
                    try:
                        fstat = sftp.stat(itempath)
                    except exceptions as e:
                        logger.debug("Could not read %s: %s" % (item, e))
                        continue

                    if stat.S_ISDIR(fstat.st_mode):
                        remote_directories.append(itempath)
                        continue

                    totalsize += fstat.st_size
                    if totalsize > settings.FILES_MAX_DOWNLOAD_SIZE:
                        messages.error(request, "Too large to download (>200MB)")
                        return redirect("/files/{}?dir={}".format(fstype, os.path.dirname(fsdir)))

                    try:
                        localpath = os.path.join(tmpdir, os.path.relpath(rd, fsdir))
                        if not os.path.exists(localpath):
                            os.makedirs(localpath)
                        with open(os.path.join(localpath, item), "wb") as fh:
                            sftp.getfo(itempath, fh)
                    except exceptions as e:
                        logger.debug("Exception %s on %s" % (e, item))
                        continue

            with zipfile.ZipFile(tmpfile, "w", zipfile.ZIP_DEFLATED) as zf:
                for root, dirs, files in os.walk(tmpdir):
                    for f in files:
                        zf.write(os.path.join(root, f), os.path.join(os.path.relpath(root, tmpdir), f))

        content_len = tmpfile.tell()
        tmpfile.seek(0)
        chunk_size = 8192
        response = StreamingHttpResponse(FileWrapper(tmpfile, chunk_size), content_type="application/octet-stream")
        response["Content-Length"] = content_len
        if not dirbase_escaped:
            dirbase_escaped = "files"
        response["Content-Disposition"] = "attachment; filename={}".format(dirbase_escaped + ".zip")
        return response

    try:
        listdir = sftp.listdir()
    except exceptions as e:
        messages.error(request, e)
        listdir = []
    files = []
    for f in listdir:
        if not f.startswith("."):
            try:
                fstat = sftp.stat(f)
            except exceptions:
                # If we can't stat the file, don't show it
                continue
            files.append({
                "name": f,
                "folder": sftp.isdir(f),
                "stat": fstat,
                "stat_mtime": datetime.datetime.fromtimestamp(int(fstat.st_mtime or 0)),
                "too_big": fstat.st_size > settings.FILES_MAX_DOWNLOAD_SIZE
            })

    logger.debug(files)

    current_dir = normpath(sftp.pwd)  # current directory
    dir_list = current_dir.split("/")
    if len(dir_list) > 1 and len(dir_list[-1]) == 0:
        dir_list.pop()
    parent_dir = "/".join(dir_list[:-1])

    if len(parent_dir) == 0:
        parent_dir = "/"

    files = sorted(files, key=lambda f: (not f["folder"], f["name"]))

    context = {
        "host": host,
        "files": files,
        "current_dir": current_dir,
        "parent_dir": parent_dir if can_access_path(parent_dir) else None,
        "max_download_mb": (settings.FILES_MAX_DOWNLOAD_SIZE / 1024 / 1024)
    }

    return render(request, "files/directory.html", context)