create a gffutils DB, in memory if we don't have write permissions
190915
def get_gtf_db(gtf, in_memory=False):
    
    db_file = gtf + ".db"
    if file_exists(db_file):
        return gffutils.FeatureDB(db_file)
    if not os.access(os.path.dirname(db_file), os.W_OK | os.X_OK):
        in_memory = True
    db_file = ":memory:" if in_memory else db_file
    if in_memory or not file_exists(db_file):
        infer_extent = guess_infer_extent(gtf)
        disable_extent = not infer_extent
        db = gffutils.create_db(gtf, dbfn=db_file,
                                disable_infer_genes=disable_extent,
                                disable_infer_transcripts=disable_extent)
    if in_memory:
        return db
    else:
        return gffutils.FeatureDB(db_file)
737728
def setPermissions(self, dbName, access) :
        """"""
        import json

        if not self.URL :
            raise CreationError("Please save user first", None, None)

        rights = []
        if access :
            rights.append("rw")

        rights = ''.join(rights)

        if not self.connection.hasDatabase(dbName) :
            raise KeyError("Unknown database: %s" % dbName)

        url = "%s/database/%s" % (self.URL, dbName)
        r = self.connection.session.put(url, data = json.dumps({"grant": rights}, default=str))
        if r.status_code < 200 or r.status_code > 202 :
            raise CreationError("Unable to grant rights", r.content)
1137668
def _writen(fd, data):
    """"""
    while data:
        n = os.write(fd, data)
        data = data[n:]
342480
public void createShareLib() {

        if (!oozieShareLibCreate) {
            LOG.info("OOZIE: Share Lib Create Disabled... skipping");
        } else {

            final String fullOozieTarFilePath = shareLibCacheDir + Path.SEPARATOR + getOozieTarFileName();

            try {

                // Get and extract the oozie release
                getOozieTarFileFromRepo();
                String oozieExtractTempDir = extractOozieTarFileToTempDir(new File(fullOozieTarFilePath));

                // Extract the sharelib tarball to a temp dir
                String fullOozieShareLibTarFilePath = oozieExtractTempDir + Path.SEPARATOR +
                        "oozie-" + getOozieVersionFromOozieTarFileName() + Path.SEPARATOR +
                        "oozie-sharelib-" + getOozieVersionFromOozieTarFileName() + ".tar.gz";
                String oozieShareLibExtractTempDir = extractOozieShareLibTarFileToTempDir(
                        new File(fullOozieShareLibTarFilePath));

                // Copy the sharelib into HDFS
                Path destPath = new Path(oozieHdfsShareLibDir + Path.SEPARATOR + "oozie" + Path.SEPARATOR +
                        SHARE_LIB_PREFIX + getTimestampDirectory());
                LOG.info("OOZIE: Writing share lib contents to: {}", destPath);
                hdfsFileSystem.copyFromLocalFile(false, new Path(new File(oozieShareLibExtractTempDir).toURI()), destPath);

                if (purgeLocalShareLibCache) {
                    FileUtils.deleteDirectory(new File(shareLibCacheDir));
                }

            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
1316002
def dump(self):
        
        dumpfile = self.args.dumpfile
        if not dumpfile:
            db, env = self.get_db_args_env()
            dumpfile = fileutils.timestamp_filename(
                'omero-database-%s' % db['name'], 'pgdump')

        log.info('Dumping database to %s', dumpfile)
        if not self.args.dry_run:
            self.pgdump('-Fc', '-f', dumpfile)
277019
def _save_db():
    """"""
    from pyci.utility import json_serial
    import json
    vms("Serializing DB to JSON in {}".format(datapath))
    with open(datapath, 'w') as f:
        json.dump(db, f, default=json_serial)
137332
def _write_pidfile(self):
        """"""
        flags = os.O_CREAT | os.O_RDWR
        try:
            # Some systems don't have os.O_EXLOCK
            flags = flags | os.O_EXLOCK
        except AttributeError:
            pass
        self._pid_fd = os.open(self.pidfile, flags, 0o666 & ~self.umask)
        os.write(self._pid_fd, str(os.getpid()).encode('utf-8'))
703843
def sanitize_gff_file(gff_fname,
                      in_memory=True,
                      in_place=False):
    
    db = None
    if is_gff_db(gff_fname):
        # It's a database filename, so load it
        db = gffutils.FeatureDB(gff_fname)
    else:
        # Need to create a database for file
        if in_memory:
            db = gffutils.create_db(gff_fname, ":memory:",
                                    verbose=False)
        else:
            db = get_gff_db(gff_fname)
    if in_place:
        gff_out = gffwriter.GFFWriter(gff_fname,
                                      in_place=in_place)
    else:
        gff_out = gffwriter.GFFWriter(sys.stdout)
    sanitized_db = sanitize_gff_db(db)
    for gene_rec in sanitized_db.all_features(featuretype="gene"):
        gff_out.write_gene_recs(sanitized_db, gene_rec.id)
    gff_out.close()
278101
def create_db(name, owner=None, encoding=u'UTF-8', template='template1',
              **kwargs):
    """"""

    flags = u''
    if encoding:
        flags = u'-E %s' % encoding
    if owner:
        flags = u'%s -O %s' % (flags, owner)
    if template and template != 'template1':
        flags = u'%s --template=%s' % (flags, template)
    sudo('createdb %s %s' % (flags, name), user='postgres', **kwargs)
809829
def create_tables(args):
  """"""

  from bob.db.utils import create_engine_try_nolock
  engine = create_engine_try_nolock(args.type, args.files[0], echo=(args.verbose > 2))
  Base.metadata.create_all(engine)
1179411
def WriteArtifact(self, artifact, cursor=None):
    """"""
    name = Text(artifact.name)

    try:
      cursor.execute("INSERT INTO artifacts (name, definition) VALUES (%s, %s)",
                     [name, artifact.SerializeToString()])
    except MySQLdb.IntegrityError as error:
      if error.args[0] == mysql_error_constants.DUP_ENTRY:
        raise db.DuplicatedArtifactError(name, cause=error)
      else:
        raise
1229938
def download_memory_dump(self, sha256, environment_id, dest_dir):
        """"""

        dest_path = os.path.join(dest_dir, 'memory.zip')
        if self.download(sha256, environment_id, VXSTREAM_DOWNLOAD_MEMORY, dest_path) is None:
            return None

        with open(dest_path, 'rb') as fp:
            blob = fp.read(1024)
            if b'No dump files available' in blob:
                logging.debug("memory dump not available for {} env {}".format(sha256, environment_id))
                return None

        logging.debug("extracting memory dump {} into {}".format(dest_path, dest_dir))
        p = Popen(['7z', 'x', '-y', '-o{}'.format(dest_dir), dest_path], stdout=PIPE, stderr=PIPE)
        stdout, stderr = p.communicate()

        file_list = []
        for file_path in [os.path.join(dest_dir, f) for f in os.listdir(dest_dir) if f != "memory.zip"]:
            file_list.append(file_path)

        # concatenate all these files into one file
        dest_path = os.path.join(dest_dir, 'memory.combined.mdmp')
        for file_path in file_list:
            logging.debug("concatenating {}".format(file_path))
            with open(file_path, 'rb') as input_fp:
                with open(dest_path, 'ab') as output_fp:
                    while True:
                        data = input_fp.read(io.DEFAULT_BUFFER_SIZE)
                        if data == b'':
                            break

                        output_fp.write(data)

        return file_list, dest_path
166572
def write_memory_dump():
    
    # first do a full garbage collection run
    gc.collect()
    if gc.garbage:
        log.warn(LOG_CHECK, "Unreachabe objects: %s", pprint.pformat(gc.garbage))
    from meliae import scanner
    fo, filename = get_temp_file(mode='wb', suffix='.json', prefix='lcdump_')
    try:
        scanner.dump_all_objects(fo)
    finally:
        fo.close()
    return filename
564341
def cli(env, identifier):
    """"""

    mgr = SoftLayer.ObjectStorageManager(env.client)
    credential_limit = mgr.limit_credential(identifier)
    table = formatting.Table(['limit'])
    table.add_row([
        credential_limit,
    ])

    env.fout(table)
343345
def prepare_writeable_dir(tree):
    '''  '''

    if tree != '/':
        tree = os.path.realpath(os.path.expanduser(tree))
    if not os.path.exists(tree):
        try:
            os.makedirs(tree)
        except (IOError, OSError), e:
            exit("Could not make dir %s: %s" % (tree, e))
    if not os.access(tree, os.W_OK):
        exit("Cannot write to path %s" % tree)
113532
def write(self):
        """"""
        self._assure_writable("write")
        if not self._dirty:
            return

        if isinstance(self._file_or_files, (list, tuple)):
            raise AssertionError("Cannot write back if there is not exactly a single file to write to, have %i files"
                                 % len(self._file_or_files))
        # end assert multiple files

        if self._has_includes():
            log.debug("Skipping write-back of configuration file as include files were merged in." +
                      "Set merge_includes=False to prevent this.")
            return
        # end

        fp = self._file_or_files

        # we have a physical file on disk, so get a lock
        is_file_lock = isinstance(fp, string_types + (FileType, ))
        if is_file_lock:
            self._lock._obtain_lock()
        if not hasattr(fp, "seek"):
            with open(self._file_or_files, "wb") as fp:
                self._write(fp)
        else:
            fp.seek(0)
            # make sure we do not overwrite into an existing file
            if hasattr(fp, 'truncate'):
                fp.truncate()
            self._write(fp)
486176
def cli(ctx, directory, uri, verbose, count):
    """"""
    # configure root logger to print to STDERR
    loglevel = "INFO"
    if verbose:
        loglevel = "DEBUG"
    coloredlogs.install(level=loglevel)
    
    p = Path(directory)
    
    if not p.is_dir():
        LOG.warning("{0} is not a valid directory".format(directory))
        ctx.abort()
    
    start_time = datetime.now()
    # Make sure that the database is indexed
    index_call = ['loqusdb', 'index']
    base_call = ['loqusdb']
    if uri:
        base_call.append('--uri')
        base_call.append(uri)
        index_call.append('--uri')
        index_call.append(uri)
    
    subprocess.run(index_call)
    base_call.append('load')
    
    nr_files = 0
    for nr_files,file_name in enumerate(list(p.glob('*.vcf')),1):
        call = deepcopy(base_call)
        case_id = file_name.stem.split('.')[0]
        call.append('--sv-variants')
        call.append(str(file_name))
        call.append('--case-id')
        call.append(case_id)
        if count:
            continue
        try:
            subprocess.run(call, check=True)
        except subprocess.CalledProcessError as err:
            LOG.warning(err)
            LOG.warning("Failed to load file %s", filename)
            LOG.info("Continue with files...")
        
        if nr_files % 100:
            LOG.info("%s files loaded", nr_files)
        
    LOG.info("%s files inserted", nr_files)
    LOG.info("Time to insert files: {}".format(datetime.now()-start_time))