Reads PSMs from file, stores them to a database backend in chunked PSMs.
188179
def create_psm_lookup(fn, fastafn, mapfn, header, pgdb, unroll=False,
                      specfncol=None, decoy=False,
                      fastadelim=None, genefield=None):
    
    proteins = store_proteins_descriptions(pgdb, fastafn, fn, mapfn, header,
                                           decoy, fastadelim, genefield)
    mzmlmap = pgdb.get_mzmlfile_map()
    sequences = {}
    for psm in tsvreader.generate_tsv_psms(fn, header):
        seq = tsvreader.get_psm_sequence(psm, unroll)
        sequences[seq] = 1
    pgdb.store_pepseqs(((seq,) for seq in sequences))
    pepseqmap = pgdb.get_peptide_seq_map()
    psms = []
    for row, psm in enumerate(tsvreader.generate_tsv_psms(fn, header)):
        specfn, psm_id, scan, seq, score = tsvreader.get_psm(psm, unroll,
                                                             specfncol)
        if len(psms) % DB_STORE_CHUNK == 0:
            pgdb.store_psms(psms)
            psms = []
        psms.append({'rownr': row,
                     'psm_id': psm_id,
                     'seq': pepseqmap[seq],
                     'score': score,
                     'specfn': mzmlmap[specfn],
                     'scannr': scan,
                     'spec_id': '{}_{}'.format(mzmlmap[specfn], scan),
                     })
    pgdb.store_psms(psms)
    pgdb.index_psms()
    store_psm_protein_relations(fn, header, pgdb, proteins)
922631
private Memory readChunkOffsets(DataInput input) {
        try {
            int chunkCount = input.readInt();
            Memory offsets = Memory.allocate(chunkCount * 8);

            for (int i = 0; i < chunkCount; i++) {
                try {
                    offsets.setLong(i * 8, input.readLong());
                } catch (EOFException e) {
                    String msg = String.format("Corrupted Index File %s: read %d but expected %d chunks.",
                            indexFilePath, i, chunkCount);
                    throw new CorruptSSTableException(new IOException(msg, e), indexFilePath);
                }
            }

            return offsets;
        } catch (IOException e) {
            throw new FSReadError(e, indexFilePath);
        }
    }
1281301
def ReadBlobs(self, blob_ids, cursor=None):
    """"""
    if not blob_ids:
      return {}

    query = ("SELECT blob_id, blob_chunk "
             "FROM blobs "
             "FORCE INDEX (PRIMARY) "
             "WHERE blob_id IN {} "
             "ORDER BY blob_id, chunk_index ASC").format(
                 mysql_utils.Placeholders(len(blob_ids)))
    cursor.execute(query, [blob_id.AsBytes() for blob_id in blob_ids])
    results = {blob_id: None for blob_id in blob_ids}
    for blob_id_bytes, blob in cursor.fetchall():
      blob_id = rdf_objects.BlobID.FromBytes(blob_id_bytes)
      if results[blob_id] is None:
        results[blob_id] = blob
      else:
        results[blob_id] += blob
    return results
77334
private byte[] getFileBytes(File file) throws CmsException {

        byte[] buffer = null;

        FileInputStream fileStream = null;
        int charsRead;
        int size;
        try {
            fileStream = new FileInputStream(file);
            charsRead = 0;
            size = new Long(file.length()).intValue();
            buffer = new byte[size];
            while (charsRead < size) {
                charsRead += fileStream.read(buffer, charsRead, size - charsRead);
            }
            return buffer;
        } catch (IOException e) {
            throw new CmsDbIoException(
                Messages.get().container(Messages.ERR_GET_FILE_BYTES_1, file.getAbsolutePath()),
                e);
        } finally {
            closeStream(fileStream);
        }
    }
1041657
def assemble_pysb():
    """"""
    if request.method == 'OPTIONS':
        return {}
    response = request.body.read().decode('utf-8')
    body = json.loads(response)
    stmts_json = body.get('statements')
    export_format = body.get('export_format')
    stmts = stmts_from_json(stmts_json)
    pa = PysbAssembler()
    pa.add_statements(stmts)
    pa.make_model()
    try:
        for m in pa.model.monomers:
            pysb_assembler.set_extended_initial_condition(pa.model, m, 0)
    except Exception as e:
        logger.exception(e)

    if not export_format:
        model_str = pa.print_model()
    elif export_format in ('kappa_im', 'kappa_cm'):
        fname = 'model_%s.png' % export_format
        root = os.path.dirname(os.path.abspath(fname))
        graph = pa.export_model(format=export_format, file_name=fname)
        with open(fname, 'rb') as fh:
            data = 'data:image/png;base64,%s' % \
                base64.b64encode(fh.read()).decode() 
            return {'image': data}
    else:
        try:
            model_str = pa.export_model(format=export_format)
        except Exception as e:
            logger.exception(e)
            model_str = ''
    res = {'model': model_str}
    return res
1220350
def read(fnames, markov, progress):
    
    with infiles(fnames, progress) as fnames:
        for fname in fnames:
            with open(fname, 'r') as fp:
                if progress:
                    fp.seek(0, SEEK_END)
                    total = fp.tell()
                    title = truncate(fname, BAR_DESC_SIZE - 1, False)
                    pbar = tqdm(total=total, desc=title,
                                leave=False, unit='byte',
                                bar_format=BAR_FORMAT, dynamic_ncols=True)
                    fp.seek(0, SEEK_SET)
                    prev = 0
                else:
                    pbar = None

                try:
                    line = fp.readline()
                    while line:
                        markov.data(line, True)
                        if pbar is not None:
                            pos = fp.tell()
                            if pos <= total:
                                pbar.update(pos - prev)
                                prev = pos
                        line = fp.readline()
                finally:
                    if pbar is not None:
                        pbar.close()

                markov.data('', False)
637027
def buffered_read(fh, lock, offsets, bytecounts, buffersize=None):
    """"""
    if buffersize is None:
        buffersize = 2**26
    length = len(offsets)
    i = 0
    while i < length:
        data = []
        with lock:
            size = 0
            while size < buffersize and i < length:
                fh.seek(offsets[i])
                bytecount = bytecounts[i]
                data.append(fh.read(bytecount))
                # buffer = bytearray(bytecount)
                # n = fh.readinto(buffer)
                # data.append(buffer[:n])
                size += bytecount
                i += 1
        for segment in data:
            yield segment
791343
def process(parsed_files, opts)
      r = aggregate(parsed_files)
      r = pre_process(r, opts)
      r = to_class_objects(r, opts)
      return post_process(r, opts)
    end
166690
def _MultiStream(cls, fds):
    """"""
    for fd in fds:
      fd.Seek(0)
      while True:
        chunk = fd.Read(cls.MULTI_STREAM_CHUNK_SIZE)
        if not chunk:
          break
        yield fd, chunk, None
131761
def dumps(columns):
    '''
    
    '''
    fp = BytesIO()
    dump(columns, fp)
    fp.seek(0)
    return fp.read()
778544
def from_file(pkg_file):
        
        phys_reader = PhysPkgReader(pkg_file)
        content_types = _ContentTypeMap.from_xml(phys_reader.content_types_xml)
        pkg_srels = PackageReader._srels_for(phys_reader, PACKAGE_URI)
        sparts = PackageReader._load_serialized_parts(
            phys_reader, pkg_srels, content_types
        )
        phys_reader.close()
        return PackageReader(content_types, pkg_srels, sparts)
556325
def _read(self, directory, filename, session, path, name, extension,
              spatial, spatialReferenceID, replaceParamFile,
              force_relative=True):
        
        self.project_directory = directory
        with tmp_chdir(directory):
            # Headers to ignore
            HEADERS = ('GSSHAPROJECT',)

            # WMS Cards to include (don't discount as comments)
            WMS_CARDS = ('#INDEXGRID_GUID', '#PROJECTION_FILE', '#LandSoil',
                         '#CHANNEL_POINT_INPUT_WMS')

            GSSHAPY_CARDS = ('#GSSHAPY_EVENT_YML', )

            with open(path, 'r') as f:
                for line in f:
                    if not line.strip():
                        # Skip empty lines
                        continue

                    elif '#' in line.split()[0] and line.split()[0] \
                            not in WMS_CARDS + GSSHAPY_CARDS:
                        # Skip comments designated by the hash symbol
                        # (with the exception of WMS_CARDS and GSSHAPY_CARDS)
                        continue

                    try:
                        card = self._extractCard(line, force_relative)

                    except:
                        card = self._extractDirectoryCard(line, force_relative)

                    # Now that the cardName and cardValue are separated
                    # load them into the gsshapy objects
                    if card['name'] not in HEADERS:
                        # Create GSSHAPY Project Card object
                        prjCard = ProjectCard(name=card['name'], value=card['value'])

                        # Associate ProjectCard with ProjectFile
                        prjCard.projectFile = self

                        # Extract MAP_TYPE card value for convenience working
                        # with output maps
                        if card['name'] == 'MAP_TYPE':
                            self.mapType = int(card['value'])

            # Assign properties
            self.srid = spatialReferenceID
            self.name = name
            self.fileExtension = extension
828339
def parse_psqs(psqs_results_file):
    

    # TODO: generalize column names for all results, save as dict instead

    psqs_results = pd.read_csv(psqs_results_file, sep='\t', header=None)
    psqs_results['pdb_file'] = psqs_results[0].apply(lambda x: str(x).strip('./').strip('.pdb'))
    psqs_results = psqs_results.rename(columns = {1:'psqs_local', 2:'psqs_burial', 3:'psqs_contact', 4:'psqs_total'}).drop(0, axis=1)
    psqs_results['u_pdb'] = psqs_results['pdb_file'].apply(lambda x: x.upper() if len(x)==4 else np.nan)
    psqs_results['i_entry_name'] = psqs_results['pdb_file'].apply(lambda x: x.split('_model1')[0] if len(x)>4 else np.nan)
    psqs_results = psqs_results[pd.notnull(psqs_results.psqs_total)]

    return psqs_results
1000715
def file_chunk(filename, size=_BUFFERING):
    

    def chunks():
        with open(filename, "rb", _BUFFERING) as fd:
            buf = fd.read(size)
            while buf:
                yield buf
                buf = fd.read(size)
    return chunks
914448
private ProjectFile readDatabaseFile(InputStream inputStream) throws MPXJException
   {
      ProjectReader reader = new AstaDatabaseFileReader();
      addListeners(reader);
      return reader.read(inputStream);
   }
340261
function _savePps(&$raList) 
    {
        // Save each PPS WK
        for ($i = 0; $i < count($raList); $i++) {
            fwrite($this->_FILEH_, $raList[$i]->_getPpsWk());
        }
        // Adjust for Block
        $iCnt = count($raList);
        $iBCnt = $this->_BIG_BLOCK_SIZE / OLE_PPS_SIZE;
        if ($iCnt % $iBCnt) {
            for ($i = 0; $i < (($iBCnt - ($iCnt % $iBCnt)) * OLE_PPS_SIZE); $i++) {
                fwrite($this->_FILEH_, "\x00");
            }
        }
    }
1049307
def read_files(*sources, **kwds):
    
    filenames = _generate_filenames(sources)
    filehandles = _generate_handles(filenames)
    for fh, source in filehandles:
        try:
            f = mwtab.MWTabFile(source)
            f.read(fh)

            if kwds.get('validate'):
                validator.validate_file(mwtabfile=f,
                                        section_schema_mapping=mwschema.section_schema_mapping,
                                        validate_samples=True,
                                        validate_factors=True)
            yield f

            if VERBOSE:
                print("Processed file: {}".format(os.path.abspath(source)))

        except Exception as e:
            if VERBOSE:
                print("Error processing file: ", os.path.abspath(source), "\nReason:", e)
            pass