Use sdmpy to get all scans and info needed for rtpipe as dict
325198
def read_scans(sdmfile, bdfdir=''):
    

    sdm = getsdm(sdmfile, bdfdir=bdfdir)
    scandict = {}
    skippedscans = []

    for scan in sdm.scans():
        scannum = int(scan.idx)
        scandict[scannum] = {}

        intentstr = ' '.join(scan.intents)
        src = scan.source

        scandict[scannum]['source'] = src
        scandict[scannum]['intent'] = intentstr

        # bdf specific properties
        try:
            startmjd = scan.bdf.startTime
            nints = scan.bdf.numIntegration
            interval = scan.bdf.get_integration(0).interval
            endmjd = startmjd + (nints*interval)/(24*3600)
            bdfstr = scan.bdf.fname
        except AttributeError, IOError:
            skippedscans.append(scannum)
        else:
            scandict[scannum]['startmjd'] = startmjd
            scandict[scannum]['endmjd'] = endmjd
            scandict[scannum]['duration'] = endmjd-startmjd
            scandict[scannum]['nints'] = nints
            scandict[scannum]['bdfstr'] = bdfstr

            # clear reference to nonexistent BDFs (either bad or not in standard locations)
            if (not os.path.exists(scandict[scannum]['bdfstr'])) or ('X1' in bdfstr):
                scandict[scannum]['bdfstr'] = None
                logger.debug('Invalid bdf for %d of %s' % (scannum, sdmfile) )

    if skippedscans:
        logger.warn('No BDF found for scans {0}'.format(skippedscans))

    return scandict
1113753
def read_exports(self):
        
        result = {}
        r = self.get_distinfo_resource(EXPORTS_FILENAME)
        if r:
            with contextlib.closing(r.as_stream()) as stream:
                result = read_exports(stream)
        return result
968107
def asDict(self):
        
        dct = super(RtiRegItem, self).asDict()
        dct['extensions'] = self.extensions
        return dct
331655
def __get_remote_index(self):
        
        # cleanup
        subprocess.getoutput("rm tmpaws.txt")
        # get the remote info
        cmd = "aws s3 ls --recursive s3://commoncrawl/crawl-data/CC-NEWS/ --no-sign-request > tmpaws.txt && " \
              "awk '{ print $4 }' tmpaws.txt && " \
              "rm tmpaws.txt"
        self.__logger.info('executing: %s', cmd)
        stdout_data = subprocess.getoutput(cmd)
        print(stdout_data)

        lines = stdout_data.splitlines()
        return lines
1203941
def _get_memmap(self):
        """"""

        with open(self.filename) as fp:

            data_dtype = self._get_data_dtype()
            hdr_size = native_header.itemsize

            return np.memmap(fp, dtype=data_dtype,
                             shape=(self.mda['number_of_lines'],),
                             offset=hdr_size, mode="r")
357738
def getrawpart(self, msgid, stream=sys.stdout):
        
        for hdr, part in self._get(msgid):
            pl = part.get_payload(decode=True)
            if pl != None:
                print(pl, file=stream)
                break
658383
def to_dict(self):
        

        task_desc_as_dict = {
            'uid': self._uid,
            'name': self._name,
            'state': self._state,
            'state_history': self._state_history,

            'pre_exec': self._pre_exec,
            'executable': self._executable,
            'arguments': self._arguments,
            'post_exec': self._post_exec,
            'cpu_reqs': self._cpu_reqs,
            'gpu_reqs': self._gpu_reqs,
            'lfs_per_process': self._lfs_per_process,

            'upload_input_data': self._upload_input_data,
            'copy_input_data': self._copy_input_data,
            'link_input_data': self._link_input_data,
            'move_input_data': self._move_input_data,
            'copy_output_data': self._copy_output_data,
            'move_output_data': self._move_output_data,
            'download_output_data': self._download_output_data,

            'stdout': self._stdout,
            'stderr': self._stderr,

            'exit_code': self._exit_code,
            'path': self._path,
            'tag': self._tag,

            'parent_stage': self._p_stage,
            'parent_pipeline': self._p_pipeline,
        }

        return task_desc_as_dict
410182
def get_contigs(self):
        ''''''
        contigs = {}
        pyfastaq.tasks.file_to_dict(self.contigs_fasta, contigs)
        return contigs
985671
def getMetadata(self, remote, address, key):
        """"""
        if self._server is not None:
            # pylint: disable=E1121
            return self._server.getAllMetadata(remote, address, key)
752941
def get_measured_regions(self):
        
        mergeon = ['project_id','project_name',
                'sample_id','sample_name',
                'frame_id','frame_name',
                ]
        temp = self.loc[:,mergeon+['regions']].\
            set_index(mergeon)['regions'].apply(json.dumps).\
            reset_index().drop_duplicates()
        temp['regions'] = temp['regions'].apply(json.loads)
        rows = []
        for i,r in temp.iterrows():
            for label in r['regions']:
                a = list(r.index)
                b = list(r.values)
                a = a+['region_label','region_area_pixels']
                b = b+[label,r['regions'][label]]
                rows.append(dict(zip(a,b)))
        rows = pd.DataFrame(rows).drop(columns='regions').\
            drop_duplicates()[mergeon+['region_label','region_area_pixels']]
        #rows = rows.loc[rows['region_area_pixels']>0].copy()
        return rows
424972
def to_dict(self):
        

        input_dict = super(StdPeriodic, self)._save_to_input_dict()
        input_dict["class"] = "GPy.kern.StdPeriodic"
        input_dict["variance"] = self.variance.values.tolist()
        input_dict["period"] = self.period.values.tolist()
        input_dict["lengthscale"] = self.lengthscale.values.tolist()
        input_dict["ARD1"] = self.ARD1
        input_dict["ARD2"] = self.ARD2
        return input_dict
368125
def get_parameters(self, packet_count=None):
        
        params = super(PipeCapture, self).get_parameters(packet_count=packet_count)
        params += ['-r', '-']
        return params
885944
def to_dict(self):
        ''''''
        d = {'connectorId': self.connector_id,
                'name': self.name,
                'dataType': self.data_type,
                'interfaceType': self.interface_type,
                'dataflowType': self.data_flow_type,
                'sourceDataPort': self.source_data_port.to_dict(),
                'targetDataPort': self.target_data_port.to_dict()}
        if self.visible != True:
            d[RTS_EXT_NS_YAML + 'visible'] = self.visible
        if self.subscription_type:
            d['subscriptionType'] = self.subscription_type
        if self.push_interval:
            d['pushInterval'] = self.push_interval
        if self.comment:
            d[RTS_EXT_NS_YAML + 'comment'] = self.comment
        props = []
        for name in self.properties:
            p = {'name': name}
            if self.properties[name]:
                p['value'] = str(self.properties[name])
            props.append(p)
        if props:
            d[RTS_EXT_NS_YAML + 'properties'] = props
        return d
850146
def as_dict(self):
        
        if self.has_omp:
            raise NotImplementedError('as_dict method of QueueAdapter not yet implemented when OpenMP is activated')
        return {'@module': self.__class__.__module__,
                '@class': self.__class__.__name__,
                'priority': self.priority,
                'hardware': self.hw.as_dict(),
                'queue': {'qtype': self.QTYPE,
                          'qname': self._qname,
                          'qnodes': self.qnodes,
                          'qparams': self._qparams},
                'limits': {'timelimit_hard': self._timelimit_hard,
                           'timelimit': self._timelimit,
                           'min_cores': self.min_cores,
                           'max_cores': self.max_cores,
                           'min_mem_per_proc': self.min_mem_per_proc,
                           'max_mem_per_proc': self.max_mem_per_proc,
                           'memory_policy': self.memory_policy
                           },
                'job': {},
                'mpi_procs': self._mpi_procs,
                'mem_per_proc': self._mem_per_proc,
                'master_mem_overhead': self._master_mem_overhead
                }
773443
def _linux_memdata():
    '''
    
    '''
    grains = {'mem_total': 0, 'swap_total': 0}

    meminfo = '/proc/meminfo'
    if os.path.isfile(meminfo):
        with salt.utils.files.fopen(meminfo, 'r') as ifile:
            for line in ifile:
                comps = line.rstrip('\n').split(':')
                if not len(comps) > 1:
                    continue
                if comps[0].strip() == 'MemTotal':
                    # Use floor division to force output to be an integer
                    grains['mem_total'] = int(comps[1].split()[0]) // 1024
                if comps[0].strip() == 'SwapTotal':
                    # Use floor division to force output to be an integer
                    grains['swap_total'] = int(comps[1].split()[0]) // 1024
    return grains
883186
def _pack_prms():
    """"""

    config_dict = {
        "Paths": prms.Paths.to_dict(),
        "FileNames": prms.FileNames.to_dict(),
        "Db": prms.Db.to_dict(),
        "DbCols": prms.DbCols.to_dict(),
        "DataSet": prms.DataSet.to_dict(),
        "Reader": prms.Reader.to_dict(),
        "Instruments": prms.Instruments.to_dict(),
        # "excel_db_cols": prms.excel_db_cols.to_dict(),
        # "excel_db_filename_cols": prms.excel_db_filename_cols.to_dict(),
        "Batch": prms.Batch.to_dict(),
    }
    return config_dict
641255
def scan(self, t, dt=None, aggfunc=None):
        
        return self.data.scan(t, dt, aggfunc)