Read risk data and sources if any
549669
def read_inputs(self):
        
        oq = self.oqparam
        self._read_risk_data()
        self.check_overflow()  # check if self.sitecol is too large
        if ('source_model_logic_tree' in oq.inputs and
                oq.hazard_calculation_id is None):
            self.csm = readinput.get_composite_source_model(
                oq, self.monitor(), srcfilter=self.src_filter)
        self.init()
290629
def read_data(self):
        """"""
        window_start = self.parent.value('window_start')
        window_end = window_start + self.parent.value('window_length')
        dataset = self.parent.info.dataset
        groups = self.parent.channels.groups

        chan_to_read = []
        for one_grp in groups:
            chan_to_read.extend(one_grp['chan_to_plot'] + one_grp['ref_chan'])

        if not chan_to_read:
            return
        data = dataset.read_data(chan=chan_to_read,
                                 begtime=window_start,
                                 endtime=window_end)

        max_s_freq = self.parent.value('max_s_freq')
        if data.s_freq > max_s_freq:
            q = int(data.s_freq / max_s_freq)
            lg.debug('Decimate (no low-pass filter) at ' + str(q))

            data.data[0] = data.data[0][:, slice(None, None, q)]
            data.axis['time'][0] = data.axis['time'][0][slice(None, None, q)]
            data.s_freq = int(data.s_freq / q)

        self.data = _create_data_to_plot(data, self.parent.channels.groups)
835576
def _readall(self):
        
        with _handle_oss_error():
            return self._bucket.get_object(key=self._key).read()
963874
def readShocks(self):
        '''
        
        '''
        IndShockConsumerType.readShocks(self)
        self.MrkvNow = self.MrkvNow.astype(int)
1188304
def read(file, system):
    """"""

    blocks = [
        'bus', 'load', 'fshunt', 'gen', 'branch', 'transf', 'area',
        'twotermdc', 'vscdc', 'impedcorr', 'mtdc', 'msline', 'zone',
        'interarea', 'owner', 'facts', 'swshunt', 'gne', 'Q'
    ]
    nol = [1, 1, 1, 1, 1, 4, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0]
    rawd = re.compile(r'rawd\d\d')

    retval = True
    version = 0
    b = 0  # current block index
    raw = {}
    for item in blocks:
        raw[item] = []

    data = []
    mdata = []  # multi-line data
    mline = 0  # line counter for multi-line models

    # parse file into raw with to_number conversions
    fid = open(file, 'r')
    for num, line in enumerate(fid.readlines()):
        line = line.strip()
        if num == 0:  # get basemva and frequency
            data = line.split('/')[0]
            data = data.split(',')

            mva = float(data[1])
            system.config.mva = mva
            system.config.freq = float(data[5])

            # get raw file version
            version = int(data[2])
            if not version:
                version = int(rawd.search(line).group(0).strip('rawd'))
            logger.debug('PSSE raw version {} detected'.format(version))

            if version < 32 or version > 33:
                logger.warning(
                    'RAW file version is not 32 or 33. Error may occur.')
            continue

        elif num == 1:  # store the case info line
            logger.info(line)
            continue
        elif num == 2:
            continue
        elif num >= 3:
            if line[0:2] == '0 ' or line[0:3] == ' 0 ':  # end of block
                b += 1
                continue
            elif line[0] == 'Q':  # end of file
                break
            data = line.split(',')

        data = [to_number(item) for item in data]
        mdata.append(data)
        mline += 1
        nol = get_nol(b, mdata)

        if mline >= nol:
            if nol == 1:
                mdata = mdata[0]
            raw[blocks[b]].append(mdata)
            mdata = []
            mline = 0
    fid.close()

    # add device elements to system
    sw = {}  # idx:a0
    max_bus = []
    for data in raw['bus']:
        """version 32:
          0,   1,      2,     3,    4,   5,  6,   7,  8
          ID, NAME, BasekV, Type, Area Zone Owner Vm, Va
        """
        idx = data[0]
        max_bus.append(idx)
        ty = data[3]
        a0 = data[8] * deg2rad

        if ty == 3:
            sw[idx] = a0
        param = {
            'idx': idx,
            'name': data[1],
            'Vn': data[2],
            'voltage': data[7],
            'angle': a0,
            'area': data[4],
            'zone': data[5],
            'owner': data[6],
        }
        system.Bus.elem_add(**param)
    max_bus = max(max_bus)
    for data in raw['load']:
        """version 32:
          0,  1,      2,    3,    4,    5,    6,      7,   8,  9, 10,   11
        Bus, Id, Status, Area, Zone, PL(MW), QL (MW), IP, IQ, YP, YQ, OWNER
        """
        bus = data[0]
        vn = system.Bus.get_field('Vn', bus)
        voltage = system.Bus.get_field('voltage', bus)
        param = {
            'bus': bus,
            'Vn': vn,
            'Sn': mva,
            'p': (data[5] + data[7] * voltage + data[9] * voltage**2) / mva,
            'q': (data[6] + data[8] * voltage - data[10] * voltage**2) / mva,
            'owner': data[11],
        }
        system.PQ.elem_add(**param)

    for data in raw['fshunt']:
        """
        0,    1,      2,      3,      4
        Bus, name, Status, g (MW), b (Mvar)
        """
        bus = data[0]
        vn = system.Bus.get_field('Vn', bus)
        param = {
            'bus': bus,
            'Vn': vn,
            'u': data[2],
            'Sn': mva,
            'g': data[3] / mva,
            'b': data[4] / mva,
        }
        system.Shunt.elem_add(**param)

    gen_idx = 0
    for data in raw['gen']:
        """
         0, 1, 2, 3, 4, 5, 6, 7,    8,   9,10,11, 12, 13, 14,   15, 16,17,18,19
         I,ID,PG,QG,QT,QB,VS,IREG,MBASE,ZR,ZX,RT,XT,GTAP,STAT,RMPCT,PT,PB,O1,F1
        """
        bus = data[0]
        vn = system.Bus.get_field('Vn', bus)
        gen_mva = data[8]  # unused yet
        gen_idx += 1
        status = data[14]
        param = {
            'Sn': gen_mva,
            'Vn': vn,
            'u': status,
            'idx': gen_idx,
            'bus': bus,
            'pg': status * data[2] / mva,
            'qg': status * data[3] / mva,
            'qmax': data[4] / mva,
            'qmin': data[5] / mva,
            'v0': data[6],
            'ra': data[9],  # ra  armature resistance
            'xs': data[10],  # xs synchronous reactance
            'pmax': data[16] / mva,
            'pmin': data[17] / mva,
        }
        if data[0] in sw.keys():
            param.update({
                'a0': sw[data[0]],
            })
            system.SW.elem_add(**param)
        else:
            system.PV.elem_add(**param)

    for data in raw['branch']:
        """
        I,J,CKT,R,X,B,RATEA,RATEB,RATEC,GI,BI,GJ,BJ,ST,LEN,O1,F1,...,O4,F4
        """
        param = {
            'bus1': data[0],
            'bus2': data[1],
            'r': data[3],
            'x': data[4],
            'b': data[5],
            'rate_a': data[6],
            'Vn': system.Bus.get_field('Vn', data[0]),
            'Vn2': system.Bus.get_field('Vn', data[1]),
        }
        system.Line.elem_add(**param)

    xf_3_count = 1
    for data in raw['transf']:
        if len(data) == 4:
            """
            I,J,K,CKT,CW,CZ,CM,MAG1,MAG2,NMETR,'NAME',STAT,O1,F1,...,O4,F4
            R1-2,X1-2,SBASE1-2
            WINDV1,NOMV1,ANG1,RATA1,RATB1,RATC1,COD1,CONT1,RMA1,RMI1,VMA1,VMI1,NTP1,TAB1,CR1,CX1
            WINDV2,NOMV2
            """

            tap = data[2][0]
            phi = data[2][2]

            if tap == 1 and phi == 0:
                trasf = False
            else:
                trasf = True
            param = {
                'trasf': trasf,
                'bus1': data[0][0],
                'bus2': data[0][1],
                'u': data[0][11],
                'b': data[0][8],
                'r': data[1][0],
                'x': data[1][1],
                'tap': tap,
                'phi': phi,
                'rate_a': data[2][3],
                'Vn': system.Bus.get_field('Vn', data[0][0]),
                'Vn2': system.Bus.get_field('Vn', data[0][1]),
            }
            system.Line.elem_add(**param)
        else:
            """
            I, J, K, CKT, CW, CZ, CM, MAG1, MAG2, NMETR, 'NAME', STAT, Ol, Fl,...,o4, F4
            R1—2, X1—2, SBASE1—2, R2—3, X2—3, SBASE2—3, R3—1, X3—1, SBASE3—1, VMSTAR, ANSTAR
            WINDV1, NOMV1, ANG1, RATA1, BATB1, RATC1, COD1, CONT1, RMA1, RMI1, VMA1, VMI1, NTP1, TAB1, CR1, CX1
            WINDV2, NOMV2, ANG2, RATA2, BATB2, RATC2, COD2, CONT2, RMA2, RMI2, VMA2, VMI2, NTP2, TAB2, CR2, CX2
            WINDV3, NOMV3, ANG3, RATA3, BATB3, RATC3, COD3, CONT3, RMA3, RMI3, VMA3, VMI3, NTP3, TAB3, CR3, CX3
            """
            param = {
                'idx': max_bus+xf_3_count,
                'name': '_'.join(str(data[0][:3])),
                'Vn': system.Bus.get_field('Vn', data[0][0]),
                'voltage': data[1][-2],
                'angle': data[1][-1]/180*3.14
            }
            system.Bus.elem_add(**param)

            r = []
            x = []
            r.append((data[1][0] + data[1][6] - data[1][3])/2)
            r.append((data[1][3] + data[1][0] - data[1][6])/2)
            r.append((data[1][6] + data[1][3] - data[1][0])/2)
            x.append((data[1][1] + data[1][7] - data[1][4])/2)
            x.append((data[1][4] + data[1][1] - data[1][7])/2)
            x.append((data[1][7] + data[1][4] - data[1][1])/2)
            for i in range(0, 3):
                param = {
                    'trasf': True,
                    'bus1': data[0][i],
                    'bus2': max_bus+xf_3_count,
                    'u': data[0][11],
                    'b': data[0][8],
                    'r': r[i],
                    'x': x[i],
                    'tap': data[2+i][0],
                    'phi': data[2+i][2],
                    'rate_a': data[2+i][3],
                    'Vn': system.Bus.get_field('Vn', data[0][i]),
                    'Vn2': system.Bus.get_field('Vn', data[0][0]),
                }
                system.Line.elem_add(**param)
            xf_3_count += 1

    for data in raw['swshunt']:
        # I, MODSW, ADJM, STAT, VSWHI, VSWLO, SWREM, RMPCT, ’RMIDNT’,
        # BINIT, N1, B1, N2, B2, ... N8, B8
        bus = data[0]
        vn = system.Bus.get_field('Vn', bus)
        param = {
            'bus': bus,
            'Vn': vn,
            'Sn': mva,
            'u': data[3],
            'b': data[9] / mva,
        }
        system.Shunt.elem_add(**param)

    for data in raw['area']:
        """ID, ISW, PDES, PTOL, ARNAME"""
        param = {
            'idx': data[0],
            'isw': data[1],
            'pdes': data[2],
            'ptol': data[3],
            'name': data[4],
        }
        system.Area.elem_add(**param)

    for data in raw['zone']:
        """ID, NAME"""
        param = {
            'idx': data[0],
            'name': data[1],
        }
        system.Zone.elem_add(**param)

    return retval
708527
def handle_read(repo, **kwargs):
    """"""
    log.info('read: %s %s' %(repo, kwargs))
    if type(repo) in [unicode, str]:
        return {'name': 'Repo', 'desc': 'Welcome to Grit', 'comment': ''}
    else:
        return repo.serialize()
968934
def load_data(verbose=False):
    ""

    df = pd.read_csv(STOPS_PATH, usecols=['stop_id', 'stop_name'])
    if verbose:
        print('- Loaded %d entries from "%s".' % (len(df), STOPS_PATH))
    return df
534249
def parse_raw_list_data(data, proxy_type='http', proxy_userpwd=None):
    """"""
    if not isinstance(data, six.text_type):
        data = data.decode('utf-8')
    for orig_line in data.splitlines():
        line = orig_line.strip().replace(' ', '')
        if line and not line.startswith('#'):
            try:
                host, port, username, password = parse_proxy_line(line)
            except InvalidProxyLine as ex:
                logger.error(ex)
            else:
                if username is None and proxy_userpwd is not None:
                    username, password = proxy_userpwd.split(':')
                yield Proxy(host, port, username, password, proxy_type)
701305
def config_read():
    """"""
    config_file = (u"{0}config.ini".format(CONFIG_DIR))
    if not os.path.isfile(config_file):
        config_make(config_file)
    config = configparser.ConfigParser(allow_no_value=True)
    try:
        config.read(config_file, encoding='utf-8')
    except IOError:
        print("Error reading config file: {}".format(config_file))
        sys.exit()
    # De-duplicate provider-list
    providers = config_prov(config)
    # Read credentials for listed providers
    (cred, to_remove) = config_cred(config, providers)
    # remove unsupported and credential-less providers
    for item in to_remove:
        providers.remove(item)
    return cred, providers
120948
def read_data(self, **kwargs):
        
        trigger_id = kwargs.get('trigger_id')
        date_triggered = kwargs.get('date_triggered')
        data = []
        project_name = 'Main Project'
        items = self.todoist.sync()
        try:
            for item in items.get('items'):
                date_added = arrow.get(item.get('date_added'), 'ddd DD MMM YYYY HH:mm:ss ZZ')
                if date_added > date_triggered:
                    for project in items.get('projects'):
                        if item.get('project_id') == project.get('id'):
                            project_name = project.get('name')
                    title = 'From TodoIst Project {0}:'.format(project_name)
                    data.append({'title': title, 'content': item.get('content')})

                    # digester
                    self.send_digest_event(trigger_id, title, '')

            cache.set('th_todoist_' + str(trigger_id), data)
        except AttributeError:
            logger.error(items)

        return data
1362724
def get_wd_data(self):
        
        wait = wx.BusyInfo('Reading in data from current working directory, please wait...')
        #wx.Yield()
        print('-I- Read in any available data from working directory')
        self.contribution = cb.Contribution(self.WD, dmodel=self.data_model)
        del wait
293617
def data(self):
        """"""
        if self.kind == 'data':
            return self._data.data
        else:
            with self.open() as f:
                return f.read()
687097
def load_data(handle, reader=None):
    ''''''
    if not reader:
        reader = os.path.splitext(handle)[1][1:].lower()

    if reader not in _READERS:
        raise NeuroMError('Do not have a loader for "%s" extension' % reader)

    filename = _get_file(handle)
    try:
        return _READERS[reader](filename)
    except Exception as e:
        L.exception('Error reading file %s, using "%s" loader', filename, reader)
        raise RawDataError('Error reading file %s:\n%s' % (filename, str(e)))
902222
def inspect_data(self, capacity=None, voltage=None,
                     err_est=False, diff_est=False):
        """"""

        logging.debug("inspecting the data")

        if capacity is None:
            capacity = self.capacity
        if voltage is None:
            voltage = self.voltage

        if capacity is None or voltage is None:
            raise NullData

        self.len_capacity = len(capacity)
        self.len_voltage = len(voltage)

        if self.len_capacity <= 1:
            raise NullData
        if self.len_voltage <= 1:
            raise NullData

        self.min_capacity, self.max_capacity = value_bounds(capacity)
        self.start_capacity, self.end_capacity = index_bounds(capacity)

        self.number_of_points = len(capacity)

        if diff_est:
            d_capacity = np.diff(capacity)
            d_voltage = np.diff(voltage)

            self.d_capacity_mean = np.mean(d_capacity)
            self.d_voltage_mean = np.mean(d_voltage)

        if err_est:

            splits = int(self.number_of_points / self.points_pr_split)
            rest = self.number_of_points % self.points_pr_split

            if splits < self.minimum_splits:
                txt = "no point in splitting, too little data"
                logging.debug(txt)
                self.errors.append("splitting: to few points")
            else:
                if rest > 0:
                    _cap = capacity[:-rest]
                    _vol = voltage[:-rest]
                else:
                    _cap = capacity
                    _vol = voltage

                c_pieces = np.split(_cap, splits)
                v_pieces = np.split(_vol, splits)
                # c_middle = int(np.amax(c_pieces) / 2)

                std_err = []
                c_pieces_avg = []
                for c, v in zip(c_pieces, v_pieces):
                    _slope, _intercept, _r_value, _p_value, _std_err = stats.linregress(c, v)
                    std_err.append(_std_err)
                    c_pieces_avg.append(np.mean(c))

                self.std_err_median = np.median(std_err)
                self.std_err_mean = np.mean(std_err)

        if not self.start_capacity == self.min_capacity:
            self.errors.append("capacity: start<>min")

        if not self.end_capacity == self.max_capacity:
            self.errors.append("capacity: end<>max")

        if self.normalizing_factor is None:
            self.normalizing_factor = self.end_capacity

        if self.normalizing_roof is not None:
            self.normalizing_factor = self.normalizing_factor * \
                                      self.end_capacity / self.normalizing_roof
1229357
def ():
    """"""
    clean_dir = ev(
        "OUTPUT_DIR",
        "/tmp")
    clean_file = ev(
        "CLEANED_FILE",
        "{}/cleaned_attack_scans.csv".format(
            clean_dir))
    fulldata_file = ev(
        "FULLDATA_FILE",
        "{}/fulldata_attack_scans.csv".format(
            clean_dir))
    dataset_dir = ev(
        "DS_DIR",
        "/opt/antinex/datasets")
    csv_glob_path = ev(
        "DS_GLOB_PATH",
        "{}/*/*.csv".format(
            dataset_dir))

    pipeline_files = find_all_pipeline_csvs(
        csv_glob_path=csv_glob_path)

    post_proc_rules = {
        "drop_columns": [
            "src_file",
            "raw_id",
            "raw_load",
            "raw_hex_load",
            "raw_hex_field_load",
            "pad_load",
            "eth_dst",  # need to make this an int
            "eth_src",  # need to make this an int
            "ip_dst",   # need to make this an int
            "ip_src"    # need to make this an int
        ],
        "predict_feature": "label_name"
    }

    label_rules = {
        "set_if_above": 85,
        "labels": ["not_attack", "attack"],
        "label_values": [0, 1]
    }

    log.info("building csv")

    save_node = build_csv(
        pipeline_files=pipeline_files,
        fulldata_file=fulldata_file,
        clean_file=clean_file,
        post_proc_rules=post_proc_rules,
        label_rules=label_rules)

    if save_node["status"] == VALID:
        log.info("Successfully process datasets:")

        if ev("SHOW_SUMMARY", "1") == "1":
            log.info(("Full csv: {}")
                     .format(save_node["fulldata_file"]))
            log.info(("Full meta: {}")
                     .format(save_node["fulldata_metadata_file"]))
            log.info(("Clean csv: {}")
                     .format(save_node["clean_file"]))
            log.info(("Clean meta: {}")
                     .format(save_node["clean_metadata_file"]))
            log.info("------------------------------------------")
            log.info(("Predicting Feature: {}")
                     .format(save_node["feature_to_predict"]))
            log.info(("Features to Process: {}")
                     .format(ppj(save_node["features_to_process"])))
            log.info(("Ignored Features: {}")
                     .format(ppj(save_node["ignore_features"])))
            log.info("------------------------------------------")
        # end of show summary

        log.info("")
        log.info("done saving csv:")
        log.info("Full: {}".format(
            save_node["fulldata_file"]))
        log.info("Cleaned (no-NaNs in columns): {}".format(
            save_node["clean_file"]))
        log.info("")
    else:
        log.info("Failed to process datasets")
48270
def read(self):
        """"""
        try:
            f = urllib.request.urlopen(self.url)
        except urllib.error.HTTPError as err:
            if err.code in (401, 403):
                self.disallow_all = True
            elif err.code >= 400:
                self.allow_all = True
        else:
            raw = f.read()
            self.parse(raw.decode("utf-8").splitlines())
340776
def get_data_statistics(interpreted_files):
    '''
    '''
    print '| *File Name* | *File Size* | *Times Stamp* | *Events* | *Bad Events* | *Measurement time* | *# SR* | *Hits* |'  # Mean Tot | Mean rel. BCID'
    for interpreted_file in interpreted_files:
        with tb.open_file(interpreted_file, mode="r") as in_file_h5:  # open the actual hit file
            n_hits = np.sum(in_file_h5.root.HistOcc[:])
            measurement_time = int(in_file_h5.root.meta_data[-1]['timestamp_stop'] - in_file_h5.root.meta_data[0]['timestamp_start'])
#             mean_tot = np.average(in_file_h5.root.HistTot[:], weights=range(0,16) * np.sum(range(0,16)))# / in_file_h5.root.HistTot[:].shape[0]
#             mean_bcid = np.average(in_file_h5.root.HistRelBcid[:], weights=range(0,16))
            n_sr = np.sum(in_file_h5.root.HistServiceRecord[:])
            n_bad_events = int(np.sum(in_file_h5.root.HistErrorCounter[2:]))
            try:
                n_events = str(in_file_h5.root.Hits[-1]['event_number'] + 1)
            except tb.NoSuchNodeError:
                n_events = '~' + str(in_file_h5.root.meta_data[-1]['event_number'] + (in_file_h5.root.meta_data[-1]['event_number'] - in_file_h5.root.meta_data[-2]['event_number']))
            else:
                print '|', os.path.basename(interpreted_file), '|', int(os.path.getsize(interpreted_file) / (1024.0 * 1024.0)), 'Mb |', time.ctime(os.path.getctime(interpreted_file)), '|', n_events, '|', n_bad_events, '|', measurement_time, 's |', n_sr, '|', n_hits, '|'