method that takes in FireEye (FE) alerts and filters FE-tests and False Positives
    :param process_time:
    :param df:
    :return:
485575
def filter_false_positive(df, process_time):
    
    result = []
    track = []
    count = 0
    for o in df['alerts']:
        count += 1
        if 'closedState' in o:
            if o['closedState'] != 'False Positive':
                if 'distinguishers' in o:
                    try:
                        if 'virus' in o['distinguishers']:
                            if o['distinguishers']['virus'] != 'fetestevent':
                                result.append(o)
                            else:
                                track.append(o)  # Track fetestevents that are skipped
                        else:
                            result.append(o)
                    except TypeError:
                        result.append(o)
                else:
                    result.append(o)
            else:
                track.append(o)  # Track false positives that are skipped
        elif 'distinguishers' in o:
            try:
                if 'virus' in o['distinguishers']:
                    if o['distinguishers']['virus'] != 'fetestevent':
                        result.append(o)
                    else:
                        track.append(o)  # Track fetestevents that are skipped
                else:
                    result.append(o)
            except TypeError:
                result.append(o)

    trackfile = open('tracking_fetest_' + process_time + '.txt', 'w')
    numskip = 1
    for item in track:
        trackfile.write("\n\n**** {:d}: Display ID {} ****\n\n{}".format(numskip, item['displayId'], item))
        numskip += 1
    return result
517940
def creation_time(self):
    """"""
    timestamp = self._fsntfs_file_entry.get_creation_time_as_integer()
    return dfdatetime_filetime.Filetime(timestamp=timestamp)
520144
def fit_df(self, dfs, pstate_col=PSTATE_COL):
        
        obs_cols = list(self.emission_name)
        obs = [df[df.columns.difference([pstate_col])][obs_cols].values for df in dfs]
        pstates = [df[pstate_col].values for df in dfs]
        return self.fit(obs, pstates)
1097809
def process_timer(self, key, fields):
        
        try:
            if key not in self.timers:
                self.timers[key] = []
            self.timers[key].append(float(fields[0]))
            if self.stats_seen >= maxint:
                self.logger.info("hit maxint, reset seen counter")
                self.stats_seen = 0
            self.stats_seen += 1
        except Exception as err:
            self.logger.info("error decoding timer event: %s" % err)
            if self.debug:
                print "error decoding timer event: %s" % err
100777
def run(self):
        

        history = pd.DataFrame()

        # get history from csv dir
        if self.backtest and self.backtest_csv:
            kind = "TICK" if self.resolution[-1] in ("S", "K", "V") else "BAR"
            dfs = []
            for symbol in self.symbols:
                file = "%s/%s.%s.csv" % (self.backtest_csv, symbol, kind)
                if not os.path.exists(file):
                    self.log_algo.error(
                        "Can't load data for %s (%s doesn't exist)",
                        symbol, file)
                    sys.exit(0)
                try:
                    df = pd.read_csv(file)
                    if "expiry" not in df.columns:
                        df.loc[:, "expiry"] = nan

                    if not validate_csv_columns(df, kind, raise_errors=False):
                        self.log_algo.error(
                            "%s isn't a QTPyLib-compatible format", file)
                        sys.exit(0)

                    if df['symbol'].values[-1] != symbol:
                        self.log_algo.error(
                            "%s Doesn't content data for %s", file, symbol)
                        sys.exit(0)

                    dfs.append(df)

                except Exception as e:
                    self.log_algo.error(
                        "Error reading data for %s (%s)", symbol, file)
                    sys.exit(0)

            history = prepare_history(
                data=pd.concat(dfs, sort=True),
                resolution=self.resolution,
                tz=self.timezone,
                continuous=self.continuous
            )
            history = history[history.index >= self.backtest_start]

        elif not self.blotter_args["dbskip"] and (
                self.backtest or self.preload):

            start = self.backtest_start if self.backtest else tools.backdate(
                self.preload)
            end = self.backtest_end if self.backtest else None

            history = self.blotter.history(
                symbols=self.symbols,
                start=start,
                end=end,
                resolution=self.resolution,
                tz=self.timezone,
                continuous=self.continuous
            )

            # history needs backfilling?
            # self.blotter.backfilled = True
            if not self.blotter.backfilled:
                # "loan" Blotter our ibConn
                self.blotter.ibConn = self.ibConn

                # call the back fill
                self.blotter.backfill(data=history,
                                      resolution=self.resolution,
                                      start=start, end=end)

                # re-get history from db
                history = self.blotter.history(
                    symbols=self.symbols,
                    start=start,
                    end=end,
                    resolution=self.resolution,
                    tz=self.timezone,
                    continuous=self.continuous
                )

                # take our ibConn back :)
                self.blotter.ibConn = None

        # optimize pandas
        if not history.empty:
            history['symbol'] = history['symbol'].astype('category')
            history['symbol_group'] = history['symbol_group'].astype('category')
            history['asset_class'] = history['asset_class'].astype('category')

        if self.backtest:
            # initiate strategy
            self.on_start()

            # drip history
            drip_handler = self._tick_handler if self.resolution[-1] in (
                "S", "K", "V") else self._bar_handler
            self.blotter.drip(history, drip_handler)

        else:
            # place history self.bars
            self.bars = history

            # add instruments to blotter in case they do not exist
            self.blotter.register(self.instruments)

            # initiate strategy
            self.on_start()

            # listen for RT data
            self.blotter.stream(
                symbols=self.symbols,
                tz=self.timezone,
                quote_handler=self._quote_handler,
                tick_handler=self._tick_handler,
                bar_handler=self._bar_handler,
                book_handler=self._book_handler
            )
931695
def _apply_backwards_compatibility(df):
    
    df.row_count = types.MethodType(lambda self: len(self.index), df)
    df.col_count = types.MethodType(lambda self: len(self.columns), df)
    df.dataframe = df
582428
def init_check_window(self):
        
        self.check_dia = pmag_er_magic_dialogs.ErMagicCheckFrame3(self, 'Check Data',
                                                                  self.WD, self.contribution)
366497
def dataflag(d, data_read):
    

    for flag in d['flaglist']:
        mode, sig, conv = flag
#        resultlist = []
#        with closing(mp.Pool(4, initializer=initreadonly, initargs=(data_read_mem,))) as flagpool:
        for ss in d['spw']:
            chans = n.arange(d['spw_chanr_select'][ss][0], d['spw_chanr_select'][ss][1])
            for pol in range(d['npol']):
                status = rtlib.dataflag(data_read, chans, pol, d, sig, mode, conv)
                logger.info(status)

    # hack to get rid of bad spw/pol combos whacked by rfi
    if 'badspwpol' in d:
        logger.info('Comparing overall power between spw/pol. Removing those with %d times typical value' % d['badspwpol'])
        spwpol = {}
        for spw in d['spw']:
            chans = n.arange(d['spw_chanr_select'][spw][0], d['spw_chanr_select'][spw][1])
            for pol in range(d['npol']):
                spwpol[(spw, pol)] = n.abs(data_read[:,:,chans,pol]).std()
        
        meanstd = n.mean(spwpol.values())
        for (spw,pol) in spwpol:
            if spwpol[(spw, pol)] > d['badspwpol']*meanstd:
                logger.info('Flagging all of (spw %d, pol %d) for excess noise.' % (spw, pol))
                chans = n.arange(d['spw_chanr_select'][spw][0], d['spw_chanr_select'][spw][1])
                data_read[:,:,chans,pol] = 0j
515502
def GetSortedEvents(self, time_range=None):
    
    filter_expression = None
    if time_range:
      filter_expression = []

      if time_range.start_timestamp:
        filter_expression.append(
            '_timestamp >= {0:d}'.format(time_range.start_timestamp))

      if time_range.end_timestamp:
        filter_expression.append(
            '_timestamp <= {0:d}'.format(time_range.end_timestamp))

      filter_expression = ' AND '.join(filter_expression)

    event_generator = self._GetAttributeContainers(
        self._CONTAINER_TYPE_EVENT, filter_expression=filter_expression,
        order_by='_timestamp')

    for event in event_generator:
      if hasattr(event, 'event_data_row_identifier'):
        event_data_identifier = identifiers.SQLTableIdentifier(
            'event_data', event.event_data_row_identifier)
        event.SetEventDataIdentifier(event_data_identifier)

        del event.event_data_row_identifier

      yield event
631984
def wf_alert_severity?(severity)
      return true if %w[INFO SMOKE WARN SEVERE].include?(severity)
      raise Wavefront::Exception::InvalidAlertSeverity
    end
950073
def _monitoring_metrics_list(args, _):
  """"""
  project_id = args['project']
  pattern = args['type'] or '*'
  descriptors = gcm.MetricDescriptors(context=_make_context(project_id))
  dataframe = descriptors.as_dataframe(pattern=pattern)
  return _render_dataframe(dataframe)
1216423
def followup_event_significance(ifo, data_reader, bank,
                                template_id, coinc_times,
                                coinc_threshold=0.005,
                                lookback=150, duration=0.095):
    
    from pycbc.waveform import get_waveform_filter_length_in_time
    tmplt = bank.table[template_id]
    length_in_time = get_waveform_filter_length_in_time(tmplt['approximant'],
                                                        tmplt)

    # calculate onsource time range
    from pycbc.detector import Detector
    onsource_start = -numpy.inf
    onsource_end = numpy.inf
    fdet = Detector(ifo)

    for cifo in coinc_times:
        time = coinc_times[cifo]
        dtravel =  Detector(cifo).light_travel_time_to_detector(fdet)
        if time - dtravel > onsource_start:
            onsource_start = time - dtravel
        if time + dtravel < onsource_end:
            onsource_end = time + dtravel

    # Source must be within this time window to be considered a possible
    # coincidence
    onsource_start -= coinc_threshold
    onsource_end += coinc_threshold

    # Calculate how much time needed to calculate significance
    trim_pad = (data_reader.trim_padding * data_reader.strain.delta_t)
    bdur = int(lookback + 2.0 * trim_pad + length_in_time)
    if bdur > data_reader.strain.duration * .75:
        bdur = data_reader.strain.duration * .75

    # Require all strain be valid within lookback time
    if data_reader.state is not None:
        state_start_time = data_reader.strain.end_time \
                - data_reader.reduced_pad * data_reader.strain.delta_t - bdur
        if not data_reader.state.is_extent_valid(state_start_time, bdur):
            return None, None, None, None

    # We won't require that all DQ checks be valid for now, except at
    # onsource time.
    if data_reader.dq is not None:
        dq_start_time = onsource_start - duration / 2.0
        dq_duration = onsource_end - onsource_start + duration
        if not data_reader.dq.is_extent_valid(dq_start_time, dq_duration):
            return None, None, None, None

    # Calculate SNR time series for this duration
    htilde = bank.get_template(template_id, min_buffer=bdur)
    stilde = data_reader.overwhitened_data(htilde.delta_f)

    sigma2 = htilde.sigmasq(stilde.psd)
    snr, _, norm = matched_filter_core(htilde, stilde, h_norm=sigma2)

    # Find peak in on-source and determine p-value
    onsrc = snr.time_slice(onsource_start, onsource_end)
    peak = onsrc.abs_arg_max()
    peak_time = peak * snr.delta_t + onsrc.start_time
    peak_value = abs(onsrc[peak])

    bstart = float(snr.start_time) + length_in_time + trim_pad
    bkg = abs(snr.time_slice(bstart, onsource_start)).numpy()

    window = int((onsource_end - onsource_start) * snr.sample_rate)
    nsamples = int(len(bkg) / window)

    peaks = bkg[:nsamples*window].reshape(nsamples, window).max(axis=1)
    pvalue = (1 + (peaks >= peak_value).sum()) / float(1 + nsamples)

    # Return recentered source SNR for bayestar, along with p-value, and trig
    baysnr = snr.time_slice(peak_time - duration / 2.0,
                            peak_time + duration / 2.0)

    logging.info('Adding %s to candidate, pvalue %s, %s samples', ifo,
                 pvalue, nsamples)

    return baysnr * norm, peak_time, pvalue, sigma2
1124760
def check_frequencies(
    feed: "Feed", *, as_df: bool = False, include_warnings: bool = False
) -> List:
    
    table = "frequencies"
    problems = []

    # Preliminary checks
    if feed.frequencies is None:
        return problems

    f = feed.frequencies.copy()
    problems = check_for_required_columns(problems, table, f)
    if problems:
        return format_problems(problems, as_df=as_df)

    if include_warnings:
        problems = check_for_invalid_columns(problems, table, f)

    # Check trip_id
    problems = check_column_linked_id(
        problems, table, f, "trip_id", feed.trips
    )

    # Check start_time and end_time
    time_cols = ["start_time", "end_time"]
    for col in time_cols:
        problems = check_column(problems, table, f, col, valid_time)

    for col in time_cols:
        f[col] = f[col].map(hp.timestr_to_seconds)

    # Start_time should be earlier than end_time
    cond = f["start_time"] >= f["end_time"]
    problems = check_table(
        problems, table, f, cond, "start_time not earlier than end_time"
    )

    # Headway periods should not overlap
    f = f.sort_values(["trip_id", "start_time"])
    for __, group in f.groupby("trip_id"):
        a = group["start_time"].values
        b = group["end_time"].values
        indices = np.flatnonzero(a[1:] < b[:-1]).tolist()
        if indices:
            problems.append(
                [
                    "error",
                    "Headway periods for the same trip overlap",
                    table,
                    indices,
                ]
            )

    # Check headway_secs
    v = lambda x: x >= 0
    problems = check_column(problems, table, f, "headway_secs", v)

    # Check exact_times
    v = lambda x: x in range(2)
    problems = check_column(
        problems, table, f, "exact_times", v, column_required=False
    )

    return format_problems(problems, as_df=as_df)
1045350
def change_time(self):
    """"""
    timestamp = self._fsapfs_file_entry.get_inode_change_time_as_integer()
    return dfdatetime_apfs_time.APFSTime(timestamp=timestamp)
1039649
def _GetDateTime(self, filetime):
    
    if filetime == 0:
      return dfdatetime_semantic_time.SemanticTime('Not set')

    return dfdatetime_filetime.Filetime(timestamp=filetime)
637880
def correlate_alert(self, alert, history):
        
        query = {
            'environment': alert.environment,
            'resource': alert.resource,
            '$or': [
                {
                    'event': alert.event,
                    'severity': {'$ne': alert.severity}
                },
                {
                    'event': {'$ne': alert.event},
                    'correlate': alert.event
                }],
            'customer': alert.customer
        }

        update = {
            '$set': {
                'event': alert.event,
                'severity': alert.severity,
                'status': alert.status,
                'service': alert.service,
                'value': alert.value,
                'text': alert.text,
                'createTime': alert.create_time,
                'timeout': alert.timeout,
                'rawData': alert.raw_data,
                'duplicateCount': alert.duplicate_count,
                'repeat': alert.repeat,
                'previousSeverity': alert.previous_severity,
                'trendIndication': alert.trend_indication,
                'receiveTime': alert.receive_time,
                'lastReceiveId': alert.last_receive_id,
                'lastReceiveTime': alert.last_receive_time
            },
            '$addToSet': {'tags': {'$each': alert.tags}},
            '$push': {
                'history': {
                    '$each': [h.serialize for h in history],
                    '$slice': -abs(current_app.config['HISTORY_LIMIT'])
                }
            }
        }

        # only update those attributes that are specifically defined
        attributes = {'attributes.' + k: v for k, v in alert.attributes.items()}
        update['$set'].update(attributes)

        if alert.update_time:
            update['$set']['updateTime'] = alert.update_time

        return self.get_db().alerts.find_one_and_update(
            query,
            update=update,
            return_document=ReturnDocument.AFTER
        )
209768
def check_time_event(oqparam, occupancy_periods):
    
    time_event = oqparam.time_event
    if time_event and time_event not in occupancy_periods:
        raise ValueError(
            'time_event is %s in %s, but the exposure contains %s' %
            (time_event, oqparam.inputs['job_ini'],
             ', '.join(occupancy_periods)))