Create and run collector process for report data.
195962
def create_and_run_collector(document, options):
        """"""
        collector = None
        if not options.report == 'off':
            collector = Collector()
            collector.store.configure(document)
            Event.configure(collector_queue=collector.queue)
            collector.start()
        return collector
880451
def run(self):
        """"""
        while True:
            try:
                self.perform_iteration()
            except Exception:
                traceback.print_exc()
                pass

            time.sleep(ray_constants.REPORTER_UPDATE_INTERVAL_MS / 1000)
254
def run_checks(collector):
    """"""
    artifact = collector.configuration["dashmat"].artifact
    chosen = artifact
    if chosen in (None, "", NotSpecified):
        chosen = None

    dashmat = collector.configuration["dashmat"]
    modules = collector.configuration["__active_modules__"]
    config_root = collector.configuration["config_root"]
    module_options = collector.configuration["modules"]

    datastore = JsonDataStore(os.path.join(config_root, "data.json"))
    if dashmat.redis_host:
        datastore = RedisDataStore(redis.Redis(dashmat.redis_host))

    scheduler = Scheduler(datastore)

    for name, module in modules.items():
        if chosen is None or name == chosen:
            server = module.make_server(module_options[name].server_options)
            scheduler.register(module, server, name)

    scheduler.twitch(force=True)
341260
def _run_custom_data_engine(self, start_id_obj, end_id_obj, start_timeperiod, end_timeperiod):
        
        collection_name = context.process_context[self.process_name].source
        iteration = 0
        while True:
            cursor = self.ds.cursor_fine(collection_name,
                                         start_id_obj,
                                         end_id_obj,
                                         iteration,
                                         start_timeperiod,
                                         end_timeperiod)

            if iteration == 0 and cursor.count(with_limit_and_skip=True) == 0:
                msg = 'No entries in {0} at range [{1} : {2}]'.format(collection_name, start_id_obj, end_id_obj)
                self.logger.warning(msg)
                break

            start_id_obj = None
            for document in cursor:
                start_id_obj = document['_id']
                self._process_single_document(document)
                self.performance_tracker.increment_success()
            if start_id_obj is None:
                break
            iteration += 1

        self._cursor_exploited()
        msg = 'Cursor exploited after {0} iterations'.format(iteration)
        self.logger.info(msg)
558665
def create_report(self):
        """"""
        data = self.create_data()
        try:
            json_string = json.dumps(data)
        except UnicodeDecodeError as e:
            log.error('ERROR: While preparing JSON:', exc_info=e)
            self.debug_bad_encoding(data)
            raise

        log_string = re.sub(r'"repo_token": "(.+?)"',
                            '"repo_token": "[secure]"', json_string)
        log.debug(log_string)
        log.debug('==\nReporting %s files\n==\n', len(data['source_files']))
        for source_file in data['source_files']:
            log.debug('%s - %s/%s', source_file['name'],
                      sum(filter(None, source_file['coverage'])),
                      len(source_file['coverage']))
        return json_string
261863
def run(self):
        """"""
        (task_id, tasks) = self.server.get_task()
        self.task_store.from_dict(tasks)
        for (index, task) in self.task_store:
            result = self.compute(index, task)
            self.results.append(result)
        self.server.task_done((task_id, self.results))
1281795
def _process_data(
        self,
        data,
        collect_aggregates_only,
        process_events,
        url=None,
        collect_status_metrics=False,
        collect_status_metrics_by_host=False,
        tag_service_check_by_host=False,
        services_incl_filter=None,
        services_excl_filter=None,
        collate_status_tags_per_host=False,
        count_status_by_service=True,
        custom_tags=None,
        tags_regex=None,
        active_tag=None,
        enable_service_check=False,
    ):
        '''  '''

        # Split the first line into an index of fields
        # The line looks like (broken up onto multiple lines)
        # "# pxname,svname,qcur,qmax,scur,smax,slim,
        # stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,
        # wredis,status,weight,act,bck,chkfail,chkdown,lastchg,
        # downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,
        # type,rate,rate_lim,rate_max,"
        fields = []
        for f in data[0].split(','):
            if f:
                f = f.replace('# ', '')
                fields.append(f.strip())

        self.hosts_statuses = defaultdict(int)

        back_or_front = None

        # Sanitize CSV, handle line breaks
        data = self._sanitize_lines(data)
        custom_tags = [] if custom_tags is None else custom_tags
        active_tag = [] if active_tag is None else active_tag

        # First initialize here so that it is defined whether or not we enter the for loop
        line_tags = list(custom_tags)

        # Skip the first line, go backwards to set back_or_front
        for line in data[:0:-1]:
            if not line.strip():
                continue

            # Store each line's values in a dictionary
            data_dict = self._line_to_dict(fields, line)

            if self._is_aggregate(data_dict):
                back_or_front = data_dict['svname']

            self._update_data_dict(data_dict, back_or_front)

            self._update_hosts_statuses_if_needed(
                collect_status_metrics, collect_status_metrics_by_host, data_dict, self.hosts_statuses
            )

            # Clone the list to avoid extending the original
            # which would carry over previous iteration tags
            line_tags = list(custom_tags)

            regex_tags = self._tag_from_regex(tags_regex, data_dict['pxname'])
            if regex_tags:
                line_tags.extend(regex_tags)

            if self._should_process(data_dict, collect_aggregates_only):
                # update status
                # Send the list of data to the metric and event callbacks
                self._process_metrics(
                    data_dict,
                    url,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                    active_tag=active_tag,
                )
            if process_events:
                self._process_event(
                    data_dict,
                    url,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                )
            if enable_service_check:
                self._process_service_check(
                    data_dict,
                    url,
                    tag_by_host=tag_service_check_by_host,
                    services_incl_filter=services_incl_filter,
                    services_excl_filter=services_excl_filter,
                    custom_tags=line_tags,
                )

        if collect_status_metrics:
            self._process_status_metric(
                self.hosts_statuses,
                collect_status_metrics_by_host,
                services_incl_filter=services_incl_filter,
                services_excl_filter=services_excl_filter,
                collate_status_tags_per_host=collate_status_tags_per_host,
                count_status_by_service=count_status_by_service,
                custom_tags=line_tags,
                active_tag=active_tag,
            )

            self._process_backend_hosts_metric(
                self.hosts_statuses,
                services_incl_filter=services_incl_filter,
                services_excl_filter=services_excl_filter,
                custom_tags=line_tags,
                active_tag=active_tag,
            )

        return data
1061125
def _create_process_thread(self):
        
        thread = threading.Thread(target=self._process_data_events)
        thread.setDaemon(True)
        thread.start()
35319
def run(self):
        """"""
        print('---- BUILDING ----')
        _build.run(self)

        # build documentation
        print('---- BUILDING DOCS ----')
        docdir = os.path.join(self.build_lib, 'pyshtools', 'doc')
        self.mkpath(docdir)
        doc_builder = os.path.join(self.build_lib, 'pyshtools', 'make_docs.py')
        doc_source = '.'
        check_call([sys.executable, doc_builder, doc_source, self.build_lib])

        print('---- ALL DONE ----')
606066
def main(folder):
    """"""
    raw_datapath, outputpath, p_queue = get_parameters(folder)
    create_preprocessed_dataset(raw_datapath, outputpath, p_queue)
    utils.create_run_logfile(folder)
222088
def process(ctx, dataset, kwargs):
    ""

    kwargs = parse_kwargs(kwargs)
    data(dataset, **ctx.obj).process(**kwargs)
1368106
def create(self):
        
        if self._track is None:
            self._track = self.db[self.tracking_collection_name]
471208
def process(self):
        """"""
        temp_filename = self.create_file_for(self.config.script)

        if len(self.config.title) > 0:
            self.logger.info(render(self.config.title, model=self.config.model, env=self.env,
                                    item=self.config.item, variables=self.config.variables))

        if temp_filename is not None:
            try:
                for line in self.process_file(temp_filename):
                    yield line
            finally:
                # removing script
                os.remove(temp_filename)

        if not self.config.internal:
            if self.exit_code == 0:
                self.event.succeeded()
            else:
                self.event.failed(exit_code=self.exit_code)
102295
private void run(ReportOperation operation) throws ReportException {
        Set<String> executedPlugins = new HashSet<>();
        run(selectedReportPlugins, operation, executedPlugins);
        run(defaultReportPlugins, operation, executedPlugins);
    }
792555
def run(self):
        ""
        # load files from source directory
        files = self.get_files()

        # loop through each middleware
        for func in self.middleware:

            # call each one, ignoring return value
            func(files, self)

        # store and return the result
        self.files.update(files)
        return files
1253715
def _create(self, postData) :
        """"""
        if self.infos is None :
            r = self.connection.session.post(self.indexesURL, params = {"collection" : self.collection.name}, data = json.dumps(postData, default=str))
            data = r.json()
            if (r.status_code >= 400) or data['error'] :
                raise CreationError(data['errorMessage'], data)
            self.infos = data
827159
def run(self):
        """"""
        env = dict(os.environ)
        file_path = self.file.real_path
        path_folders = self.pycore.project.get_source_folders() + \
            self.pycore.project.get_python_path_folders()
        env['PYTHONPATH'] = os.pathsep.join(folder.real_path
                                            for folder in path_folders)
        runmod_path = self.pycore.project.find_module('rope.base.oi.runmod').real_path
        self.receiver = None
        self._init_data_receiving()
        send_info = '-'
        if self.receiver:
            send_info = self.receiver.get_send_info()
        args = [sys.executable, runmod_path, send_info,
                self.pycore.project.address, self.file.real_path]
        if self.analyze_data is None:
            del args[1:4]
        if self.args is not None:
            args.extend(self.args)
        self.process = subprocess.Popen(
            executable=sys.executable, args=args, env=env,
            cwd=os.path.split(file_path)[0], stdin=self.stdin,
            stdout=self.stdout, stderr=self.stdout, close_fds=os.name != 'nt')