Spin up the multiprocess event returner
431281
def run(self):
        '''
        
        '''
        salt.utils.process.appendproctitle(self.__class__.__name__)
        self.event = get_event('master', opts=self.opts, listen=True)
        events = self.event.iter_events(full=True)
        self.event.fire_event({}, 'salt/event_listen/start')
        try:
            for event in events:
                if event['tag'] == 'salt/event/exit':
                    self.stop = True
                if self._filter(event):
                    self.event_queue.append(event)
                if len(self.event_queue) >= self.event_return_queue:
                    self.flush_events()
                if self.stop:
                    break
        finally:  # flush all we have at this moment
            if self.event_queue:
                self.flush_events()
434538
def run_in_subprocess(func, *args, **kwargs):
    """"""
    from multiprocessing import Process
    thread = Process(target=func, args=args, kwargs=kwargs)
    thread.daemon = True
    thread.start()
    return thread
1012435
def eventloop(cls, custom_actions=[]):
        
        iiter = cls([None], custom_actions=custom_actions, verbose=False)
        print('[IITER] Begining interactive main loop')
        for _ in iiter:
            pass
        return iiter
907405
def Main():
  """"""
  multiprocessing.freeze_support()

  tool = psteal_tool.PstealTool()
  if not tool.ParseArguments():
    return False

  have_list_option = False

  if tool.list_timezones:
    tool.ListTimeZones()
    have_list_option = True

  if tool.list_output_modules:
    tool.ListOutputModules()
    have_list_option = True

  if tool.list_timezones:
    tool.ListTimeZones()
    have_list_option = True

  if tool.list_parsers_and_plugins:
    tool.ListParsersAndPlugins()
    have_list_option = True

  if tool.list_hashers:
    tool.ListHashers()
    have_list_option = True

  if tool.list_language_identifiers:
    tool.ListLanguageIdentifiers()
    have_list_option = True

  if have_list_option:
    return True

  try:
    tool.ExtractEventsFromSources()
    tool.AnalyzeEvents()

  except (KeyboardInterrupt, errors.UserAbort):
    logging.warning('Aborted by user.')
    return False

  except errors.SourceScannerError as exception:
    logging.warning(exception)
    return False

  return True
815984
def multi_p_run(tot_num, _func, worker, params, n_process):
    
    from multiprocessing import Process, Queue
    out_q = Queue()
    procs = []

    split_num = split_seq(list(range(0, tot_num)), n_process)

    print(tot_num, ">>", split_num)

    split_len = len(split_num)
    if n_process > split_len:
        n_process = split_len

    for i in range(n_process):
        _p = Process(target=_func,
                     args=(worker, split_num[i][0], split_num[i][1],
                           params, out_q))
        _p.daemon = True
        procs.append(_p)
        _p.start()

    try:
        result = []
        for i in range(n_process):
            result.append(out_q.get())
        for i in procs:
            i.join()
    except KeyboardInterrupt:
        print('Killing all the children in the pool.')
        for i in procs:
            i.terminate()
            i.join()
        return -1

    while not out_q.empty():
        print(out_q.get(block=False))

    return result
1345226
def retry(self, func, partition_id, retry_message, final_failure_message, max_retries, host_id):
        
        loop = asyncio.new_event_loop()
        loop.run_until_complete(self.retry_async(func, partition_id, retry_message,
                                                 final_failure_message, max_retries, host_id))
412004
def multithreader(args, paths):
    """"""

    def shellprocess(path):
        """Return a ready-to-use subprocess."""
        import subprocess
        return subprocess.Popen(args + [path],
                                stderr=subprocess.DEVNULL,
                                stdout=subprocess.DEVNULL)

    processes = [shellprocess(path) for path in paths]

    for process in processes:
        process.wait()
474976
async def trigger(self, event, kwargs):
        
        await self._queue.put((event, kwargs))
        self._resume_processing.set()
501475
def drain_events(self, allowed_methods=None, timeout=None):
        """"""
        return self.wait_multi(self.channels.values(), timeout=timeout)
610592
def main(event_loop=None):
    
    context, credentials = get_context_from_cmdln(sys.argv[1:])
    log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format()))
    cleanup(context)
    context.event_loop = event_loop or asyncio.get_event_loop()

    done = False

    async def _handle_sigterm():
        log.info("SIGTERM received; shutting down")
        nonlocal done
        done = True
        if context.running_tasks is not None:
            await context.running_tasks.cancel()

    context.event_loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))

    while not done:
        try:
            context.event_loop.run_until_complete(async_main(context, credentials))
        except Exception:
            log.critical("Fatal exception", exc_info=1)
            raise
3626
def handle_func(self, multiprocessing_enabled, func, data):
        '''
        
        '''
        if salt.utils.platform.is_windows() \
                or self.opts.get('transport') == 'zeromq':
            # Since function references can't be pickled and pickling
            # is required when spawning new processes on Windows, regenerate
            # the functions and returners.
            # This also needed for ZeroMQ transport to reset all functions
            # context data that could keep paretns connections. ZeroMQ will
            # hang on polling parents connections from the child process.
            if self.opts['__role'] == 'master':
                self.functions = salt.loader.runner(self.opts, utils=self.utils)
            else:
                self.functions = salt.loader.minion_mods(self.opts, proxy=self.proxy, utils=self.utils)
            self.returners = salt.loader.returners(self.opts, self.functions, proxy=self.proxy)
        ret = {'id': self.opts.get('id', 'master'),
               'fun': func,
               'fun_args': [],
               'schedule': data['name'],
               'jid': salt.utils.jid.gen_jid(self.opts)}

        if 'metadata' in data:
            if isinstance(data['metadata'], dict):
                ret['metadata'] = data['metadata']
                ret['metadata']['_TOS'] = self.time_offset
                ret['metadata']['_TS'] = time.ctime()
                ret['metadata']['_TT'] = time.strftime('%Y %B %d %a %H %m', time.gmtime())
            else:
                log.warning('schedule: The metadata parameter must be '
                            'specified as a dictionary.  Ignoring.')

        if multiprocessing_enabled:
            # We just want to modify the process name if we're on a different process
            salt.utils.process.appendproctitle('{0} {1}'.format(self.__class__.__name__, ret['jid']))
        data_returner = data.get('returner', None)

        if not self.standalone:
            proc_fn = os.path.join(
                salt.minion.get_proc_dir(self.opts['cachedir']),
                ret['jid']
            )

        if multiprocessing_enabled and not salt.utils.platform.is_windows():
            # Reconfigure multiprocessing logging after daemonizing
            log_setup.setup_multiprocessing_logging()

        if multiprocessing_enabled:
            # Don't *BEFORE* to go into try to don't let it triple execute the finally section.
            salt.utils.process.daemonize_if(self.opts)

        # TODO: Make it readable! Splt to funcs, remove nested try-except-finally sections.
        try:

            minion_blackout_violation = False
            if self.opts.get('pillar', {}).get('minion_blackout', False):
                whitelist = self.opts.get('pillar', {}).get('minion_blackout_whitelist', [])
                # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist
                if func != 'saltutil.refresh_pillar' and func not in whitelist:
                    minion_blackout_violation = True
            elif self.opts.get('grains', {}).get('minion_blackout', False):
                whitelist = self.opts.get('grains', {}).get('minion_blackout_whitelist', [])
                if func != 'saltutil.refresh_pillar' and func not in whitelist:
                    minion_blackout_violation = True
            if minion_blackout_violation:
                raise SaltInvocationError('Minion in blackout mode. Set \'minion_blackout\' '
                                          'to False in pillar or grains to resume operations. Only '
                                          'saltutil.refresh_pillar allowed in blackout mode.')

            ret['pid'] = os.getpid()

            if not self.standalone:
                if 'jid_include' not in data or data['jid_include']:
                    log.debug(
                        'schedule.handle_func: adding this job to the '
                        'jobcache with data %s', ret
                    )
                    # write this to /var/cache/salt/minion/proc
                    with salt.utils.files.fopen(proc_fn, 'w+b') as fp_:
                        fp_.write(salt.payload.Serial(self.opts).dumps(ret))

            args = tuple()
            if 'args' in data:
                args = data['args']
                ret['fun_args'].extend(data['args'])

            kwargs = {}
            if 'kwargs' in data:
                kwargs = data['kwargs']
                ret['fun_args'].append(copy.deepcopy(kwargs))

            if func not in self.functions:
                ret['return'] = self.functions.missing_fun_string(func)
                salt.utils.error.raise_error(
                    message=self.functions.missing_fun_string(func))

            # if the func support **kwargs, lets pack in the pub data we have
            # TODO: pack the *same* pub data as a minion?
            argspec = salt.utils.args.get_function_argspec(self.functions[func])
            if argspec.keywords:
                # this function accepts **kwargs, pack in the publish data
                for key, val in six.iteritems(ret):
                    if key is not 'kwargs':
                        kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val)

            # Only include these when running runner modules
            if self.opts['__role'] == 'master':
                jid = salt.utils.jid.gen_jid(self.opts)
                tag = salt.utils.event.tagify(jid, prefix='salt/scheduler/')

                event = salt.utils.event.get_event(
                        self.opts['__role'],
                        self.opts['sock_dir'],
                        self.opts['transport'],
                        opts=self.opts,
                        listen=False)

                namespaced_event = salt.utils.event.NamespacedEvent(
                    event,
                    tag,
                    print_func=None
                )

                func_globals = {
                    '__jid__': jid,
                    '__user__': salt.utils.user.get_user(),
                    '__tag__': tag,
                    '__jid_event__': weakref.proxy(namespaced_event),
                }
                self_functions = copy.copy(self.functions)
                salt.utils.lazy.verify_fun(self_functions, func)

                # Inject some useful globals to *all* the function's global
                # namespace only once per module-- not per func
                completed_funcs = []

                for mod_name in six.iterkeys(self_functions):
                    if '.' not in mod_name:
                        continue
                    mod, _ = mod_name.split('.', 1)
                    if mod in completed_funcs:
                        continue
                    completed_funcs.append(mod)
                    for global_key, value in six.iteritems(func_globals):
                        self.functions[mod_name].__globals__[global_key] = value

            self.functions.pack['__context__']['retcode'] = 0

            ret['return'] = self.functions[func](*args, **kwargs)

            if not self.standalone:
                # runners do not provide retcode
                if 'retcode' in self.functions.pack['__context__']:
                    ret['retcode'] = self.functions.pack['__context__']['retcode']

                ret['success'] = True

                if data_returner or self.schedule_returner:
                    if 'return_config' in data:
                        ret['ret_config'] = data['return_config']
                    if 'return_kwargs' in data:
                        ret['ret_kwargs'] = data['return_kwargs']
                    rets = []
                    for returner in [data_returner, self.schedule_returner]:
                        if isinstance(returner, six.string_types):
                            rets.append(returner)
                        elif isinstance(returner, list):
                            rets.extend(returner)
                    # simple de-duplication with order retained
                    for returner in OrderedDict.fromkeys(rets):
                        ret_str = '{0}.returner'.format(returner)
                        if ret_str in self.returners:
                            self.returners[ret_str](ret)
                        else:
                            log.info(
                                'Job %s using invalid returner: %s. Ignoring.',
                                func, returner
                            )

        except Exception:
            log.exception('Unhandled exception running %s', ret['fun'])
            # Although catch-all exception handlers are bad, the exception here
            # is to let the exception bubble up to the top of the thread context,
            # where the thread will die silently, which is worse.
            if 'return' not in ret:
                ret['return'] = "Unhandled exception running {0}".format(ret['fun'])
            ret['success'] = False
            ret['retcode'] = 254
        finally:
            # Only attempt to return data to the master if the scheduled job is running
            # on a master itself or a minion.
            if '__role' in self.opts and self.opts['__role'] in ('master', 'minion'):
                # The 'return_job' option is enabled by default even if not set
                if 'return_job' in data and not data['return_job']:
                    pass
                else:
                    # Send back to master so the job is included in the job list
                    mret = ret.copy()
                    # No returners defined, so we're only sending back to the master
                    if not data_returner and not self.schedule_returner:
                        mret['jid'] = 'req'
                        if data.get('return_job') == 'nocache':
                            # overwrite 'req' to signal to master that
                            # this job shouldn't be stored
                            mret['jid'] = 'nocache'
                    load = {'cmd': '_return', 'id': self.opts['id']}
                    for key, value in six.iteritems(mret):
                        load[key] = value

                    if '__role' in self.opts and self.opts['__role'] == 'minion':
                        event = salt.utils.event.get_event('minion',
                                                           opts=self.opts,
                                                           listen=False)
                    elif '__role' in self.opts and self.opts['__role'] == 'master':
                        event = salt.utils.event.get_master_event(self.opts,
                                                                  self.opts['sock_dir'])
                    try:
                        event.fire_event(load, '__schedule_return')
                    except Exception as exc:
                        log.exception('Unhandled exception firing __schedule_return event')

            if not self.standalone:
                log.debug('schedule.handle_func: Removing %s', proc_fn)

                try:
                    os.unlink(proc_fn)
                except OSError as exc:
                    if exc.errno == errno.EEXIST or exc.errno == errno.ENOENT:
                        # EEXIST and ENOENT are OK because the file is gone and that's what
                        # we wanted
                        pass
                    else:
                        log.error("Failed to delete '%s': %s", proc_fn, exc.errno)
                        # Otherwise, failing to delete this file is not something
                        # we can cleanly handle.
                        raise
                finally:
                    if multiprocessing_enabled:
                        # Let's make sure we exit the process!
                        sys.exit(salt.defaults.exitcodes.EX_GENERIC)
312308
def from_events(self, instance, ev_args, ctx):
        
        obj = yield from self._process(instance, ev_args, ctx)
        self.__set__(instance, obj)
        return obj
59596
def main():
    
    args = parse_args()
    logging.basicConfig(level=LOG_LEVELS[min(
        args.verbosity, len(LOG_LEVELS) - 1)])

    loop = asyncio.get_event_loop()
    loop.run_until_complete(_run(args))
1027276
def eventloop(self, *args, **kwargs):
        

        # Setting status
        status = None

        # Starting the clock
        start = time.time()

        first_loop = True
        # loop running target, maybe more than once
        while not self.exit.is_set():

            if first_loop:
                first_loop = False
                # signalling startup only the first time, just after having check for exit request.
                # We need to return control before starting, but after entering context...
                self.started.set()
                # TODO : check if better outside of loop maybe ??
                # It will change semantics, but might be more intuitive...

            # time is ticking
            # TODO : move this out of here. this class should require only generic interface to any method.
            now = time.time()
            timedelta = now - start
            start = now

            # replacing the original Process.run() call, passing arguments to our target
            if self._target:
                # bwcompat
                kwargs['timedelta'] = timedelta

                # TODO : use return code to determine when/how we need to run this the next time...
                # Also we need to keep the exit status to be able to call external process as an update...

                logging.debug(
                    "[{self.name}] calling {self._target.__name__} with args {args} and kwargs {kwargs}...".format(
                        **locals()))
                status = self._target(*args, **kwargs)

            if status is not None:
                break

        if self.started.is_set() and status is None and self.exit.is_set():
            # in the not so special case where we started, we didnt get exit code and we exited,
            # this is expected as a normal result and we set an exitcode here of 0
            # As 0 is the conventional success for unix process successful run
            status = 0

        return status
1073686
def process_iter(proc, cmd=""):
    
    try:
        for l in proc.stdout:
            yield l
    finally:
        if proc.poll() is None:
            # there was an exception
            return
        else:
            proc.wait()
            if proc.returncode not in (0, None, signal.SIGPIPE, signal.SIGPIPE + 128):
                sys.stderr.write("cmd was:%s\n" % cmd)
                sys.stderr.write("return code was:%s\n" % proc.returncode)
                raise ProcessException(cmd)
156961
def event_emitter(cls_=None, *, events=('*', )):
    

    # create a dictionary from items in the 'events' parameter and with empty
    # lists as values
    event_dict = dict.fromkeys(events, [])

    # if '*' was in the events tuple - then pop it out of the event_dict
    # and store the fact that we may allow any event name to be added to the
    # event emitter.
    allow_any_eventname = event_dict.pop('*', False) == []

    def _event_emitter(cls):

        def on(self, name, callback):
            """
            Add a callback to the event named 'name'. Returns the object for
            chained 'on' calls.
            """
            if not (callable(callback) or isawaitable(callback)):
                raise ValueError("Callback not callable: %r" % callback)

            try:
                event_dict[name].append(callback)
            except KeyError:
                if allow_any_eventname:
                    event_dict[name] = [callback]
                else:
                    msg = "Event Emitter has no event {!r}".format(name)
                    raise KeyError(msg)

            return self

        async def emit(self, name):
            """
            Coroutine which executes each of the callbacks added to the event
            identified by 'name'
            """
            for cb in event_dict[name]:
                if isawaitable(cb):
                    await cb
                else:
                    cb()

        cls.on = on
        cls.emit = emit

        return cls

    if cls_ is None:
        return _event_emitter
    else:
        return _event_emitter(cls_)
1085239
def supervisor_events(stdin, stdout):
    

    while True:
        stdout.write('READY\n')
        stdout.flush()

        line = stdin.readline()
        headers = get_headers(line)

        payload = stdin.read(int(headers['len']))
        event_headers, event_data = eventdata(payload)

        yield event_headers, event_data

        stdout.write('RESULT 2\nOK')
        stdout.flush()