Receive a syndic minion return and format it to look like returns from
        individual minions.
339508
def _syndic_return(self, load):
        '''
        
        '''
        # Verify the load
        if any(key not in load for key in ('return', 'jid', 'id')):
            return None
        # if we have a load, save it
        if 'load' in load:
            fstr = '{0}.save_load'.format(self.opts['master_job_cache'])
            self.mminion.returners[fstr](load['jid'], load['load'])

        # Format individual return loads
        for key, item in six.iteritems(load['return']):
            ret = {'jid': load['jid'],
                   'id': key,
                   'return': item}
            if 'out' in load:
                ret['out'] = load['out']
            self._return(ret)
862995
def returner(ret):
    '''
    
    '''
    write_profile = __opts__.get('etcd.returner_write_profile')
    if write_profile:
        ttl = __opts__.get(write_profile, {}).get('etcd.ttl')
    else:
        ttl = __opts__.get('etcd.ttl')

    client, path = _get_conn(__opts__, write_profile)
    # Make a note of this minion for the external job cache
    client.set(
        '/'.join((path, 'minions', ret['id'])),
        ret['jid'],
        ttl=ttl,
    )

    for field in ret:
        # Not using os.path.join because we're not dealing with file paths
        dest = '/'.join((
            path,
            'jobs',
            ret['jid'],
            ret['id'],
            field
        ))
        client.set(dest, salt.utils.json.dumps(ret[field]), ttl=ttl)
1179555
def run(self):
        '''
        
        '''
        args = [[],
                self.opts['fun'],
                self.opts['arg'],
                self.opts['timeout'],
                'list',
                ]
        bnum = self.get_bnum()
        # No targets to run
        if not self.minions:
            return
        to_run = copy.deepcopy(self.minions)
        active = []
        ret = {}
        iters = []
        # wait the specified time before decide a job is actually done
        bwait = self.opts.get('batch_wait', 0)
        wait = []

        if self.options:
            show_jid = self.options.show_jid
            show_verbose = self.options.verbose
        else:
            show_jid = False
            show_verbose = False

        # the minion tracker keeps track of responses and iterators
        # - it removes finished iterators from iters[]
        # - if a previously detected minion does not respond, its
        #   added with an empty answer to ret{} once the timeout is reached
        # - unresponsive minions are removed from active[] to make
        #   sure that the main while loop finishes even with unresp minions
        minion_tracker = {}

        if not self.quiet:
            # We already know some minions didn't respond to the ping, so inform
            # the user we won't be attempting to run a job on them
            for down_minion in self.down_minions:
                salt.utils.stringutils.print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion))

        # Iterate while we still have things to execute
        while len(ret) < len(self.minions):
            next_ = []
            if bwait and wait:
                self.__update_wait(wait)
            if len(to_run) <= bnum - len(wait) and not active:
                # last bit of them, add them all to next iterator
                while to_run:
                    next_.append(to_run.pop())
            else:
                for i in range(bnum - len(active) - len(wait)):
                    if to_run:
                        minion_id = to_run.pop()
                        if isinstance(minion_id, dict):
                            next_.append(minion_id.keys()[0])
                        else:
                            next_.append(minion_id)

            active += next_
            args[0] = next_

            if next_:
                if not self.quiet:
                    salt.utils.stringutils.print_cli('\nExecuting run on {0}\n'.format(sorted(next_)))
                # create a new iterator for this batch of minions
                new_iter = self.local.cmd_iter_no_block(
                                *args,
                                raw=self.opts.get('raw', False),
                                ret=self.opts.get('return', ''),
                                show_jid=show_jid,
                                verbose=show_verbose,
                                gather_job_timeout=self.opts['gather_job_timeout'],
                                **self.eauth)
                # add it to our iterators and to the minion_tracker
                iters.append(new_iter)
                minion_tracker[new_iter] = {}
                # every iterator added is 'active' and has its set of minions
                minion_tracker[new_iter]['minions'] = next_
                minion_tracker[new_iter]['active'] = True

            else:
                time.sleep(0.02)
            parts = {}

            # see if we found more minions
            for ping_ret in self.ping_gen:
                if ping_ret is None:
                    break
                m = next(six.iterkeys(ping_ret))
                if m not in self.minions:
                    self.minions.append(m)
                    to_run.append(m)

            for queue in iters:
                try:
                    # Gather returns until we get to the bottom
                    ncnt = 0
                    while True:
                        part = next(queue)
                        if part is None:
                            time.sleep(0.01)
                            ncnt += 1
                            if ncnt > 5:
                                break
                            continue
                        if self.opts.get('raw'):
                            parts.update({part['data']['id']: part})
                            if part['data']['id'] in minion_tracker[queue]['minions']:
                                minion_tracker[queue]['minions'].remove(part['data']['id'])
                            else:
                                salt.utils.stringutils.print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id']))
                        else:
                            parts.update(part)
                            for id in part:
                                if id in minion_tracker[queue]['minions']:
                                    minion_tracker[queue]['minions'].remove(id)
                                else:
                                    salt.utils.stringutils.print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id))
                except StopIteration:
                    # if a iterator is done:
                    # - set it to inactive
                    # - add minions that have not responded to parts{}

                    # check if the tracker contains the iterator
                    if queue in minion_tracker:
                        minion_tracker[queue]['active'] = False

                        # add all minions that belong to this iterator and
                        # that have not responded to parts{} with an empty response
                        for minion in minion_tracker[queue]['minions']:
                            if minion not in parts:
                                parts[minion] = {}
                                parts[minion]['ret'] = {}

            for minion, data in six.iteritems(parts):
                if minion in active:
                    active.remove(minion)
                    if bwait:
                        wait.append(datetime.now() + timedelta(seconds=bwait))
                # Munge retcode into return data
                failhard = False
                if 'retcode' in data and isinstance(data['ret'], dict) and 'retcode' not in data['ret']:
                    data['ret']['retcode'] = data['retcode']
                    if self.opts.get('failhard') and data['ret']['retcode'] > 0:
                        failhard = True

                if self.opts.get('raw'):
                    ret[minion] = data
                    yield data
                else:
                    ret[minion] = data['ret']
                    yield {minion: data['ret']}
                if not self.quiet:
                    ret[minion] = data['ret']
                    data[minion] = data.pop('ret')
                    if 'out' in data:
                        out = data.pop('out')
                    else:
                        out = None
                    salt.output.display_output(
                            data,
                            out,
                            self.opts)
                if failhard:
                    log.error(
                        'Minion %s returned with non-zero exit code. '
                        'Batch run stopped due to failhard', minion
                    )
                    raise StopIteration

            # remove inactive iterators from the iters list
            for queue in minion_tracker:
                # only remove inactive queues
                if not minion_tracker[queue]['active'] and queue in iters:
                    iters.remove(queue)
                    # also remove the iterator's minions from the active list
                    for minion in minion_tracker[queue]['minions']:
                        if minion in active:
                            active.remove(minion)
                            if bwait:
                                wait.append(datetime.now() + timedelta(seconds=bwait))
1085635
def get_load(jid):
    '''
    
    '''
    cb_ = _get_connection()

    try:
        jid_doc = cb_.get(six.text_type(jid))
    except couchbase.exceptions.NotFoundError:
        return {}

    ret = {}
    try:
        ret = jid_doc.value['load']
        ret['Minions'] = jid_doc.value['minions']
    except KeyError as e:
        log.error(e)

    return ret
1034945
def get_minions():
    '''
    
    '''
    serv = _get_serv(ret=None)
    sql = "select distinct(id) from returns"

    data = serv.query(sql)
    ret = []
    if data:
        for jid in data[0]['points']:
            ret.append(jid[1])

    return ret
727164
def get_fun(fun):
    '''
    
    '''
    serv = _get_serv(ret=None)
    ret = {}
    for minion in serv.smembers('minions'):
        ind_str = '{0}:{1}'.format(minion, fun)
        try:
            jid = serv.get(ind_str)
        except Exception:
            continue
        if not jid:
            continue
        data = serv.get('{0}:{1}'.format(minion, jid))
        if data:
            ret[minion] = salt.utils.json.loads(data)
    return ret
1080836
def _get_token_and_url_from_master():
    '''
    
    '''
    minion_id = __grains__['id']
    pki_dir = __opts__['pki_dir']

    # When rendering pillars, the module executes on the master, but the token
    # should be issued for the minion, so that the correct policies are applied
    if __opts__.get('__role', 'minion') == 'minion':
        private_key = '{0}/minion.pem'.format(pki_dir)
        log.debug('Running on minion, signing token request with key %s',
                  private_key)
        signature = base64.b64encode(salt.crypt.sign_message(
            private_key,
            minion_id
        ))
        result = __salt__['publish.runner'](
            'vault.generate_token',
            arg=[minion_id, signature]
        )
    else:
        private_key = '{0}/master.pem'.format(pki_dir)
        log.debug('Running on master, signing token request for %s with key %s',
                  minion_id, private_key)
        signature = base64.b64encode(salt.crypt.sign_message(
            private_key,
            minion_id
        ))
        result = __salt__['saltutil.runner'](
            'vault.generate_token',
            minion_id=minion_id,
            signature=signature,
            impersonated_by_master=True
        )

    if not result:
        log.error('Failed to get token from master! No result returned - '
                  'is the peer publish configuration correct?')
        raise salt.exceptions.CommandExecutionError(result)
    if not isinstance(result, dict):
        log.error('Failed to get token from master! '
                  'Response is not a dict: %s', result)
        raise salt.exceptions.CommandExecutionError(result)
    if 'error' in result:
        log.error('Failed to get token from master! '
                  'An error was returned: %s', result['error'])
        raise salt.exceptions.CommandExecutionError(result)
    return {
        'url': result['url'],
        'token': result['token'],
        'verify': result.get('verify', None),
    }
172649
def _thread_multi_return(cls, minion_instance, opts, data):
        '''
        
        '''
        fn_ = os.path.join(minion_instance.proc_dir, data['jid'])

        if opts['multiprocessing'] and not salt.utils.platform.is_windows():
            # Shutdown the multiprocessing before daemonizing
            salt.log.setup.shutdown_multiprocessing_logging()

            salt.utils.process.daemonize_if(opts)

            # Reconfigure multiprocessing logging after daemonizing
            salt.log.setup.setup_multiprocessing_logging()

        salt.utils.process.appendproctitle('{0}._thread_multi_return {1}'.format(cls.__name__, data['jid']))

        sdata = {'pid': os.getpid()}
        sdata.update(data)
        log.info('Starting a new job with PID %s', sdata['pid'])
        with salt.utils.files.fopen(fn_, 'w+b') as fp_:
            fp_.write(minion_instance.serial.dumps(sdata))

        multifunc_ordered = opts.get('multifunc_ordered', False)
        num_funcs = len(data['fun'])
        if multifunc_ordered:
            ret = {
                'return': [None] * num_funcs,
                'retcode': [None] * num_funcs,
                'success': [False] * num_funcs
            }
        else:
            ret = {
                'return': {},
                'retcode': {},
                'success': {}
            }

        for ind in range(0, num_funcs):
            if not multifunc_ordered:
                ret['success'][data['fun'][ind]] = False
            try:
                minion_blackout_violation = False
                if minion_instance.connected and minion_instance.opts['pillar'].get('minion_blackout', False):
                    whitelist = minion_instance.opts['pillar'].get('minion_blackout_whitelist', [])
                    # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist
                    if data['fun'][ind] != 'saltutil.refresh_pillar' and data['fun'][ind] not in whitelist:
                        minion_blackout_violation = True
                elif minion_instance.opts['grains'].get('minion_blackout', False):
                    whitelist = minion_instance.opts['grains'].get('minion_blackout_whitelist', [])
                    if data['fun'][ind] != 'saltutil.refresh_pillar' and data['fun'][ind] not in whitelist:
                        minion_blackout_violation = True
                if minion_blackout_violation:
                    raise SaltInvocationError('Minion in blackout mode. Set \'minion_blackout\' '
                                             'to False in pillar or grains to resume operations. Only '
                                             'saltutil.refresh_pillar allowed in blackout mode.')

                func = minion_instance.functions[data['fun'][ind]]

                args, kwargs = load_args_and_kwargs(
                    func,
                    data['arg'][ind],
                    data)
                minion_instance.functions.pack['__context__']['retcode'] = 0
                key = ind if multifunc_ordered else data['fun'][ind]
                ret['return'][key] = func(*args, **kwargs)
                retcode = minion_instance.functions.pack['__context__'].get(
                    'retcode',
                    0
                )
                if retcode == 0:
                    # No nonzero retcode in __context__ dunder. Check if return
                    # is a dictionary with a "result" or "success" key.
                    try:
                        func_result = all(ret['return'][key].get(x, True)
                                          for x in ('result', 'success'))
                    except Exception:
                        # return data is not a dict
                        func_result = True
                    if not func_result:
                        retcode = 1

                ret['retcode'][key] = retcode
                ret['success'][key] = retcode == 0
            except Exception as exc:
                trb = traceback.format_exc()
                log.warning('The minion function caused an exception: %s', exc)
                if multifunc_ordered:
                    ret['return'][ind] = trb
                else:
                    ret['return'][data['fun'][ind]] = trb
            ret['jid'] = data['jid']
            ret['fun'] = data['fun']
            ret['fun_args'] = data['arg']
        if 'metadata' in data:
            ret['metadata'] = data['metadata']
        if minion_instance.connected:
            minion_instance._return_pub(
                ret,
                timeout=minion_instance._return_retry_timer()
            )
        if data['ret']:
            if 'ret_config' in data:
                ret['ret_config'] = data['ret_config']
            if 'ret_kwargs' in data:
                ret['ret_kwargs'] = data['ret_kwargs']
            for returner in set(data['ret'].split(',')):
                ret['id'] = opts['id']
                try:
                    minion_instance.returners['{0}.returner'.format(
                        returner
                    )](ret)
                except Exception as exc:
                    log.error(
                        'The return failed for job %s: %s',
                        data['jid'], exc
                    )
433667
def _create_minion_object(self, opts, timeout, safe,
                              io_loop=None, loaded_base_name=None,
                              jid_queue=None):
        '''
        
        '''
        return Minion(opts,
                      timeout,
                      safe,
                      io_loop=io_loop,
                      loaded_base_name=loaded_base_name,
                      jid_queue=jid_queue)
697329
def ext_pillar(minion_id,
               pillar,
               *args,
               **kwargs):
    '''
    
    '''
    return POSTGRESExtPillar().fetch(minion_id, pillar, *args, **kwargs)
704697
def ret_list_minions(self):
        '''
        
        '''
        tgt = _tgt_set(self.tgt)
        return self._ret_minions(tgt.intersection)
83259
def get_minions():
    '''
    
    '''
    log.debug('sqlite3 returner <get_minions> called')
    conn = _get_conn(ret=None)
    cur = conn.cursor()
    sql = '''SELECT DISTINCT id FROM salt_returns'''
    cur.execute(sql)
    data = cur.fetchall()
    ret = []
    for minion in data:
        ret.append(minion[0])
    _close_conn(conn)
    return ret
1026858
def mine_get(tgt, fun, tgt_type='glob', opts=None):
    '''
    
    '''
    ret = {}
    serial = salt.payload.Serial(opts)
    checker = CkMinions(opts)
    _res = checker.check_minions(
            tgt,
            tgt_type)
    minions = _res['minions']
    cache = salt.cache.factory(opts)

    if isinstance(fun, six.string_types):
        functions = list(set(fun.split(',')))
        _ret_dict = len(functions) > 1
    elif isinstance(fun, list):
        functions = fun
        _ret_dict = True
    else:
        return {}

    for minion in minions:
        mdata = cache.fetch('minions/{0}'.format(minion), 'mine')

        if not isinstance(mdata, dict):
            continue

        if not _ret_dict and functions and functions[0] in mdata:
            ret[minion] = mdata.get(functions)
        elif _ret_dict:
            for fun in functions:
                if fun in mdata:
                    ret.setdefault(fun, {})[minion] = mdata.get(fun)

    return ret
914961
def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None,
                     pillar_override=None, pillarenv=None,
                     extra_minion_data=None):
    '''
    
    '''
    file_client = opts['file_client']
    if opts.get('master_type') == 'disable' and file_client == 'remote':
        file_client = 'local'
    ptype = {
        'remote': AsyncRemotePillar,
        'local': AsyncPillar,
    }.get(file_client, AsyncPillar)
    return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs,
                 pillar_override=pillar_override, pillarenv=pillarenv,
                 extra_minion_data=extra_minion_data)
440712
def get_returns(
            self,
            jid,
            minions,
            timeout=None):
        '''
        
        '''
        minions = set(minions)
        if timeout is None:
            timeout = self.opts['timeout']
        start = int(time.time())
        timeout_at = start + timeout
        log.debug(
            'get_returns for jid %s sent to %s will timeout at %s',
            jid, minions, datetime.fromtimestamp(timeout_at).time()
        )

        found = set()
        ret = {}
        # Check to see if the jid is real, if not return the empty dict
        try:
            if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}:
                log.warning('jid does not exist')
                return ret
        except Exception as exc:
            raise SaltClientError('Master job cache returner [{0}] failed to verify jid. '
                                  'Exception details: {1}'.format(self.opts['master_job_cache'], exc))

        # Wait for the hosts to check in
        while True:
            time_left = timeout_at - int(time.time())
            wait = max(1, time_left)
            raw = self.event.get_event(wait, jid, auto_reconnect=self.auto_reconnect)
            if raw is not None and 'return' in raw:
                found.add(raw['id'])
                ret[raw['id']] = raw['return']
                if len(found.intersection(minions)) >= len(minions):
                    # All minions have returned, break out of the loop
                    log.debug('jid %s found all minions', jid)
                    break
                continue
            # Then event system timeout was reached and nothing was returned
            if len(found.intersection(minions)) >= len(minions):
                # All minions have returned, break out of the loop
                log.debug('jid %s found all minions', jid)
                break
            if int(time.time()) > timeout_at:
                log.info(
                    'jid %s minions %s did not return in time',
                    jid, (minions - found)
                )
                break
            time.sleep(0.01)
        return ret
1114157
def returner(load):
    '''
    
    '''
    cb_ = _get_connection()

    hn_key = '{0}/{1}'.format(load['jid'], load['id'])
    try:
        ret_doc = {'return': load['return'],
                   'full_ret': salt.utils.json.dumps(load)}

        cb_.add(hn_key,
               ret_doc,
               ttl=_get_ttl(),
               )
    except couchbase.exceptions.KeyExistsError:
        log.error(
            'An extra return was detected from minion %s, please verify '
            'the minion, this could be a replay attack', load['id']
        )
        return False
375195
def get_fun(fun):
    '''
    
    '''
    serv = _get_serv(ret=None)
    minions = _get_list(serv, 'minions')
    returns = serv.get_multi(minions, key_prefix='{0}:'.format(fun))
    # returns = {minion: return, minion: return, ...}
    ret = {}
    for minion, data in six.iteritems(returns):
        ret[minion] = salt.utils.json.loads(data)
    return ret