Return the load data that marks a specified jid
325841
def get_load(jid):
    '''
    
    '''
    jid_dir = salt.utils.jid.jid_dir(jid, _job_dir(), __opts__['hash_type'])
    load_fn = os.path.join(jid_dir, LOAD_P)
    if not os.path.exists(jid_dir) or not os.path.exists(load_fn):
        return {}
    serial = salt.payload.Serial(__opts__)
    ret = {}
    load_p = os.path.join(jid_dir, LOAD_P)
    num_tries = 5
    for index in range(1, num_tries + 1):
        with salt.utils.files.fopen(load_p, 'rb') as rfh:
            try:
                ret = serial.load(rfh)
                break
            except Exception as exc:
                if index == num_tries:
                    time.sleep(0.25)
    else:
        log.critical('Failed to unpack %s', load_p)
        raise exc
    if ret is None:
        ret = {}
    minions_cache = [os.path.join(jid_dir, MINIONS_P)]
    minions_cache.extend(
        glob.glob(os.path.join(jid_dir, SYNDIC_MINIONS_P.format('*')))
    )
    all_minions = set()
    for minions_path in minions_cache:
        log.debug('Reading minion list from %s', minions_path)
        try:
            with salt.utils.files.fopen(minions_path, 'rb') as rfh:
                all_minions.update(serial.load(rfh))
        except IOError as exc:
            salt.utils.files.process_read_exception(exc, minions_path)

    if all_minions:
        ret['Minions'] = sorted(all_minions)

    return ret
1118605
def get_jid(jid):
    '''
    
    '''
    log.debug('sdstack_etcd returner <get_jid> called jid: %s', jid)
    ret = {}
    client, path = _get_conn(__opts__)
    items = client.get('/'.join((path, 'jobs', jid)))
    for item in items.children:
        if str(item.key).endswith('.load.p'):
            continue
        comps = str(item.key).split('/')
        data = client.get('/'.join((path, 'jobs', jid, comps[-1], 'return'))).value
        ret[comps[-1]] = {'return': salt.utils.json.loads(data)}
    return ret
584876
def find_cached_job(jid):
    '''
    
    '''
    serial = salt.payload.Serial(__opts__)
    proc_dir = os.path.join(__opts__['cachedir'], 'minion_jobs')
    job_dir = os.path.join(proc_dir, six.text_type(jid))
    if not os.path.isdir(job_dir):
        if not __opts__.get('cache_jobs'):
            return ('Local jobs cache directory not found; you may need to'
                    ' enable cache_jobs on this minion')
        else:
            return 'Local jobs cache directory {0} not found'.format(job_dir)
    path = os.path.join(job_dir, 'return.p')
    with salt.utils.files.fopen(path, 'rb') as fp_:
        buf = fp_.read()
    if buf:
        try:
            data = serial.loads(buf)
        except NameError:
            # msgpack error in salt-ssh
            pass
        else:
            if isinstance(data, dict):
                # if not a dict, this was an invalid serialized object
                return data
    return None
1116898
function (jid)
    {
        var bare = Strophe.getBareJidFromJid(jid);
        if (bare.indexOf("@") < 0) {
            return bare;
        } else {
            var parts = bare.split("@");
            parts.splice(0, 1);
            return parts.join('@');
        }
    }
713720
def pub_ret(self, load, skip_verify=False):
        '''
        
        '''
        if not skip_verify and any(key not in load for key in ('jid', 'id')):
            return {}
        else:
            auth_cache = os.path.join(
                    self.opts['cachedir'],
                    'publish_auth')
            if not os.path.isdir(auth_cache):
                os.makedirs(auth_cache)
            jid_fn = os.path.join(auth_cache, load['jid'])
            with salt.utils.files.fopen(jid_fn, 'r') as fp_:
                if not load['id'] == salt.utils.stringutils.to_unicode(fp_.read()):
                    return {}

            return self.local.get_cache_returns(load['jid'])
151801
def get_jids(returner):
    '''
    
    '''
    returners = salt.loader.returners(__opts__, __salt__)
    return returners['{0}.get_jids'.format(returner)]()
712487
def get_jid(jid):
    '''
    
    '''
    cb_ = _get_connection()
    _verify_views()

    ret = {}

    for result in cb_.query(DESIGN_NAME, 'jid_returns', key=six.text_type(jid), include_docs=True):
        ret[result.value] = result.doc.value

    return ret
307182
def get_jids_filter(count, filter_find_job=True):
    '''
    
    '''
    with _get_serv(ret=None, commit=True) as cur:

        sql = '''SELECT * FROM (
                     SELECT DISTINCT `jid` ,`load` FROM `jids`
                     {0}
                     ORDER BY `jid` DESC limit {1}
                     ) `tmp`
                 ORDER BY `jid`;'''
        where = '''WHERE `load` NOT LIKE '%"fun": "saltutil.find_job"%' '''

        cur.execute(sql.format(where if filter_find_job else '', count))
        data = cur.fetchall()
        ret = []
        for jid in data:
            ret.append(salt.utils.jid.format_jid_instance_ext(
                jid[0],
                salt.utils.json.loads(jid[1])))
        return ret
197634
def get_iter_returns(
            self,
            jid,
            minions,
            timeout=None,
            tgt='*',
            tgt_type='glob',
            expect_minions=False,
            block=True,
            **kwargs):
        '''
        
        '''
        if not isinstance(minions, set):
            if isinstance(minions, six.string_types):
                minions = set([minions])
            elif isinstance(minions, (list, tuple)):
                minions = set(list(minions))

        if timeout is None:
            timeout = self.opts['timeout']
        gather_job_timeout = int(kwargs.get('gather_job_timeout', self.opts['gather_job_timeout']))
        start = int(time.time())

        # timeouts per minion, id_ -> timeout time
        minion_timeouts = {}

        found = set()
        missing = set()
        # Check to see if the jid is real, if not return the empty dict
        try:
            if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}:
                log.warning('jid does not exist')
                yield {}
                # stop the iteration, since the jid is invalid
                raise StopIteration()
        except Exception as exc:
            log.warning('Returner unavailable: %s', exc, exc_info_on_loglevel=logging.DEBUG)
        # Wait for the hosts to check in
        last_time = False
        # iterator for this job's return
        if self.opts['order_masters']:
            # If we are a MoM, we need to gather expected minions from downstreams masters.
            ret_iter = self.get_returns_no_block('(salt/job|syndic/.*)/{0}'.format(jid), 'regex')
        else:
            ret_iter = self.get_returns_no_block('salt/job/{0}'.format(jid))
        # iterator for the info of this job
        jinfo_iter = []
        # open event jids that need to be un-subscribed from later
        open_jids = set()
        timeout_at = time.time() + timeout
        gather_syndic_wait = time.time() + self.opts['syndic_wait']
        # are there still minions running the job out there
        # start as True so that we ping at least once
        minions_running = True
        log.debug(
            'get_iter_returns for jid %s sent to %s will timeout at %s',
            jid, minions, datetime.fromtimestamp(timeout_at).time()
        )
        while True:
            # Process events until timeout is reached or all minions have returned
            for raw in ret_iter:
                # if we got None, then there were no events
                if raw is None:
                    break
                if 'minions' in raw.get('data', {}):
                    minions.update(raw['data']['minions'])
                    if 'missing' in raw.get('data', {}):
                        missing.update(raw['data']['missing'])
                    continue
                if 'return' not in raw['data']:
                    continue
                if kwargs.get('raw', False):
                    found.add(raw['data']['id'])
                    yield raw
                else:
                    found.add(raw['data']['id'])
                    ret = {raw['data']['id']: {'ret': raw['data']['return']}}
                    if 'out' in raw['data']:
                        ret[raw['data']['id']]['out'] = raw['data']['out']
                    if 'retcode' in raw['data']:
                        ret[raw['data']['id']]['retcode'] = raw['data']['retcode']
                    if 'jid' in raw['data']:
                        ret[raw['data']['id']]['jid'] = raw['data']['jid']
                    if kwargs.get('_cmd_meta', False):
                        ret[raw['data']['id']].update(raw['data'])
                    log.debug('jid %s return from %s', jid, raw['data']['id'])
                    yield ret

            # if we have all of the returns (and we aren't a syndic), no need for anything fancy
            if len(found.intersection(minions)) >= len(minions) and not self.opts['order_masters']:
                # All minions have returned, break out of the loop
                log.debug('jid %s found all minions %s', jid, found)
                break
            elif len(found.intersection(minions)) >= len(minions) and self.opts['order_masters']:
                if len(found) >= len(minions) and len(minions) > 0 and time.time() > gather_syndic_wait:
                    # There were some minions to find and we found them
                    # However, this does not imply that *all* masters have yet responded with expected minion lists.
                    # Therefore, continue to wait up to the syndic_wait period (calculated in gather_syndic_wait) to see
                    # if additional lower-level masters deliver their lists of expected
                    # minions.
                    break
            # If we get here we may not have gathered the minion list yet. Keep waiting
            # for all lower-level masters to respond with their minion lists

            # let start the timeouts for all remaining minions

            for id_ in minions - found:
                # if we have a new minion in the list, make sure it has a timeout
                if id_ not in minion_timeouts:
                    minion_timeouts[id_] = time.time() + timeout

            # if the jinfo has timed out and some minions are still running the job
            # re-do the ping
            if time.time() > timeout_at and minions_running:
                # since this is a new ping, no one has responded yet
                jinfo = self.gather_job_info(jid, list(minions - found), 'list', **kwargs)
                minions_running = False
                # if we weren't assigned any jid that means the master thinks
                # we have nothing to send
                if 'jid' not in jinfo:
                    jinfo_iter = []
                else:
                    jinfo_iter = self.get_returns_no_block('salt/job/{0}'.format(jinfo['jid']))
                timeout_at = time.time() + gather_job_timeout
                # if you are a syndic, wait a little longer
                if self.opts['order_masters']:
                    timeout_at += self.opts.get('syndic_wait', 1)

            # check for minions that are running the job still
            for raw in jinfo_iter:
                # if there are no more events, lets stop waiting for the jinfo
                if raw is None:
                    break
                try:
                    if raw['data']['retcode'] > 0:
                        log.error('saltutil returning errors on minion %s', raw['data']['id'])
                        minions.remove(raw['data']['id'])
                        break
                except KeyError as exc:
                    # This is a safe pass. We're just using the try/except to
                    # avoid having to deep-check for keys.
                    missing_key = exc.__str__().strip('\'"')
                    if missing_key == 'retcode':
                        log.debug('retcode missing from client return')
                    else:
                        log.debug(
                            'Passing on saltutil error. Key \'%s\' missing '
                            'from client return. This may be an error in '
                            'the client.', missing_key
                        )
                # Keep track of the jid events to unsubscribe from later
                open_jids.add(jinfo['jid'])

                # TODO: move to a library??
                if 'minions' in raw.get('data', {}):
                    minions.update(raw['data']['minions'])
                    continue
                if 'syndic' in raw.get('data', {}):
                    minions.update(raw['syndic'])
                    continue
                if 'return' not in raw.get('data', {}):
                    continue

                # if the job isn't running there anymore... don't count
                if raw['data']['return'] == {}:
                    continue

                # if the minion throws an exception containing the word "return"
                # the master will try to handle the string as a dict in the next
                # step. Check if we have a string, log the issue and continue.
                if isinstance(raw['data']['return'], six.string_types):
                    log.error("unexpected return from minion: %s", raw)
                    continue

                if 'return' in raw['data']['return'] and \
                    raw['data']['return']['return'] == {}:
                    continue

                # if we didn't originally target the minion, lets add it to the list
                if raw['data']['id'] not in minions:
                    minions.add(raw['data']['id'])
                # update this minion's timeout, as long as the job is still running
                minion_timeouts[raw['data']['id']] = time.time() + timeout
                # a minion returned, so we know its running somewhere
                minions_running = True

            # if we have hit gather_job_timeout (after firing the job) AND
            # if we have hit all minion timeouts, lets call it
            now = time.time()
            # if we have finished waiting, and no minions are running the job
            # then we need to see if each minion has timedout
            done = (now > timeout_at) and not minions_running
            if done:
                # if all minions have timeod out
                for id_ in minions - found:
                    if now < minion_timeouts[id_]:
                        done = False
                        break
            if done:
                break

            # don't spin
            if block:
                time.sleep(0.01)
            else:
                yield

        # If there are any remaining open events, clean them up.
        if open_jids:
            for jid in open_jids:
                self.event.unsubscribe(jid)

        if expect_minions:
            for minion in list((minions - found)):
                yield {minion: {'failed': True}}

        # Filter out any minions marked as missing for which we received
        # returns (prevents false events sent due to higher-level masters not
        # knowing about lower-level minions).
        missing -= found

        # Report on missing minions
        if missing:
            for minion in missing:
                yield {minion: {'failed': True}}
1264746
def _prep_pub(self, minions, jid, clear_load, extra, missing):
        '''
        
        '''
        clear_load['jid'] = jid
        delimiter = clear_load.get('kwargs', {}).get('delimiter', DEFAULT_TARGET_DELIM)

        # TODO Error reporting over the master event bus
        self.event.fire_event({'minions': minions}, clear_load['jid'])
        new_job_load = {
            'jid': clear_load['jid'],
            'tgt_type': clear_load['tgt_type'],
            'tgt': clear_load['tgt'],
            'user': clear_load['user'],
            'fun': clear_load['fun'],
            'arg': clear_load['arg'],
            'minions': minions,
            'missing': missing,
            }

        # Announce the job on the event bus
        self.event.fire_event(new_job_load, tagify([clear_load['jid'], 'new'], 'job'))

        if self.opts['ext_job_cache']:
            fstr = '{0}.save_load'.format(self.opts['ext_job_cache'])
            save_load_func = True

            # Get the returner's save_load arg_spec.
            try:
                arg_spec = salt.utils.args.get_function_argspec(self.mminion.returners[fstr])

                # Check if 'minions' is included in returner's save_load arg_spec.
                # This may be missing in custom returners, which we should warn about.
                if 'minions' not in arg_spec.args:
                    log.critical(
                        'The specified returner used for the external job cache '
                        '\'%s\' does not have a \'minions\' kwarg in the returner\'s '
                        'save_load function.', self.opts['ext_job_cache']
                    )
            except (AttributeError, KeyError):
                save_load_func = False
                log.critical(
                    'The specified returner used for the external job cache '
                    '"%s" does not have a save_load function!',
                    self.opts['ext_job_cache']
                )

            if save_load_func:
                try:
                    self.mminion.returners[fstr](clear_load['jid'], clear_load, minions=minions)
                except Exception:
                    log.critical(
                        'The specified returner threw a stack trace:\n',
                        exc_info=True
                    )

        # always write out to the master job caches
        try:
            fstr = '{0}.save_load'.format(self.opts['master_job_cache'])
            self.mminion.returners[fstr](clear_load['jid'], clear_load, minions)
        except KeyError:
            log.critical(
                'The specified returner used for the master job cache '
                '"%s" does not have a save_load function!',
                self.opts['master_job_cache']
            )
        except Exception:
            log.critical(
                'The specified returner threw a stack trace:\n',
                exc_info=True
            )
        # Set up the payload
        payload = {'enc': 'aes'}
        # Altering the contents of the publish load is serious!! Changes here
        # break compatibility with minion/master versions and even tiny
        # additions can have serious implications on the performance of the
        # publish commands.
        #
        # In short, check with Thomas Hatch before you even think about
        # touching this stuff, we can probably do what you want to do another
        # way that won't have a negative impact.
        load = {
            'fun': clear_load['fun'],
            'arg': clear_load['arg'],
            'tgt': clear_load['tgt'],
            'jid': clear_load['jid'],
            'ret': clear_load['ret'],
        }
        # if you specified a master id, lets put that in the load
        if 'master_id' in self.opts:
            load['master_id'] = self.opts['master_id']
        # if someone passed us one, use that
        if 'master_id' in extra:
            load['master_id'] = extra['master_id']
        # Only add the delimiter to the pub data if it is non-default
        if delimiter != DEFAULT_TARGET_DELIM:
            load['delimiter'] = delimiter

        if 'id' in extra:
            load['id'] = extra['id']
        if 'tgt_type' in clear_load:
            load['tgt_type'] = clear_load['tgt_type']
        if 'to' in clear_load:
            load['to'] = clear_load['to']

        if 'kwargs' in clear_load:
            if 'ret_config' in clear_load['kwargs']:
                load['ret_config'] = clear_load['kwargs'].get('ret_config')

            if 'metadata' in clear_load['kwargs']:
                load['metadata'] = clear_load['kwargs'].get('metadata')

            if 'module_executors' in clear_load['kwargs']:
                load['module_executors'] = clear_load['kwargs'].get('module_executors')

            if 'executor_opts' in clear_load['kwargs']:
                load['executor_opts'] = clear_load['kwargs'].get('executor_opts')

            if 'ret_kwargs' in clear_load['kwargs']:
                load['ret_kwargs'] = clear_load['kwargs'].get('ret_kwargs')

        if 'user' in clear_load:
            log.info(
                'User %s Published command %s with jid %s',
                clear_load['user'], clear_load['fun'], clear_load['jid']
            )
            load['user'] = clear_load['user']
        else:
            log.info(
                'Published command %s with jid %s',
                clear_load['fun'], clear_load['jid']
            )
        log.debug('Published command details %s', load)
        return load
41182
def get_jids():
    '''
    
    '''
    serv = _get_serv(ret=None)
    sql = "select distinct(jid) from jids group by load"

    # [{u'points': [[0, jid, load],
    #               [0, jid, load]],
    #   u'name': u'jids',
    #   u'columns': [u'time', u'distinct', u'load']}]
    data = serv.query(sql)
    ret = {}
    if data:
        for _, jid, load in data[0]['points']:
            ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load))
    return ret
917047
def save_load(jid, load, minions=None):
    '''
    
    '''
    serv = _get_serv(ret=None)
    serv.setex('load:{0}'.format(jid), _get_ttl(), salt.utils.json.dumps(load))
1363063
def get_event_iter_returns(self, jid, minions, timeout=None):
        '''
        
        '''
        log.trace('entered - function get_event_iter_returns()')
        if timeout is None:
            timeout = self.opts['timeout']

        timeout_at = time.time() + timeout

        found = set()
        # Check to see if the jid is real, if not return the empty dict
        if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}:
            log.warning('jid does not exist')
            yield {}
            # stop the iteration, since the jid is invalid
            raise StopIteration()
        # Wait for the hosts to check in
        while True:
            raw = self.event.get_event(timeout, auto_reconnect=self.auto_reconnect)
            if raw is None or time.time() > timeout_at:
                # Timeout reached
                break
            if 'minions' in raw.get('data', {}):
                continue
            try:
                found.add(raw['id'])
                ret = {raw['id']: {'ret': raw['return']}}
            except KeyError:
                # Ignore other erroneous messages
                continue
            if 'out' in raw:
                ret[raw['id']]['out'] = raw['out']
            yield ret
            time.sleep(0.02)
727506
def get(self, measurementId):
        
        logger.info('Loading raw data for ' + measurementId)
        measurement = self._measurementController.getMeasurement(measurementId, MeasurementStatus.COMPLETE)
        if measurement is not None:
            if measurement.inflate():
                data = {
                    name: {
                        'raw': {
                            'x': self._jsonify(data.raw('x')),
                            'y': self._jsonify(data.raw('y')),
                            'z': self._jsonify(data.raw('z'))
                        },
                        'vibration': {
                            'x': self._jsonify(data.vibration('x')),
                            'y': self._jsonify(data.vibration('y')),
                            'z': self._jsonify(data.vibration('z'))
                        },
                        'tilt': {
                            'x': self._jsonify(data.tilt('x')),
                            'y': self._jsonify(data.tilt('y')),
                            'z': self._jsonify(data.tilt('z'))
                        }
                    }
                    for name, data in measurement.data.items()
                }
                return data, 200
            else:
                return None, 404
        else:
            return None, 404
619482
def gather_job_info(self, jid, tgt, tgt_type, listen=True, **kwargs):
        '''
        
        '''
        log.debug('Checking whether jid %s is still running', jid)
        timeout = int(kwargs.get('gather_job_timeout', self.opts['gather_job_timeout']))

        pub_data = self.run_job(tgt,
                                'saltutil.find_job',
                                arg=[jid],
                                tgt_type=tgt_type,
                                timeout=timeout,
                                listen=listen,
                                **kwargs
                               )

        if 'jid' in pub_data:
            self.event.subscribe(pub_data['jid'])

        return pub_data
365832
def get_jids():
    '''
    
    '''
    options = _get_options(ret=None)
    _response = _request("GET", options['url'] + options['db'] + "/_all_docs?include_docs=true")

    # Make sure the 'total_rows' is returned.. if not error out.
    if 'total_rows' not in _response:
        log.error('Didn\'t get valid response from requesting all docs: %s', _response)
        return {}

    # Return the rows.
    ret = {}
    for row in _response['rows']:
        # Because this shows all the documents in the database, including the
        # design documents, verify the id is salt jid
        jid = row['id']
        if not salt.utils.jid.is_jid(jid):
            continue

        ret[jid] = salt.utils.jid.format_jid_instance(jid, row['doc'])

    return ret
1343513
def get_raw_data(self, ptr, unbuffered=False):
        """"""
        if unbuffered:
            self._data_pos = None
        # round down ptr to a 'block boundary'
        idx = ptr - (ptr % 0x20)
        ptr -= idx
        count = self.reading_len[self.ws_type]
        if self._data_pos == idx:
            # cache contains useful data
            result = self._data_block[ptr:ptr + count]
            if len(result) >= count:
                return result
        else:
            result = list()
        if ptr + count > 0x20:
            # need part of next block, which may be in cache
            if self._data_pos != idx + 0x20:
                self._data_pos = idx + 0x20
                self._data_block = self._read_block(self._data_pos)
            result += self._data_block[0:ptr + count - 0x20]
            if len(result) >= count:
                return result
        # read current block
        self._data_pos = idx
        self._data_block = self._read_block(self._data_pos)
        result = self._data_block[ptr:ptr + count] + result
        return result