Compile the master side low state data, and build the hidden state file
965266
def master_compile(master_opts, minion_opts, grains, id_, saltenv):
    '''
    
    '''
    st_ = MasterHighState(master_opts, minion_opts, grains, id_, saltenv)
    return st_.compile_highstate()
371498
def main(unused_argv):
    """"""
    utils.ensure_dir_exists(os.path.dirname(FLAGS.export_path))
    if FLAGS.create_bootstrap:
        dual_net.bootstrap()
    dual_net.export_model(FLAGS.export_path)
713581
def save_state(state, output_dir, keep=False):
  """"""
  params_file = os.path.join(output_dir, "model.pkl")
  with gfile.GFile(params_file, "wb") as f:
    pickle.dump((state.params, state.step, state.history), f)
  if keep:
    params_file = os.path.join(output_dir, "model_{}.pkl".format(state.step))
    with gfile.GFile(params_file, "wb") as f:
      pickle.dump((state.params, state.step, state.history), f)
  log("Model saved to %s" % params_file, stdout=False)
482575
def exec_lowstate(self, client=None, token=None):
        '''
        
        '''
        lowstate = cherrypy.request.lowstate

        # Release the session lock before executing any potentially
        # long-running Salt commands. This allows different threads to execute
        # Salt commands concurrently without blocking.
        if cherrypy.request.config.get('tools.sessions.on', False):
            cherrypy.session.release_lock()

        # if the lowstate loaded isn't a list, lets notify the client
        if not isinstance(lowstate, list):
            raise cherrypy.HTTPError(400, 'Lowstates must be a list')

        # Make any requested additions or modifications to each lowstate, then
        # execute each one and yield the result.
        for chunk in lowstate:
            if token:
                chunk['token'] = token

            if 'token' in chunk:
                # Make sure that auth token is hex
                try:
                    int(chunk['token'], 16)
                except (TypeError, ValueError):
                    raise cherrypy.HTTPError(401, 'Invalid token')

            if 'token' in chunk:
                # Make sure that auth token is hex
                try:
                    int(chunk['token'], 16)
                except (TypeError, ValueError):
                    raise cherrypy.HTTPError(401, 'Invalid token')

            if client:
                chunk['client'] = client

            # Make any 'arg' params a list if not already.
            # This is largely to fix a deficiency in the urlencoded format.
            if 'arg' in chunk and not isinstance(chunk['arg'], list):
                chunk['arg'] = [chunk['arg']]

            ret = self.api.run(chunk)

            # Sometimes Salt gives us a return and sometimes an iterator
            if isinstance(ret, collections.Iterator):
                for i in ret:
                    yield i
            else:
                yield ret
1141006
def build(filenames, uri, cl_args, link_args, x64, native):
    
    logging.info(_('This is source file building mode.'))
    logging.debug(_('filenames: %s'), filenames)
    logging.debug(_('uri: %s'), uri)
    logging.debug(_('cl_args: %s'), cl_args)
    logging.debug(_('link_args: %s'), link_args)
    logging.debug(_('native: %s'), native)
    logging.debug(_('x64: %s'), x64)

    if is_windows():
        pass
        # ret = msbuild(uri, native, list(filenames), x64=x64,
        #               cl_args=cl_args, link_args=link_args)
    else:
        builder = LinuxBuilder()
        ret = builder.build(filenames, x64, 'src', 'out')
    sys.exit(ret)
26250
def _ssh_state(chunks, st_kwargs,
              kwargs, test=False):
    '''
    
    '''
    file_refs = salt.client.ssh.state.lowstate_file_refs(
            chunks,
            _merge_extra_filerefs(
                kwargs.get('extra_filerefs', ''),
                __opts__.get('extra_filerefs', '')
                )
            )
    # Create the tar containing the state pkg and relevant files.
    trans_tar = salt.client.ssh.state.prep_trans_tar(
            __context__['fileclient'],
            chunks,
            file_refs,
            __pillar__,
            st_kwargs['id_'])
    trans_tar_sum = salt.utils.hashutils.get_hash(trans_tar, __opts__['hash_type'])
    cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format(
            __opts__['thin_dir'],
            test,
            trans_tar_sum,
            __opts__['hash_type'])
    single = salt.client.ssh.Single(
            __opts__,
            cmd,
            fsclient=__context__['fileclient'],
            minion_opts=__salt__.minion_opts,
            **st_kwargs)
    single.shell.send(
            trans_tar,
            '{0}/salt_state.tgz'.format(__opts__['thin_dir']))
    stdout, stderr, _ = single.cmd_block()

    # Clean up our tar
    try:
        os.remove(trans_tar)
    except (OSError, IOError):
        pass

    # Read in the JSON data and return the data structure
    try:
        return salt.utils.data.decode(salt.utils.json.loads(stdout, object_hook=salt.utils.data.encode_dict))
    except Exception as e:
        log.error("JSON Render failed for: %s\n%s", stdout, stderr)
        log.error(str(e))

    # If for some reason the json load fails, return the stdout
    return salt.utils.data.decode(stdout)
822532
def _change_source_state(name, state):
    '''
    

    '''
    choc_path = _find_chocolatey(__context__, __salt__)
    cmd = [choc_path, 'source', state, '--name', name]
    result = __salt__['cmd.run_all'](cmd, python_shell=False)

    if result['retcode'] != 0:
        raise CommandExecutionError(
            'Running chocolatey failed: {0}'.format(result['stdout'])
        )

    return result['stdout']
859
private void serializeMasterState(MasterState state, DataOutputStream dos) throws IOException {
		// magic number for error detection
		dos.writeInt(MASTER_STATE_MAGIC_NUMBER);

		// for safety, we serialize first into an array and then write the array and its
		// length into the checkpoint
		final ByteArrayOutputStream baos = new ByteArrayOutputStream();
		final DataOutputStream out = new DataOutputStream(baos);

		out.writeInt(state.version());
		out.writeUTF(state.name());

		final byte[] bytes = state.bytes();
		out.writeInt(bytes.length);
		out.write(bytes, 0, bytes.length);

		out.close();
		byte[] data = baos.toByteArray();

		dos.writeInt(data.length);
		dos.write(data, 0, data.length);
	}
129204
def render_pstate(self, sls, saltenv, mods, defaults=None):
        '''
        
        '''
        if defaults is None:
            defaults = {}
        err = ''
        errors = []
        state_data = self.client.get_state(sls, saltenv)
        fn_ = state_data.get('dest', False)
        if not fn_:
            if sls in self.ignored_pillars.get(saltenv, []):
                log.debug('Skipping ignored and missing SLS \'%s\' in '
                          'environment \'%s\'', sls, saltenv)
                return None, mods, errors
            elif self.opts['pillar_roots'].get(saltenv):
                msg = ('Specified SLS \'{0}\' in environment \'{1}\' is not'
                       ' available on the salt master').format(sls, saltenv)
                log.error(msg)
                errors.append(msg)
            else:
                msg = ('Specified SLS \'{0}\' in environment \'{1}\' was not '
                       'found. '.format(sls, saltenv))
                if self.opts.get('__git_pillar', False) is True:
                    msg += (
                        'This is likely caused by a git_pillar top file '
                        'containing an environment other than the one for the '
                        'branch in which it resides. Each git_pillar '
                        'branch/tag must have its own top file.'
                    )
                else:
                    msg += (
                        'This could be because SLS \'{0}\' is in an '
                        'environment other than \'{1}\', but \'{1}\' is '
                        'included in that environment\'s Pillar top file. It '
                        'could also be due to environment \'{1}\' not being '
                        'defined in \'pillar_roots\'.'.format(sls, saltenv)
                    )
                log.debug(msg)
                # return state, mods, errors
                return None, mods, errors
        state = None
        try:
            state = compile_template(fn_,
                                     self.rend,
                                     self.opts['renderer'],
                                     self.opts['renderer_blacklist'],
                                     self.opts['renderer_whitelist'],
                                     saltenv,
                                     sls,
                                     _pillar_rend=True,
                                     **defaults)
        except Exception as exc:
            msg = 'Rendering SLS \'{0}\' failed, render error:\n{1}'.format(
                sls, exc
            )
            log.critical(msg, exc_info=True)
            if self.opts.get('pillar_safe_render_error', True):
                errors.append(
                    'Rendering SLS \'{0}\' failed. Please see master log for '
                    'details.'.format(sls)
                )
            else:
                errors.append(msg)
        mods.add(sls)
        nstate = None
        if state:
            if not isinstance(state, dict):
                msg = 'SLS \'{0}\' does not render to a dictionary'.format(sls)
                log.error(msg)
                errors.append(msg)
            else:
                if 'include' in state:
                    if not isinstance(state['include'], list):
                        msg = ('Include Declaration in SLS \'{0}\' is not '
                               'formed as a list'.format(sls))
                        log.error(msg)
                        errors.append(msg)
                    else:
                        # render included state(s)
                        include_states = []

                        matched_pstates = []
                        for sub_sls in state.pop('include'):
                            if isinstance(sub_sls, dict):
                                sub_sls, v = next(six.iteritems(sub_sls))
                                defaults = v.get('defaults', {})
                                key = v.get('key', None)
                            else:
                                key = None

                            try:
                                if sub_sls.startswith('.'):
                                    if state_data.get('source', '').endswith('/init.sls'):
                                        include_parts = sls.split('.')
                                    else:
                                        include_parts = sls.split('.')[:-1]
                                    sub_sls = '.'.join(include_parts+[sub_sls[1:]])
                                matches = fnmatch.filter(
                                    self.avail[saltenv],
                                    sub_sls,
                                )
                                matched_pstates.extend(matches)
                            except KeyError:
                                errors.extend(
                                    ['No matching pillar environment for environment '
                                     '\'{0}\' found'.format(saltenv)]
                                )

                        for sub_sls in matched_pstates:
                            if sub_sls not in mods:
                                nstate, mods, err = self.render_pstate(
                                        sub_sls,
                                        saltenv,
                                        mods,
                                        defaults
                                        )
                                if nstate:
                                    if key:
                                        # If key is x:y, convert it to {x: {y: nstate}}
                                        for key_fragment in reversed(key.split(":")):
                                            nstate = {
                                                key_fragment: nstate
                                            }
                                    if not self.opts.get('pillar_includes_override_sls', False):
                                        include_states.append(nstate)
                                    else:
                                        state = merge(
                                            state,
                                            nstate,
                                            self.merge_strategy,
                                            self.opts.get('renderer', 'yaml'),
                                            self.opts.get('pillar_merge_lists', False))
                                if err:
                                    errors += err

                        if not self.opts.get('pillar_includes_override_sls', False):
                            # merge included state(s) with the current state
                            # merged last to ensure that its values are
                            # authoritative.
                            include_states.append(state)
                            state = None
                            for s in include_states:
                                if state is None:
                                    state = s
                                else:
                                    state = merge(
                                        state,
                                        s,
                                        self.merge_strategy,
                                        self.opts.get('renderer', 'yaml'),
                                        self.opts.get('pillar_merge_lists', False))
        return state, mods, errors
777271
def build(self):
        
        try:
            self._delete_dir()
            try:
                tar = tarfile.open(self.script)
            except Exception as err:
                print err
                raise SystemExit()
            tar.extractall()
            tar.close()
            self._makeflags()
            self._delete_sbo_tar_gz()
            self._create_md5_dict()
            if not self.auto:
                os.chdir(self._SOURCES)
            for src in self.sources:
                if not os.path.isfile(src):
                    continue
                # fix build sources with spaces
                src = src.replace("%20", " ")
                check_md5(self.sbo_md5[src], src)
                # copy source and fix passing char '+' from file name
                shutil.copy2(src.replace("%2B", "+"), self.path + self.prgnam)
            os.chdir(self.path + self.prgnam)
            # change permissions
            subprocess.call("chmod +x {0}.SlackBuild".format(self.prgnam),
                            shell=True)
            pass_var = self._pass_variable()
            if self.meta.sbo_build_log in ["on", "ON"]:
                if os.path.isfile(self.build_logs + self.log_file):
                    os.remove(self.build_logs + self.log_file)
                # start log write
                log_head(self.build_logs, self.log_file, self.start_log_time)
                subprocess.Popen("{0} ./{1}.SlackBuild 2>&1 | tee -a "
                                 "{2}{3}".format(" ".join(pass_var),
                                                 self.prgnam, self.build_logs,
                                                 self.log_file), shell=True,
                                 stdout=sys.stdout).communicate()
                sum_time = build_time(self.start_time)
                # write end in log file
                log_end(self.build_logs, self.log_file, sum_time)
                print("Total build time for the package {0} : {1}\n".format(
                    self.prgnam, sum_time))
            else:
                subprocess.call("{0} ./{1}.SlackBuild".format(
                    " ".join(pass_var), self.prgnam), shell=True)
            os.chdir(self.path)
        except KeyboardInterrupt:   # (OSError, IOError):
            self.msg.pkg_not_found("\n", self.prgnam, "Wrong file", "\n")
204127
def show_lowstate(**kwargs):
    '''
    
    '''
    __opts__['grains'] = __grains__
    opts = salt.utils.state.get_sls_opts(__opts__, **kwargs)
    st_ = salt.client.ssh.state.SSHHighState(
            opts,
            __pillar__,
            __salt__,
            __context__['fileclient'])
    st_.push_active()
    chunks = st_.compile_low_chunks()
    _cleanup_slsmod_low_data(chunks)
    return chunks
77820
def PushState(self, **unused_kwargs):
    """"""
    logging.debug('Storing state {0:s}'.format(repr(self.state)))
    self.state_stack.append(self.state)
814532
def _state_run(self):
        '''
        
        '''
        if self.opts['startup_states']:
            if self.opts.get('master_type', 'str') == 'disable' and \
                        self.opts.get('file_client', 'remote') == 'remote':
                log.warning(
                    'Cannot run startup_states when \'master_type\' is set '
                    'to \'disable\' and \'file_client\' is set to '
                    '\'remote\'. Skipping.'
                )
            else:
                data = {'jid': 'req', 'ret': self.opts.get('ext_job_cache', '')}
                if self.opts['startup_states'] == 'sls':
                    data['fun'] = 'state.sls'
                    data['arg'] = [self.opts['sls_list']]
                elif self.opts['startup_states'] == 'top':
                    data['fun'] = 'state.top'
                    data['arg'] = [self.opts['top_file']]
                else:
                    data['fun'] = 'state.highstate'
                    data['arg'] = []
                self._handle_decoded_payload(data)
33725
def build(self):
        
        pkg_security([self.name])
        self.error_uns()
        if self.FAULT:
            print("")
            self.msg.template(78)
            print("| Package {0} {1} {2} {3}".format(self.prgnam, self.red,
                                                     self.FAULT, self.endc))
            self.msg.template(78)
        else:
            sources = []
            if not os.path.exists(self.meta.build_path):
                os.makedirs(self.meta.build_path)
            if not os.path.exists(self._SOURCES):
                os.makedirs(self._SOURCES)
            os.chdir(self.meta.build_path)
            Download(self.meta.build_path, self.sbo_dwn.split(),
                     repo="sbo").start()
            Download(self._SOURCES, self.source_dwn, repo="sbo").start()
            script = self.sbo_dwn.split("/")[-1]
            for src in self.source_dwn:
                sources.append(src.split("/")[-1])
            BuildPackage(script, sources, self.meta.build_path,
                         auto=False).build()
            slack_package(self.prgnam)
64810
def _master_opts(self, load):
        '''
        
        '''
        mopts = {}
        file_roots = {}
        envs = self._file_envs()
        for saltenv in envs:
            if saltenv not in file_roots:
                file_roots[saltenv] = []
        mopts['file_roots'] = file_roots
        mopts['top_file_merging_strategy'] = self.opts['top_file_merging_strategy']
        mopts['env_order'] = self.opts['env_order']
        mopts['default_top'] = self.opts['default_top']
        if load.get('env_only'):
            return mopts
        mopts['renderer'] = self.opts['renderer']
        mopts['failhard'] = self.opts['failhard']
        mopts['state_top'] = self.opts['state_top']
        mopts['state_top_saltenv'] = self.opts['state_top_saltenv']
        mopts['nodegroups'] = self.opts['nodegroups']
        mopts['state_auto_order'] = self.opts['state_auto_order']
        mopts['state_events'] = self.opts['state_events']
        mopts['state_aggregate'] = self.opts['state_aggregate']
        mopts['jinja_env'] = self.opts['jinja_env']
        mopts['jinja_sls_env'] = self.opts['jinja_sls_env']
        mopts['jinja_lstrip_blocks'] = self.opts['jinja_lstrip_blocks']
        mopts['jinja_trim_blocks'] = self.opts['jinja_trim_blocks']
        return mopts
306684
def build_state(
            cls,
            db: BaseAtomicDB,
            header: BlockHeader,
            previous_hashes: Iterable[Hash32] = ()) -> BaseState:
        

        execution_context = header.create_execution_context(previous_hashes)
        return cls.get_state_class()(db, execution_context, header.state_root)
1213285
def compile(source, options = {})
      source  = prepare_source(source)
      options = prepare_options(options)
      command = ["node", "--eval"]
      command.push compilation_snippet \
        method:    "compile#{ "Client" if options[:client] }",
        arguments: [source, options],
        locals:    options.fetch(:locals, {}),
        options:   options
      stdout, stderr, exit_status = Open3.capture3(*command)
      raise engine::CompilationError, stderr unless exit_status.success?
      process_result(source, stdout, options)
    end