Process local commands
337200
def _list(self, args):
        '''
        
        '''
        args.pop(0)
        command = args[0]
        if command == 'packages':
            self._list_packages(args)
        elif command == 'files':
            self._list_files(args)
        elif command == 'repos':
            self._repo_list(args)
        else:
            raise SPMInvocationError('Invalid list command \'{0}\''.format(command))
539336
def call(cmd, input_file=None, input_text=None, encoding=None):
    """"""

    process = get_process(cmd)

    if input_file is not None:
        with open(input_file, 'rb') as f:
            process.stdin.write(f.read())
    if input_text is not None:
        process.stdin.write(input_text)

    return get_process_output(process, encoding)
995595
def execute_command(command=None):
    """"""
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stdin = process.communicate()
    process.wait()
    return (stdout, stdin), process.returncode
1329598
def shell(command, *args):
	''''''
	if args:
		command = command.format(*args)

	print LOCALE['shell'].format(command)

	try:
		return subprocess.check_output(command, shell=True)
	except subprocess.CalledProcessError, ex:
		return ex
1243759
def run_command(self, args=None, node_paths=None):
    """"""
    return command_gen(
      self.tool_installations,
      self.name,
      args=args,
      node_paths=node_paths
    )
918432
def run_locally(self, steps=None, **kwargs):
        """"""
        self.slurm_job = LoggedJobSLURM(self.command(steps),
                                        base_dir = self.parent.p.logs_dir,
                                        modules  = self.modules,
                                        **kwargs)
        self.slurm_job.run_locally()
450473
def cli(ctx, path, renku_home, use_external_storage):
    """"""
    ctx.obj = LocalClient(
        path=path,
        renku_home=renku_home,
        use_external_storage=use_external_storage,
    )
598867
def process_locals(self, node, node_visited, *skipped):
        
        local_vars = self.scope[node].difference(skipped)
        local_vars = local_vars.difference(self.openmp_deps)
        if not local_vars:
            return node_visited  # no processing

        locals_visited = []
        for varname in local_vars:
            vartype = self.typeof(varname)
            decl = Statement("{} {}".format(vartype, varname))
            locals_visited.append(decl)
        self.ldecls.difference_update(local_vars)
        return Block(locals_visited + [node_visited])
554035
def _exec(object, globals, locals):
    
    """
    if sys.version_info < (3,):
        exec('exec object in globals, locals')
    else:
        exec(object, globals, locals)
1157129
def run_locally(self):
        """"""
        self.thread = threading.Thread(target=self.execute_locally)
        self.thread.daemon = True # So that they die when we die
        self.thread.start()
930399
def execute(self, command):
        '''
        
        '''
        process = Popen(command, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
        while True:
            if process.poll() is not None:
                self.returncode = process.returncode  # pylint: disable=W0201
                break
            yield process.stdout.readline()
276246
def process(input_dir, output_dir, function):
        
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        logger = log.get_global_console_logger()
        logger.info("Processing files in {}.".format(input_dir))
        input_file_names = os.listdir(input_dir)
        for input_file_name in input_file_names:
            logger.info("Processing {}.".format(input_file_name))
            input_file = os.path.join(input_dir, input_file_name)
            with codecs.open(input_file, "r", encoding="UTF-8") as f:
                input_string = f.read()
            output_string = function(input_string)
            output_file = os.path.join(output_dir, input_file_name)
            with codecs.open(output_file, "w", encoding="UTF-8") as f:
                f.write(output_string)
        logger.info("Saved processed files to {}.".format(output_dir))
1298359
def main() -> None:
    """"""

    # Make sure we have access to self
    if 'self' not in globals():
        print("Run 'set locals_in_py true' and then rerun this script")
        return

    # Make sure the user passed in an output file
    if len(sys.argv) != 2:
        print("Usage: {} <output_file>".format(os.path.basename(sys.argv[0])))
        return

    # Open the output file
    outfile_path = os.path.expanduser(sys.argv[1])
    try:
        outfile = open(outfile_path, 'w')
    except OSError as e:
        print("Error opening {} because: {}".format(outfile_path, e))
        return

    # Write the help summary
    header = '{0}\nSUMMARY\n{0}\n'.format(ASTERISKS)
    outfile.write(header)

    result = app('help -v')
    outfile.write(result.stdout)

    # Get a list of all commands and help topics and then filter out duplicates
    all_commands = set(self.get_all_commands())
    all_topics = set(self.get_help_topics())
    to_save = list(all_commands | all_topics)
    to_save.sort()

    for item in to_save:
        is_command = item in all_commands
        add_help_to_file(item, outfile, is_command)

        if is_command:
            # Add any sub-commands
            for subcmd in get_sub_commands(getattr(self.cmd_func(item), 'argparser', None)):
                full_cmd = '{} {}'.format(item, subcmd)
                add_help_to_file(full_cmd, outfile, is_command)

    outfile.close()
    print("Output written to {}".format(outfile_path))
238196
def handle_cmd(self, cmd):
        """"""
        cmd = cmd.strip()
        segments = []
        for s in cmd.split():
            # remove bash-like comments
            if s.startswith('#'):
                break
            # TODO implement escape sequences (also for \#)
            segments.append(s)
        args = []
        if not len(segments):
            return
        # process more specific commands first
        while segments:
            cur_cmd = "_".join(segments)
            if cur_cmd in self._cmd_methods:
                argc = self._cmd_argc[cur_cmd]
                if argc is not None and len(args) != argc:
                    msg('command {0} expects {1} argument(s), got {2}',
                        " ".join(segments), argc, len(args))
                    return
                self._cmd_methods[cur_cmd](args)
                return
            args.insert(0, segments.pop())
        # invalid command
        prefix = '_'.join(args) + '_'
        matches = filter(
            lambda cmd: cmd.startswith(prefix), self._cmd_methods.keys())
        candidates = set([])
        for m in matches:
            if len(m) <= len(prefix):
                continue
            m = m[len(prefix):]
            if '_' in m:
                m = m[:m.index('_')]
            candidates.add(m)
        if len(candidates):
            msg('command "{0}" needs more arguments:', ' '.join(args))
            for c in candidates:
                msg('    {0}', c)
        else:
            msg('command "{0}" invalid; type ' +
                'help or use <TAB> for a list of commands',
                ' '.join(args))
546065
def handle_routine(self, que, opts, host, target, mine=False):
        '''
        
        '''
        opts = copy.deepcopy(opts)
        single = Single(
                opts,
                opts['argv'],
                host,
                mods=self.mods,
                fsclient=self.fsclient,
                thin=self.thin,
                mine=mine,
                **target)
        ret = {'id': single.id}
        stdout, stderr, retcode = single.run()
        # This job is done, yield
        try:
            data = salt.utils.json.find_json(stdout)
            if len(data) < 2 and 'local' in data:
                ret['ret'] = data['local']
            else:
                ret['ret'] = {
                    'stdout': stdout,
                    'stderr': stderr,
                    'retcode': retcode,
                }
        except Exception:
            ret['ret'] = {
                'stdout': stdout,
                'stderr': stderr,
                'retcode': retcode,
            }
        que.put(ret)
521667
def handle(self, *args, **options):
        """"""
        verbosity = int(options.get('verbosity'))

        # Check that the specified output format is valid
        if options['format'] != 'plain' and options['format'] != 'yaml':
            raise CommandError("Unknown output format: %s" % options['format'])

        # Gather only unique latest custom Docker requirements that the processes are using
        # The 'image' field is optional, so be careful about that as well
        unique_docker_images = set(
            p.requirements['executor']['docker']['image']
            for p in Process.objects.filter(is_active=True).order_by(
                'slug', '-version'
            ).distinct(
                'slug'
            ).only(
                'requirements'
            ).filter(
                requirements__icontains='docker'
            )
            if 'image' in p.requirements.get('executor', {}).get('docker', {})
        )

        # Add the default image.
        unique_docker_images.add(DEFAULT_CONTAINER_IMAGE)

        # Pull images if requested or just output the list in specified format
        if options['pull']:
            # Remove set of already pulled images.
            with PULLED_IMAGES_LOCK:
                unique_docker_images.difference_update(PULLED_IMAGES)

            # Get the desired 'docker' command from settings or use the default
            docker = getattr(settings, 'FLOW_DOCKER_COMMAND', 'docker')

            # Pull each image
            for img in unique_docker_images:
                ret = subprocess.call(
                    shlex.split('{} pull {}'.format(docker, img)),
                    stdout=None if verbosity > 0 else subprocess.DEVNULL,
                    stderr=None if verbosity > 0 else subprocess.DEVNULL,
                )

                # Update set of pulled images.
                with PULLED_IMAGES_LOCK:
                    PULLED_IMAGES.add(img)

                if ret != 0:
                    errmsg = "Failed to pull Docker image '{}'!".format(img)

                    if not options['ignore_pull_errors']:
                        # Print error and stop execution
                        raise CommandError(errmsg)
                    else:
                        # Print error, but keep going
                        logger.error(errmsg)
                        if verbosity > 0:
                            self.stderr.write(errmsg)
                else:
                    msg = "Docker image '{}' pulled successfully!".format(img)
                    logger.info(msg)
                    if verbosity > 0:
                        self.stdout.write(msg)
        else:
            # Sort the set of unique Docker images for nicer output.
            unique_docker_images = sorted(unique_docker_images)

            # Convert the set of unique Docker images into a list of dicts for easier output
            imgs = [
                dict(name=s[0], tag=s[1] if len(s) == 2 else 'latest')
                for s in (img.split(':') for img in unique_docker_images)
            ]

            # Output in YAML or plaintext (one image per line), as requested
            if options['format'] == 'yaml':
                out = yaml.safe_dump(imgs, default_flow_style=True, default_style="'")
            else:
                out = functools.reduce(operator.add,
                                       ('{name}:{tag}\n'.format(**i) for i in imgs), '')

            self.stdout.write(out, ending='')
131496
def _execute(cmd):
        """"""
        cmd_parts = shlex.split(cmd)
        if sys.version_info[0] < 3:

            cmd_parts = shlex.split(cmd.encode('ascii'))

        proc = subprocess.Popen(
            cmd_parts,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )
        out, err = proc.communicate()
        if proc.returncode != 0:

            raise subprocess.CalledProcessError(
                returncode=proc.returncode,
                cmd=cmd,
                output=err,
            )

        return CommandResult(
            code=proc.returncode,
            out=out.decode('utf8'),
            err=err.decode('utf8'),
        )