Run Fabric commands against EC2 instances
436219
def ec2_fab(service, args):
    
    instance_ids = args.instances
    instances = service.list(elb=args.elb, instance_ids=instance_ids)
    hosts = service.resolve_hosts(instances)

    fab.env.hosts = hosts
    fab.env.key_filename = settings.get('SSH', 'KEY_FILE')
    fab.env.user = settings.get('SSH', 'USER', getpass.getuser())
    fab.env.parallel = True

    fabfile = find_fabfile(args.file)
    if not fabfile:
        print 'Couldn\'t find any fabfiles!'
        return

    fab.env.real_fabile = fabfile
    docstring, callables, default = load_fabfile(fabfile)
    fab_state.commands.update(callables)

    commands_to_run = parse_arguments(args.methods)
    for name, args, kwargs, arg_hosts, arg_roles, arg_exclude_hosts in commands_to_run:
        fab.execute(name,
                    hosts=arg_hosts,
                    roles=arg_roles,
                    exclude_hosts=arg_exclude_hosts,
                    *args, **kwargs)
609381
def _run_command(self, packet):
        
        create_temp_cli = self.client_states is None

        if create_temp_cli:
            # If this client doesn't have a CLI. Create a Fake CLI where the
            # window containing this pane, is the active one. (The CLI instance
            # will be removed before the render function is called, so it doesn't
            # hurt too much and makes the code easier.)
            pane_id = int(packet['pane_id'])
            self._create_app()
            with set_app(self.client_state.app):
                self.pymux.arrangement.set_active_window_from_pane_id(pane_id)

        with set_app(self.client_state.app):
            try:
                self.pymux.handle_command(packet['data'])
            finally:
                self._close_connection()
103285
def create_ssh_command(session, instance_id, instance_name, username, key_file, port, ssh_options,
                       use_private_ip, gateway_instance_id, gateway_username):
    """"""
    ec2 = session.resource('ec2')
    if instance_id is not None:
        try:
            instance = ec2.Instance(instance_id)
            hostname = _get_instance_ip_address(instance, use_private_ip)
        except botocore.exceptions.ClientError as e:
            click.echo("Invalid instance ID {0} ({1})".format(instance_id, e), err=True)
            sys.exit(2)
    elif instance_name is not None:
        try:
            conditions = [
                {'Name': 'tag:Name', 'Values': [instance_name]},
                {'Name': 'instance-state-name', 'Values': ['running']},
            ]
            instances = ec2.instances.filter(Filters=conditions)
            target_instances = []
            for idx, i in enumerate(instances):
                target_instances.append(i)
            if len(target_instances) == 1:
                instance = target_instances[0]
                hostname = _get_instance_ip_address(instance, use_private_ip)
            else:
                for idx, i in enumerate(instances):
                    tag_name = get_tag_value(i.tags, 'Name')
                    click.echo('[{0}]: {1}\t{2}\t{3}\t{4}\t{5}'.format(
                        idx, i.id, i.public_ip_address, i.state['Name'], tag_name, i.key_name))
                selected_idx = click.prompt("Please enter a valid number", type=int, default=0)
                if len(target_instances) - 1 < selected_idx or selected_idx < 0:
                    click.echo("selected number [{0}] is invalid".format(selected_idx), err=True)
                    sys.exit(2)
                click.echo("{0} is selected.".format(selected_idx))
                instance = target_instances[selected_idx]
                hostname = _get_instance_ip_address(instance, use_private_ip)
        except botocore.exceptions.ClientError as e:
            click.echo("Invalid instance ID {0} ({1})".format(instance_id, e), err=True)
            sys.exit(2)
    # TODO: need to refactor and make it testable
    if key_file is None:
        key_file_option = ''
    else:
        key_file_option = ' -i {0}'.format(key_file)

    gateway_username_option = build_option_username(gateway_username)
    username_option = build_option_username(username)

    if ssh_options is None:
        ssh_options = ''
    else:
        ssh_options = ' {0}'.format(ssh_options)
    if gateway_instance_id is not None:
        gateway_instance = ec2.Instance(gateway_instance_id)
        gateway_public_ip = gateway_instance.public_ip_address
        hostname = instance.private_ip_address
        cmd = 'ssh -tt{0} {1}{2} -p {3}{4} ssh{5} {6}'.format(
            gateway_username_option, gateway_public_ip, key_file_option,
            port, ssh_options, username_option, hostname)
    else:
        cmd = 'ssh{0} {1}{2} -p {3}{4}'.format(username_option, hostname, key_file_option, port, ssh_options)
    return cmd
1356321
def remote_run(cmd, instance_name, detach=False, retries=1):
  """"""
  if detach:
    cmd = SCREEN.format(command=cmd)
  args = SSH.format(instance_name=instance_name).split()
  args.append(cmd)
  for i in range(retries + 1):
    try:
      if i > 0:
        tf.logging.info("Retry %d for %s", i, args)
      return sp.check_call(args)
    except sp.CalledProcessError as e:
      if i == retries:
        raise e
423904
def RunCommandOnHosts(cmd, hostnames, ssh_key):
  
  if not hostnames:
    return
  p = multiprocessing.Pool(1)
  remote_command_args = [(cmd, hostname, ssh_key) for hostname in hostnames]
  #print remote_command_args
  result = None
  while not result:
    try:    
      result = p.map_async(__RemoteExecuteHelper, remote_command_args).get(999999) # allows for keyboard interrupt
    except paramiko.SSHException as e:      
      logging.info( 'failure in RunCommandOnHosts: %s' % (e))
      time.sleep(5)
    
  p.close()
  p.join()
  return result
534715
def create_server_ec2(connection,
                      region,
                      disk_name,
                      disk_size,
                      ami,
                      key_pair,
                      instance_type,
                      tags={},
                      security_groups=None,
                      delete_on_termination=True,
                      log=False,
                      wait_for_ssh_available=True):
    

    if log:
        log_green("Started...")
        log_yellow("...Creating EC2 instance...")

    ebs_volume = EBSBlockDeviceType()
    ebs_volume.size = disk_size
    bdm = BlockDeviceMapping()
    bdm[disk_name] = ebs_volume

    # get an ec2 ami image object with our choosen ami
    image = connection.get_all_images(ami)[0]
    # start a new instance
    reservation = image.run(1, 1,
                            key_name=key_pair,
                            security_groups=security_groups,
                            block_device_map=bdm,
                            instance_type=instance_type)

    # and get our instance_id
    instance = reservation.instances[0]

    #  and loop and wait until ssh is available
    while instance.state == u'pending':
        if log:
            log_yellow("Instance state: %s" % instance.state)
        sleep(10)
        instance.update()
    if log:
        log_green("Instance state: %s" % instance.state)
    if wait_for_ssh_available:
        wait_for_ssh(instance.public_dns_name)

    # update the EBS volumes to be deleted on instance termination
    if delete_on_termination:
        for dev, bd in instance.block_device_mapping.items():
            instance.modify_attribute('BlockDeviceMapping',
                                      ["%s=%d" % (dev, 1)])

    # add a tag to our instance
    connection.create_tags([instance.id], tags)

    if log:
        log_green("Public dns: %s" % instance.public_dns_name)

    # returns our new instance
    return instance
1229104
def run(self, conn, tmp, module_name, module_args, inject):
        '''  '''

        tokens  = shlex.split(module_args)
        source  = tokens[0]
        # FIXME: error handling
        args    = " ".join(tokens[1:])
        source  = utils.template(self.runner.basedir, source, inject)
        source  = utils.path_dwim(self.runner.basedir, source)

        # transfer the file to a remote tmp location
        source  = source.replace('\x00','') # why does this happen here?
        args    = args.replace('\x00','') # why does this happen here?
        tmp_src = os.path.join(tmp, os.path.basename(source))
        tmp_src = tmp_src.replace('\x00', '') 

        conn.put_file(source, tmp_src)

        # fix file permissions when the copy is done as a different user
        if self.runner.sudo and self.runner.sudo_user != 'root':
            prepcmd = 'chmod a+rx %s' % tmp_src
        else:
            prepcmd = 'chmod +x %s' % tmp_src

        # add preparation steps to one ssh roundtrip executing the script
        module_args = prepcmd + '; ' + tmp_src + ' ' + args

        handler = utils.plugins.action_loader.get('raw', self.runner)
        result = handler.run(conn, tmp, 'raw', module_args, inject)

        # clean up after
        if tmp.find("tmp") != -1 and C.DEFAULT_KEEP_REMOTE_FILES != '1':
            self.runner._low_level_exec_command(conn, 'rm -rf %s >/dev/null 2>&1' % tmp, tmp)

        return result
546078
def execute_as(collector):
    """"""
    # Gonna assume role anyway...
    collector.configuration['amazon']._validated = True

    # Find the arn we want to assume
    account_id = collector.configuration['accounts'][collector.configuration['aws_syncr'].environment]
    arn = "arn:aws:iam::{0}:role/{1}".format(account_id, collector.configuration['aws_syncr'].artifact)

    # Determine the command to run
    parts = shlex.split(collector.configuration["aws_syncr"].extra)
    if not parts:
        suggestion = " ".join(sys.argv) + " -- /path/to/command_to_run"
        msg = "No command was provided. Try something like:\n\t\t{0}".format(suggestion)
        raise AwsSyncrError(msg)

    # Get our aws credentials environment variables from the assumed role
    env = dict(os.environ)
    env.update(collector.configuration['amazon'].iam.assume_role_credentials(arn))

    # Turn into the command we want to execute
    os.execvpe(parts[0], parts, env)
1235953
def run(self):
        """"""
        port, tensorboard_process = self.create_tensorboard_process()

        LOGGER.info('TensorBoard 0.1.7 at http://localhost:{}'.format(port))
        while not self.estimator.checkpoint_path:
            self.event.wait(1)
        with self._temporary_directory() as aws_sync_dir:
            while not self.event.is_set():
                args = ['aws', 's3', 'sync', self.estimator.checkpoint_path, aws_sync_dir]
                subprocess.call(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                self._sync_directories(aws_sync_dir, self.logdir)
                self.event.wait(10)
        tensorboard_process.terminate()
1205156
def run():
    """"""
    
    # Grab server args
    args = client_helper.grab_server_args()

    # Start up workbench connection
    workbench = zerorpc.Client(timeout=300, heartbeat=60)
    workbench.connect('tcp://'+args['server']+':'+args['port'])

    # Test out zip data
    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),'../data/zip')
    file_list = [os.path.join(data_path, child) for child in os.listdir(data_path)]
    for filename in file_list:
        with open(filename,'rb') as f:
            base_name = os.path.basename(filename)
            md5 = workbench.store_sample(f.read(), base_name, 'zip')
            results = workbench.work_request('view', md5)
            print 'Filename: %s ' % (base_name)
            pprint.pprint(results)

            # The unzip worker gives you a list of md5s back
            # Run meta on all the unzipped files.
            results = workbench.work_request('unzip', md5)
            print '\n*** Filename: %s ***' % (base_name)
            for child_md5 in results['unzip']['payload_md5s']:
                pprint.pprint(workbench.work_request('meta', child_md5))
891629
def run(self, command):
        
        boto.log.debug('running:%s on %s' % (command, self.server.instance_id))
        status = 0
        try:
            t = self._ssh_client.exec_command(command)
        except paramiko.SSHException:
            status = 1
        std_out = t[1].read()
        std_err = t[2].read()
        t[0].close()
        t[1].close()
        t[2].close()
        boto.log.debug('stdout: %s' % std_out)
        boto.log.debug('stderr: %s' % std_err)
        return (status, std_out, std_err)
361730
def perform(action_name, container, **kwargs):
    
    cf = container_fabric()
    cf.call(action_name, container, **kwargs)
1206266
def run(conf, only):
    
    with errorprint():
        config = ConfModule(conf)
        spawned = config.spawn_uwsgi(only)

        for alias, pid in spawned:
            click.secho("Spawned uWSGI for configuration aliased '%s'. PID %s" % (alias, pid), fg='green')
538034
def execute_ssh_command(self, client, command):
        """"""
        try:
            out = ipa_utils.execute_ssh_command(client, command)
        except Exception as error:
            raise IpaCloudException(
                'Command: "{0}", failed execution: {1}.'.format(
                    command, error
                )
            )
        else:
            self._write_to_log(out)
120746
def run():
    """"""

    # Use print() instead of logging when running in CLI mode:
    set_pyftpsync_logger(None)

    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync",
        parents=[verbose_parser],
    )

    # Note: we want to allow --version to be combined with --verbose. However
    # on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used.
    if check_cli_verbose(3) > 3:
        version_info = "pyftpsync/{} Python/{} {}".format(
            __version__, PYTHON_VERSION, platform.platform()
        )
    else:
        version_info = "{}".format(__version__)

    parser.add_argument("-V", "--version", action="version", version=version_info)

    subparsers = parser.add_subparsers(help="sub-command help")

    # --- Create the parser for the "upload" command ---------------------------

    sp = subparsers.add_parser(
        "upload",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files to remote folder",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite remote files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["local", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove remote files if they don't exist locally",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove remote files if they don't exist locally "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="upload")

    # --- Create the parser for the "download" command -------------------------

    sp = subparsers.add_parser(
        "download",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files from remote folder to local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite local files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove local files if they don't exist on remote target",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove local files if they don't exist on remote target "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="download")

    # --- Create the parser for the "sync" command -----------------------------

    sp = subparsers.add_parser(
        "sync",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="synchronize new and modified files between remote folder and local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["old", "new", "local", "remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )

    sp.set_defaults(command="sync")

    # --- Create the parser for the "run" command -----------------------------

    add_run_parser(subparsers)

    # --- Create the parser for the "scan" command -----------------------------

    add_scan_parser(subparsers)

    # --- Parse command line ---------------------------------------------------

    args = parser.parse_args()

    args.verbose -= args.quiet
    del args.quiet

    # print("verbose", args.verbose)

    ftp_debug = 0
    if args.verbose >= 6:
        ftp_debug = 1

    # Modify the `args` from the `pyftpsync.yaml` config:
    if getattr(args, "command", None) == "run":
        handle_run_command(parser, args)

    if callable(getattr(args, "command", None)):
        # scan_handler
        try:
            return args.command(parser, args)
        except KeyboardInterrupt:
            print("\nAborted by user.", file=sys.stderr)
            sys.exit(3)

    elif not hasattr(args, "command"):
        parser.error(
            "missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')"
        )

    # Post-process and check arguments
    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True

    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})

    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(
        args.remote_target, FsTarget
    ):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "sync":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command '{}'".format(args.command))

    s.is_script = True

    try:
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.", file=sys.stderr)
        sys.exit(3)
    finally:
        # Prevent sporadic exceptions in ftplib, when closing in __del__
        s.local.close()
        s.remote.close()

    stats = s.get_stats()
    if args.verbose >= 5:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print(
            "Wrote {}/{} files in {} directories, skipped: {}.".format(
                stats["files_written"],
                stats["local_files"],
                stats["local_dirs"],
                stats["conflict_files_skipped"],
            ),
            end="",
        )
        if stats["interactive_ask"]:
            print()
        else:
            print(" Elap: {}.".format(stats["elap_str"]))

    return
939188
def validate_ec2_action(self):
        
        import_cmd = 'aws ec2 import-image --dry-run --profile {} --region {}'\
            .format(self.aws_project, self.aws_regions[0])
        print "Attempting ec2 import dry run: {}".format(import_cmd)
        try:
            subprocess.check_output(shlex.split(import_cmd), stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            if "(DryRunOperation)" in e.output:
                # If it failed because of a dry run (what we asked for) then this except can be ignored
                print "Dry run operation successful!"
                return
            print "Error: {}".format(e.output)
            print "It doesn't seem like your user has the required permissions to import an ami image from s3"
            sys.exit(5)
543532
def Launcher(self, config=None):
        """"""
        class _launcher(EC2ApiClient):
            """Configurable launcher for EC2 instances. Create the Launcher
            (passing an optional dict of its attributes), set its attributes
            (as described in the RunInstances API docs), then launch().
            """
            def __init__(self, aws, config):
                super(_launcher, self).__init__(aws)
                self.config = config
                self._attr = list(self.__dict__.keys()) + ['_attr']

            def launch(self, ami, min_count, max_count=0):
                """Use given AMI to launch min_count instances with the
                current configuration. Returns instance info list.
                """
                params = config.copy()
                params.update(dict([i for i in self.__dict__.items()
                                    if i[0] not in self._attr]))
                return self.call("RunInstances",
                                 ImageId=ami,
                                 MinCount=min_count,
                                 MaxCount=max_count or min_count,
                                 response_data_key="Instances",
                                 **params)

        if not config:
            config = {}
        return _launcher(self._aws, config)