run an aws script across accounts
483705
def run_script(config, output_dir, accounts, tags, region, echo, serial, script_args):
    """"""
    # TODO count up on success / error / error list by account
    accounts_config, custodian_config, executor = init(
        config, None, serial, True, accounts, tags, (), ())

    if echo:
        print("command to run: `%s`" % (" ".join(script_args)))
        return

    # Support fully quoted scripts, which are common to avoid parameter
    # overlap with c7n-org run-script.
    if len(script_args) == 1 and " " in script_args[0]:
        script_args = script_args[0].split()

    with executor(max_workers=WORKER_COUNT) as w:
        futures = {}
        for a in accounts_config.get('accounts', ()):
            for r in resolve_regions(region or a.get('regions', ())):
                futures[
                    w.submit(run_account_script, a, r, output_dir,
                             serial, script_args)] = (a, r)
        for f in as_completed(futures):
            a, r = futures[f]
            if f.exception():
                if serial:
                    raise
                log.warning(
                    "Error running script in %s @ %s exception: %s",
                    a['name'], r, f.exception())
            exit_code = f.result()
            if exit_code == 0:
                log.info(
                    "ran script on account:%s region:%s script: `%s`",
                    a['name'], r, " ".join(script_args))
            else:
                log.info(
                    "error running script on account:%s region:%s script: `%s`",
                    a['name'], r, " ".join(script_args))
636790
function awsPublisher() {
  if (!process.env.AWS_KEY || !process.env.AWS_SECRET) {
    throw 'AWS credentials are required!';
  }
  return aws.create({
      key: process.env.AWS_KEY,
      secret: process.env.AWS_SECRET,
      bucket: pkg.name
  });
}
1159367
function (account) {
	
	// get all local accounts & perform login
	var onDone = function (err, data) {
		if (err) {
			LOG.warn(err.toString());
		}
		else {
			for (var server in data.accounts) {
				LOG.warn('local server: ' + server);

				var user_list = data.accounts[server];
				for (var uid in user_list) {
					var token = user_list[uid];
					LOG.warn('local uid: ' + uid + ' token: ' + token);
					l_send_remote_login(server, uid, token);	
				}
			}
		}
	};
	SR.User.getUser(account, onDone);
}
1110730
function loadViaCredentialProcess(profile, callback) {
    proc.exec(profile['credential_process'], function(err, stdOut, stdErr) {
      if (err) {
        callback(AWS.util.error(
          new Error('credential_process returned error'),
          { code: 'ProcessCredentialsProviderFailure'}
        ), null);
      } else {
        try {
          var credData = JSON.parse(stdOut);
          if (credData.Expiration) {
            var currentTime = AWS.util.date.getDate();
            var expireTime = new Date(credData.Expiration);
            if (expireTime < currentTime) {
              throw Error('credential_process returned expired credentials');
            }
          }

          if (credData.Version !== 1) {
            throw Error('credential_process does not return Version == 1');
          }
          callback(null, credData);
        } catch (err) {
          callback(AWS.util.error(
            new Error(err.message),
            { code: 'ProcessCredentialsProviderFailure'}
          ), null);
        }
      }
    });
  }
1323670
def deploy(self, source_zip=None):
        

        if not source_zip:
            # Make sure we're in a venv.
            self.check_venv()

            # Execute the prebuild script
            if self.prebuild_script:
                self.execute_prebuild_script()

            # Make sure this isn't already deployed.
            deployed_versions = self.zappa.get_lambda_function_versions(self.lambda_name)
            if len(deployed_versions) > 0:
                raise ClickException("This application is " + click.style("already deployed", fg="red") +
                                     " - did you mean to call " + click.style("update", bold=True) + "?")

            # Make sure the necessary IAM execution roles are available
            if self.manage_roles:
                try:
                    self.zappa.create_iam_roles()
                except botocore.client.ClientError:
                    raise ClickException(
                        click.style("Failed", fg="red") + " to " + click.style("manage IAM roles", bold=True) + "!\n" +
                        "You may " + click.style("lack the necessary AWS permissions", bold=True) +
                        " to automatically manage a Zappa execution role.\n" +
                        "To fix this, see here: " +
                        click.style("https://github.com/Miserlou/Zappa#custom-aws-iam-roles-and-policies-for-deployment", bold=True)
                        + '\n')

            # Create the Lambda Zip
            self.create_package()
            self.callback('zip')

            # Upload it to S3
            success = self.zappa.upload_to_s3(
                    self.zip_path, self.s3_bucket_name, disable_progress=self.disable_progress)
            if not success: # pragma: no cover
                raise ClickException("Unable to upload to S3. Quitting.")

            # If using a slim handler, upload it to S3 and tell lambda to use this slim handler zip
            if self.stage_config.get('slim_handler', False):
                # https://github.com/Miserlou/Zappa/issues/510
                success = self.zappa.upload_to_s3(self.handler_path, self.s3_bucket_name, disable_progress=self.disable_progress)
                if not success:  # pragma: no cover
                    raise ClickException("Unable to upload handler to S3. Quitting.")

                # Copy the project zip to the current project zip
                current_project_name = '{0!s}_{1!s}_current_project.tar.gz'.format(self.api_stage, self.project_name)
                success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
                                                bucket_name=self.s3_bucket_name)
                if not success:  # pragma: no cover
                    raise ClickException("Unable to copy the zip to be the current project. Quitting.")

                handler_file = self.handler_path
            else:
                handler_file = self.zip_path

        # Fixes https://github.com/Miserlou/Zappa/issues/613
        try:
            self.lambda_arn = self.zappa.get_lambda_function(
                function_name=self.lambda_name)
        except botocore.client.ClientError:
            # Register the Lambda function with that zip as the source
            # You'll also need to define the path to your lambda_handler code.
            kwargs = dict(
                handler=self.lambda_handler,
                description=self.lambda_description,
                vpc_config=self.vpc_config,
                dead_letter_config=self.dead_letter_config,
                timeout=self.timeout_seconds,
                memory_size=self.memory_size,
                runtime=self.runtime,
                aws_environment_variables=self.aws_environment_variables,
                aws_kms_key_arn=self.aws_kms_key_arn,
                use_alb=self.use_alb
            )
            if source_zip and source_zip.startswith('s3://'):
                bucket, key_name = parse_s3_url(source_zip)
                kwargs['function_name'] = self.lambda_name
                kwargs['bucket'] = bucket
                kwargs['s3_key'] = key_name
            elif source_zip and not source_zip.startswith('s3://'):
                with open(source_zip, mode='rb') as fh:
                    byte_stream = fh.read()
                kwargs['function_name'] = self.lambda_name
                kwargs['local_zip'] = byte_stream
            else:
                kwargs['function_name'] = self.lambda_name
                kwargs['bucket'] = self.s3_bucket_name
                kwargs['s3_key'] = handler_file

            self.lambda_arn = self.zappa.create_lambda_function(**kwargs)

        # Schedule events for this deployment
        self.schedule()

        endpoint_url = ''
        deployment_string = click.style("Deployment complete", fg="green", bold=True) + "!"

        if self.use_alb:
            kwargs = dict(
                lambda_arn=self.lambda_arn,
                lambda_name=self.lambda_name,
                alb_vpc_config=self.alb_vpc_config,
                timeout=self.timeout_seconds
            )
            self.zappa.deploy_lambda_alb(**kwargs)

        if self.use_apigateway:

            # Create and configure the API Gateway
            template = self.zappa.create_stack_template(
                                                        lambda_arn=self.lambda_arn,
                                                        lambda_name=self.lambda_name,
                                                        api_key_required=self.api_key_required,
                                                        iam_authorization=self.iam_authorization,
                                                        authorizer=self.authorizer,
                                                        cors_options=self.cors,
                                                        description=self.apigateway_description,
                                                        endpoint_configuration=self.endpoint_configuration
                                                    )

            self.zappa.update_stack(
                                    self.lambda_name,
                                    self.s3_bucket_name,
                                    wait=True,
                                    disable_progress=self.disable_progress
                                )

            api_id = self.zappa.get_api_id(self.lambda_name)

            # Add binary support
            if self.binary_support:
                self.zappa.add_binary_support(api_id=api_id, cors=self.cors)

            # Add payload compression
            if self.stage_config.get('payload_compression', True):
                self.zappa.add_api_compression(
                    api_id=api_id,
                    min_compression_size=self.stage_config.get('payload_minimum_compression_size', 0))

            # Deploy the API!
            endpoint_url = self.deploy_api_gateway(api_id)
            deployment_string = deployment_string + ": {}".format(endpoint_url)

            # Create/link API key
            if self.api_key_required:
                if self.api_key is None:
                    self.zappa.create_api_key(api_id=api_id, stage_name=self.api_stage)
                else:
                    self.zappa.add_api_stage_to_api_key(api_key=self.api_key, api_id=api_id, stage_name=self.api_stage)

            if self.stage_config.get('touch', True):
                self.touch_endpoint(endpoint_url)

        # Finally, delete the local copy our zip package
        if not source_zip:
            if self.stage_config.get('delete_local_zip', True):
                self.remove_local_zip()

        # Remove the project zip from S3.
        if not source_zip:
            self.remove_uploaded_zip()

        self.callback('post')

        click.echo(deployment_string)
1235953
def run(self):
        """"""
        port, tensorboard_process = self.create_tensorboard_process()

        LOGGER.info('TensorBoard 0.1.7 at http://localhost:{}'.format(port))
        while not self.estimator.checkpoint_path:
            self.event.wait(1)
        with self._temporary_directory() as aws_sync_dir:
            while not self.event.is_set():
                args = ['aws', 's3', 'sync', self.estimator.checkpoint_path, aws_sync_dir]
                subprocess.call(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                self._sync_directories(aws_sync_dir, self.logdir)
                self.event.wait(10)
        tensorboard_process.terminate()
1234252
async def execute(self, keys=[], args=[], client=None):
        ""
        if client is None:
            client = self.registered_client
        args = tuple(keys) + tuple(args)
        # make sure the Redis server knows about the script
        if isinstance(client, BasePipeline):
            # make sure this script is good to go on pipeline
            client.scripts.add(self)
        try:
            return await client.evalsha(self.sha, len(keys), *args)
        except NoScriptError:
            # Maybe the client is pointed to a differnet server than the client
            # that created this instance?
            # Overwrite the sha just in case there was a discrepancy.
            self.sha = await client.script_load(self.script)
            return await client.evalsha(self.sha, len(keys), *args)
314966
def cli(execute, region, aws_access_key_id, aws_secret_access_key,
        s3_staging_dir, athenaclirc, profile, database):
    '''
    '''
    if (athenaclirc == ATHENACLIRC) and (not os.path.exists(os.path.expanduser(ATHENACLIRC))):
        err_msg = '''
        Welcome to athenacli!

        It seems this is your first time to run athenacli,
        we generated a default config file for you
            %s
        Please change it accordingly, and run athenacli again.
        ''' % ATHENACLIRC
        print(err_msg)
        write_default_config(DEFAULT_CONFIG_FILE, ATHENACLIRC)
        sys.exit(1)

    if profile != 'default':
        os.environ['AWS_PROFILE'] = profile

    athenacli = AthenaCli(
        region=region,
        aws_access_key_id=aws_access_key_id,
        aws_secret_access_key= aws_secret_access_key,
        s3_staging_dir=s3_staging_dir,
        athenaclirc=athenaclirc,
        profile=profile,
        database=database
    )

    #  --execute argument
    if execute:
        if os.path.exists(execute):
            with open(execute) as f:
                query = f.read()
        else:
            query = execute
        try:
            athenacli.formatter.format_name = 'csv'
            athenacli.run_query(query)
            exit(0)
        except Exception as e:
            click.secho(str(e), err=True, fg='red')
            exit(1)

    athenacli.run_cli()
186801
def RunOnce(self):
    """"""
    global WEBAUTH_MANAGER  # pylint: disable=global-statement

    # pylint: disable=g-bad-name
    WEBAUTH_MANAGER = BaseWebAuthManager.GetPlugin(
        config.CONFIG["AdminUI.webauth_manager"])()

    # pylint: enable=g-bad-name
    logging.info("Using webauth manager %s", WEBAUTH_MANAGER)
384955
def main():
    """"""
    parser = ArgumentParser()
    subs = parser.add_subparsers(dest='cmd')
    setup_parser = subs.add_parser('setup')
    setup_parser.add_argument('-r', '--region', default=PRIMARY_REGION,
                              help='AWS region to install all services')
    setup_parser.add_argument('-d', '--debug', action='store_true',
                              help='Run in debug mode')
    setup_parser.add_argument('--rollback', action='store_true',
                              help='Rollback configuration on failure')
    setup_parser.add_argument('--skip-node-setup', action='store_true',
                              help='Rollback configuration on failure')
    setup_parser = subs.add_parser('teardown')
    setup_parser.add_argument('-r', '--region', default=PRIMARY_REGION,
                              help='AWS region to delete all services')
    setup_parser.add_argument('-d', '--debug', action='store_true',
                              help='Run in debug mode')
    args = parser.parse_args()

    if args.region:
        if args.region not in SUPPORTED_REGIONS:
            raise Exception("INVALID_REGION: Region must be one of: %s"
                            % (', '.join(SUPPORTED_REGIONS)))
        global PRIMARY_REGION
        PRIMARY_REGION = args.region

    if args.debug:
        logger.setLevel(logging.DEBUG)

    if args.cmd == 'setup':
        try:
            generate_handler()
            time.sleep(7)
            generate_s3_bucket()
            generate_dynamodb_tables()
            generate_lambda_functions()
            generate_api_gateway()
            generate_admin_resource()
            api_node = generate_api_resources()
        except Exception as e:
            logger.error(str(e))
            if args.rollback:
                logger.info("[!] Rolling back deployed services")
                remove_s3_bucket()
                remove_dynamodb_tables()
                remove_lambda_functions()
                remove_api_gateway()
                remove_handler()

        if not args.skip_node_setup:
            email = input("[*] Blockade admin email: ")
            if email.find('@') < 0:
                logger.error("[!] %s does not appear to be an email" % email)
                email = input("[*] Blockade admin email: ")
            name = input("[*] Blockade admin name: ")

            url = api_node + "admin/add-user"
            params = {'user_email': email, 'user_name': name,
                      'user_role': 'admin'}
            headers = {'Content-Type': 'application/json'}
            response = requests.post(url, data=json.dumps(params),
                                     headers=headers)
            loaded = json.loads(response.content)
            logger.info("[#] Successfully added Blockade admin")
            logger.debug(loaded)
            print("---------------------------------------------------------------------------")
            print("Blockade Server: %s" % api_node)
            print("Blockade Admin: %s" % loaded['email'])
            print("Blockade API Key: %s" % loaded['api_key'])
            print("---------------------------------------------------------------------------")
            print("Quick Config: blockade-cfg setup %s %s --api-node=%s"
                  % (loaded['email'], loaded['api_key'], api_node))
            print("Next steps: Add indicators to your node with our analyst toolbench: %s" % (ANALYST_TOOLBENCH))
            print("For more documentation on other cloud node actions, see %s." % (CLOUD_NODE_DOCS))

    if args.cmd == 'teardown':
        remove_s3_bucket()
        remove_dynamodb_tables()
        remove_lambda_functions()
        remove_api_gateway()
        remove_handler()
1374324
def app_run(app_name_or_id, alias=None, input_params={}, always_retry=True, **kwargs):
    
    input_params_cp = Nonce.update_nonce(input_params)
    fully_qualified_version = app_name_or_id + (('/' + alias) if alias else '')
    return DXHTTPRequest('/%s/run' % fully_qualified_version, input_params_cp, always_retry=always_retry, **kwargs)
479736
def run():
    """"""
    args = parse_args()

    appname = sys.argv[0]
    hostname = platform.node()

    codetools.setup_logging(args.debug)

    password = ''

    if args.token_path is None and args.delete_role is True:
        cred_path = os.path.expanduser('~/.sq_github_token_delete')
    elif args.token_path is None and args.delete_role is False:
        cred_path = os.path.expanduser('~/.sq_github_token')
    else:
        cred_path = os.path.expandvars(os.path.expanduser(args.token_path))

    if not os.path.isfile(cred_path):
        print("""
        Type in your password to get an auth token from github
        It will be stored in {0}
        and used in subsequent occasions.
        """.format(cred_path))

        while not password:
            password = getpass('Password for {0}: '.format(args.user))

        note = textwrap.dedent("""\
            {app} via bored^H^H^H^H^H terrified opossums[1]
            on {host}
            by {user} {creds}
            [1] https://youtu.be/ZtLrn2zPTxQ?t=1m10s
            """).format(
            app=appname,
            host=hostname,
            user=args.user,
            creds=cred_path
        )
        note_url = 'https://www.youtube.com/watch?v=cFvijBpzD_Y'

        if args.delete_role:
            scopes = ['repo', 'user', 'delete_repo', 'admin:org']
        else:
            scopes = ['repo', 'user']

        global g
        g = github.Github(args.user, password)
        u = g.get_user()

        try:
            auth = u.create_authorization(
                scopes=scopes,
                note=note,
                note_url=note_url,
            )
        except github.TwoFactorException:
            auth = u.create_authorization(
                scopes=scopes,
                note=note,
                note_url=note_url,
                # not a callback
                onetime_password=codetools.github_2fa_callback()
            )
        g = github.Github(auth.token)

        with open(cred_path, 'w') as fdo:
            fdo.write(auth.token + '\n')
            fdo.write(str(auth.id))

        print('Token written to {0}'.format(cred_path))

    else:
        print("You already have an auth file: {0} ".format(cred_path))
        print("Delete it if you want a new one and run again")
        print("Remember to also remove the corresponding token on Github")
942751
def run_script(dist_spec, script_name):
    """"""
    ns = sys._getframe(1).f_globals
    name = ns['__name__']
    ns.clear()
    ns['__name__'] = name
    require(dist_spec)[0].run_script(script_name, ns)
338673
def sync(config, group, accounts=(), dryrun=False, region=None):
    
    config = validate.callback(config)
    destination = config.get('destination')
    client = boto3.Session().client('s3')

    for account in config.get('accounts', ()):
        if accounts and account['name'] not in accounts:
            continue

        session = get_session(account['role'], region)
        account_id = session.client('sts').get_caller_identity()['Account']
        prefix = destination.get('prefix', '').rstrip('/') + '/%s' % account_id
        prefix = "%s/%s" % (prefix, group)

        exports = get_exports(client, destination['bucket'], prefix + "/")

        role = account.pop('role')
        if isinstance(role, six.string_types):
            account['account_id'] = role.split(':')[4]
        else:
            account['account_id'] = role[-1].split(':')[4]
        account.pop('groups')

        if exports:
            last_export = exports.pop()
            account['export'] = last_export
        else:
            account['export'] = 'missing'
            last_export = None
        try:
            tag_set = client.get_object_tagging(
                Bucket=destination['bucket'], Key=prefix).get('TagSet', [])
        except ClientError:
            tag_set = []

        tags = {t['Key']: t['Value'] for t in tag_set}
        tagged_last_export = None

        if 'LastExport' in tags:
            le = parse(tags['LastExport'])
            tagged_last_export = (le.year, le.month, le.day)
            account['sync'] = tagged_last_export
        else:
            account['sync'] = account['export'] != 'missing' and 'sync' or 'missing'

        if last_export is None:
            continue

        if tagged_last_export == last_export or account['export'] == 'missing':
            continue

        if dryrun:
            continue

        client.put_object(
            Bucket=destination['bucket'],
            Key=prefix,
            Body=json.dumps({}),
            ACL="bucket-owner-full-control",
            ServerSideEncryption="AES256")

        export_time = datetime.now().replace(tzinfo=tzlocal()).astimezone(tzutc())
        export_time = export_time.replace(
            year=last_export[0], month=last_export[1], day=last_export[2],
            minute=0, second=0, microsecond=0, hour=0)
        client.put_object_tagging(
            Bucket=destination['bucket'], Key=prefix,
            Tagging={
                'TagSet': [{
                    'Key': 'LastExport',
                    'Value': export_time.isoformat()}]})

    accounts_report = []
    for a in config.get('accounts'):
        if accounts and a['name'] not in accounts:
            continue
        if isinstance(a['sync'], tuple):
            a['sync'] = "%s/%s/%s" % (a['sync'])
        if isinstance(a['export'], tuple):
            a['export'] = "%s/%s/%s" % (a['export'])
        accounts_report.append(a)

    accounts_report.sort(key=operator.itemgetter('export'), reverse=True)
    print(tabulate(accounts_report, headers='keys'))
621297
def run_job(clusterID, packageID, args = {})
      query = args.inject({}) {|v, kv|
        v.merge(
          {
            "job[variables][#{kv.first}]" => kv.last
          })
      }
      query = query.merge(
        {
          'job[cluster_id]' => clusterID,
          'job[package_id]' => packageID
        })

      request(
        :expects => 201,
        :method => :post,
        :path => "/#{account_id}/api/jobs",
        :query => query
      )
    end
1044229
def run_script(self, filename, wdir, args, debug, post_mortem,
                   current_client, clear_variables):
        """"""
        norm = lambda text: remove_backslashes(to_text_string(text))

        # Run Cython files in a dedicated console
        is_cython = osp.splitext(filename)[1] == '.pyx'
        if is_cython:
            current_client = False

        # Select client to execute code on it
        is_new_client = False
        if current_client:
            client = self.get_current_client()
        else:
            client = self.get_client_for_file(filename)
            if client is None:
                self.create_client_for_file(filename, is_cython=is_cython)
                client = self.get_current_client()
                is_new_client = True

        if client is not None:
            # Internal kernels, use runfile
            if client.get_kernel() is not None:
                line = "%s('%s'" % ('debugfile' if debug else 'runfile',
                                    norm(filename))
                if args:
                    line += ", args='%s'" % norm(args)
                if wdir:
                    line += ", wdir='%s'" % norm(wdir)
                if post_mortem:
                    line += ", post_mortem=True"
                line += ")"
            else: # External kernels, use %run
                line = "%run "
                if debug:
                    line += "-d "
                line += "\"%s\"" % to_text_string(filename)
                if args:
                    line += " %s" % norm(args)

            try:
                if client.shellwidget._executing:
                    # Don't allow multiple executions when there's
                    # still an execution taking place
                    # Fixes issue 7293
                    pass
                elif client.shellwidget._reading:
                    client.shellwidget._append_html(
                        _("<br><b>Please exit from debugging before trying to "
                          "run a file in this console.</b>\n<hr><br>"),
                        before_prompt=True)
                    return
                elif current_client:
                    self.execute_code(line, current_client, clear_variables)
                else:
                    if is_new_client:
                        client.shellwidget.silent_execute('%clear')
                    else:
                        client.shellwidget.execute('%clear')
                    client.shellwidget.sig_prompt_ready.connect(
                            lambda: self.execute_code(line, current_client,
                                                      clear_variables))
            except AttributeError:
                pass
            self.switch_to_plugin()
        else:
            #XXX: not sure it can really happen
            QMessageBox.warning(self, _('Warning'),
                _("No IPython console is currently available to run <b>%s</b>."
                  "<br><br>Please open a new one and try again."
                  ) % osp.basename(filename), QMessageBox.Ok)
1229104
def run(self, conn, tmp, module_name, module_args, inject):
        '''  '''

        tokens  = shlex.split(module_args)
        source  = tokens[0]
        # FIXME: error handling
        args    = " ".join(tokens[1:])
        source  = utils.template(self.runner.basedir, source, inject)
        source  = utils.path_dwim(self.runner.basedir, source)

        # transfer the file to a remote tmp location
        source  = source.replace('\x00','') # why does this happen here?
        args    = args.replace('\x00','') # why does this happen here?
        tmp_src = os.path.join(tmp, os.path.basename(source))
        tmp_src = tmp_src.replace('\x00', '') 

        conn.put_file(source, tmp_src)

        # fix file permissions when the copy is done as a different user
        if self.runner.sudo and self.runner.sudo_user != 'root':
            prepcmd = 'chmod a+rx %s' % tmp_src
        else:
            prepcmd = 'chmod +x %s' % tmp_src

        # add preparation steps to one ssh roundtrip executing the script
        module_args = prepcmd + '; ' + tmp_src + ' ' + args

        handler = utils.plugins.action_loader.get('raw', self.runner)
        result = handler.run(conn, tmp, 'raw', module_args, inject)

        # clean up after
        if tmp.find("tmp") != -1 and C.DEFAULT_KEEP_REMOTE_FILES != '1':
            self.runner._low_level_exec_command(conn, 'rm -rf %s >/dev/null 2>&1' % tmp, tmp)

        return result