index traildbs directly from s3 for multiple accounts.

    context: assumes a daily traildb file in s3 with key path
             specified by key_template in config file for each account
552513
def index(config, start, end, incremental=False, concurrency=5, accounts=None,
          verbose=False):
    
    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)

    if verbose:
        logging.root.setLevel(logging.DEBUG)
    log.info("tmpdir %s" % os.environ.get('TMPDIR'))

    with ProcessPoolExecutor(max_workers=concurrency) as w:
        futures = {}

        if incremental:
            account_starts = get_incremental_starts(config, start)
        else:
            account_starts = defaultdict(lambda : start) # NOQA E203

        for account in config.get('accounts'):
            if accounts and account['name'] not in accounts:
                continue
            for region in account.get('regions'):
                for d in get_date_range(
                        account_starts[(account['name'], region)], end):
                    i = bool(d.hour or d.minute)
                    p = (config, region, account, d, i)
                    futures[w.submit(index_account, *p)] = p

        for f in as_completed(futures):
            _, region, account, d, incremental = futures[f]

            result = f.result()
            if result is None:
                continue
            log.info(
                ("processed account:%(account)s day:%(day)s region:%(region)s "
                 "records:%(records)s time:%(time)0.2f db-date:%(db-date)s"
                 ) % result)
967994
def create_trackers
      @trackers = Hash.new
      @accounts.each do |name, account|
        @log.debug "Setting up tracker for account #{name}"
        @trackers[name] = AccountTracker.new(name, account, {
            :delay => @delay, :error_callback => @error_proc, :logger => @log,
            :callback => Proc.new do |resources|
              # attach a QueryProcessor to all returned resources
              qp = FogTracker::Query::QueryProcessor.new(@trackers, :logger => @log)
              resources.each {|resource| resource._query_processor = qp}
              # now relay the resources back to the client software
              @callback.call(resources) if @callback
            end
          }
        )
      end
    end
275224
def index_all(self, index_name):
        
        oks = 0
        notoks = 0
        for ok, item in streaming_bulk(
            self.es_client,
            self._iter_documents(index_name)
        ):
            if ok:
                oks += 1
            else:
                notoks += 1
        logging.info(
            "Import results: %d ok, %d not ok",
            oks,
            notoks
        )
604028
def UseRangeIndexesOnStrings(client, database_id):
    
    try:
        DeleteContainerIfExists(client, database_id, COLLECTION_ID)
        database_link = GetDatabaseLink(database_id)
        # collections = Query_Entities(client, 'collection', parent_link = database_link)
        # print(collections)

        # Use range indexes on strings
        
        # This is how you can specify a range index on strings (and numbers) for all properties.
        # This is the recommended indexing policy for collections. i.e. precision -1
        #indexingPolicy = { 
        #    'indexingPolicy': {
        #        'includedPaths': [
        #            {
        #                'indexes': [
        #                    {
        #                        'kind': documents.IndexKind.Range,
        #                        'dataType': documents.DataType.String,
        #                        'precision': -1
        #                    }
        #                ]
        #            }
        #        ]
        #    }
        #}

        # For demo purposes, we are going to use the default (range on numbers, hash on strings) for the whole document (/* )
        # and just include a range index on strings for the "region".
        collection_definition = {
            'id': COLLECTION_ID,
            'indexingPolicy': {
                'includedPaths': [
                    {
                        'path': '/region/?',
                        'indexes': [
                            {
                                'kind': documents.IndexKind.Range,
                                'dataType': documents.DataType.String,
                                'precision': -1
                            }
                        ]
                    },
                    {
                        'path': '/*'
                    }
                ]
            }
        }

        created_Container = client.CreateContainer(database_link, collection_definition)
        print(created_Container)
        print("\n" + "-" * 25 + "\n6. Collection created with index policy")
        print_dictionary_items(created_Container["indexingPolicy"])

        collection_link = GetContainerLink(database_id, COLLECTION_ID)
        client.CreateItem(collection_link, { "id" : "doc1", "region" : "USA" })
        client.CreateItem(collection_link, { "id" : "doc2", "region" : "UK" })
        client.CreateItem(collection_link, { "id" : "doc3", "region" : "Armenia" })
        client.CreateItem(collection_link, { "id" : "doc4", "region" : "Egypt" })

        # Now ordering against region is allowed. You can run the following query
        query = { "query" : "SELECT * FROM r ORDER BY r.region" }
        message = "Documents ordered by region"
        QueryDocumentsWithCustomQuery(client, collection_link, query, message)

        # You can also perform filters against string comparison like >= 'UK'. Note that you can perform a prefix query, 
        # the equivalent of LIKE 'U%' (is >= 'U' AND < 'U')
        query = { "query" : "SELECT * FROM r WHERE r.region >= 'U'" }
        message = "Documents with region begining with U"
        QueryDocumentsWithCustomQuery(client, collection_link, query, message)

        # Cleanup
        client.DeleteContainer(collection_link)
        print("\n")
    except errors.HTTPFailure as e:
        if e.status_code == 409:
            print("Entity already exists")
        elif e.status_code == 404:
            print("Entity doesn't exist")
        else:
            raise
372228
def s3_upload(source_file_path, bucket_name, s3_access_key_id, s3_secret_key):
    
    key = s3_key(bucket_name, s3_access_key_id, s3_secret_key)
    file_name = source_file_path.split("/")[-1]
    key.key = file_name
    if key.exists():
        raise Exception("s3 key %s already exists for current period."
                        % (file_name))
    key.set_contents_from_filename(source_file_path)
482391
function routeAndCorrectedPathIndexes(routes, pathIndex) {
        for(var n=0; n<routes.length; n++) {
            var pathIndexes = routes[n].pathIndexes
            if(pathIndexes.start <= pathIndex && pathIndex <= pathIndexes.end || routes[n].route.isDefault) {
                return {routeIndex: n, pathIndex: pathIndexes.start}
            }
        }
        // else
        return {routeIndex: routes.length, pathIndex: pathIndex}
    }
675312
def cli(env):
    """"""

    mgr = SoftLayer.ObjectStorageManager(env.client)
    accounts = mgr.list_accounts()
    table = formatting.Table(['id', 'name', 'apiType'])
    table.sortby = 'id'
    api_type = None
    for account in accounts:
        if 'vendorName' in account and account['vendorName'] == 'Swift':
            api_type = 'Swift'
        elif 'Cleversafe' in account['serviceResource']['name']:
            api_type = 'S3'

        table.add_row([
            account['id'],
            account['username'],
            api_type,
        ])

    env.fout(table)
942064
private Map<String, ComponentDto> indexExistingDtosByUuids(DbSession session) {
    return dbClient.componentDao().selectAllComponentsFromProjectKey(session, treeRootHolder.getRoot().getDbKey())
      .stream()
      .collect(Collectors.toMap(ComponentDto::uuid, Function.identity()));
  }
843209
def accounts(dbpath, output, format, account,
             config=None, tag=None, tagprefix=None, region=(),
             not_region=(), not_bucket=None):
    """"""
    d = db.db(dbpath)
    accounts = d.accounts()
    formatter = (
        format == 'csv' and format_accounts_csv or format_accounts_plain)

    if region:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.region in region]
        accounts = [a for a in accounts if a.bucket_count]

    if not_region:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.region not in not_region]
        accounts = [a for a in accounts if a.bucket_count]

    if not_bucket:
        for a in accounts:
            a.buckets = [b for b in a.buckets if b.name not in not_bucket]
    if config and tagprefix:
        account_map = {account.name: account for account in accounts}

        with open(config) as fh:
            account_data = json.load(fh).get('accounts')
        tag_groups = {}
        for a in account_data:
            if tag is not None and tag not in a['tags']:
                continue

            for t in a['tags']:
                if t.startswith(tagprefix):
                    tvalue = t[len(tagprefix):]
                    if not tvalue:
                        continue
                    if tvalue not in tag_groups:
                        tag_groups[tvalue] = db.Account(tvalue, [])
                    account_results = account_map.get(a['name'])
                    if not account_results:
                        print("missing %s" % a['name'])
                        continue
                    tag_groups[tvalue].buckets.extend(
                        account_map[a['name']].buckets)
        accounts = tag_groups.values()

    formatter(accounts, output)
1287412
def s3(self, url, account_acessor=None, access=None, secret=None):
        """"""
        from ambry.util.ambrys3 import AmbryS3FS
        from ambry.util import parse_url_to_dict
        import ssl

        pd = parse_url_to_dict(url)

        if account_acessor:
            account = account_acessor(pd['hostname'])

            assert account['account_id'] == pd['hostname']
            aws_access_key = account['access_key'],
            aws_secret_key = account['secret']
        else:
            aws_access_key = access
            aws_secret_key = secret

        assert access, url
        assert secret, url

        s3 = AmbryS3FS(
            bucket=pd['netloc'],
            prefix=pd['path'].strip('/')+'/',
            aws_access_key=aws_access_key,
            aws_secret_key=aws_secret_key,

        )

        return s3
1372159
def sync(context, provider, **kwargs):  # pylint: disable=too-many-locals
    """"""
    session = get_session(provider.region)
    bucket_name = OutputLookup.handle(kwargs.get('bucket_output_lookup'),
                                      provider=provider,
                                      context=context)

    if context.hook_data['staticsite']['deploy_is_current']:
        LOGGER.info('staticsite: skipping upload; latest version already '
                    'deployed')
    else:
        distribution_id = OutputLookup.handle(
            kwargs.get('distributionid_output_lookup'),
            provider=provider,
            context=context
        )
        distribution_domain = OutputLookup.handle(
            kwargs.get('distributiondomain_output_lookup'),
            provider=provider,
            context=context
        )

        # Using the awscli for s3 syncing is incredibly suboptimal, but on
        # balance it's probably the most stable/efficient option for syncing
        # the files until https://github.com/boto/boto3/issues/358 is resolved
        aws_cli(['s3',
                 'sync',
                 context.hook_data['staticsite']['app_directory'],
                 "s3://%s/" % bucket_name,
                 '--delete'])

        cf_client = session.client('cloudfront')
        cf_client.create_invalidation(
            DistributionId=distribution_id,
            InvalidationBatch={'Paths': {'Quantity': 1, 'Items': ['/*']},
                               'CallerReference': str(time.time())}
        )
        LOGGER.info("staticsite: sync & CF invalidation of %s (domain %s) "
                    "complete",
                    distribution_id,
                    distribution_domain)

        if not context.hook_data['staticsite'].get('hash_tracking_disabled'):
            LOGGER.info("staticsite: updating environment SSM parameter %s "
                        "with hash %s",
                        context.hook_data['staticsite']['hash_tracking_parameter'],  # noqa
                        context.hook_data['staticsite']['hash'])
            ssm_client = session.client('ssm')
            ssm_client.put_parameter(
                Name=context.hook_data['staticsite']['hash_tracking_parameter'],  # noqa
                Description='Hash of currently deployed static website source',
                Value=context.hook_data['staticsite']['hash'],
                Type='String',
                Overwrite=True
            )
    LOGGER.info("staticsite: cleaning up old site archives...")
    archives = []
    s3_client = session.client('s3')
    list_objects_v2_paginator = s3_client.get_paginator('list_objects_v2')
    response_iterator = list_objects_v2_paginator.paginate(
        Bucket=context.hook_data['staticsite']['artifact_bucket_name'],
        Prefix=context.hook_data['staticsite']['artifact_key_prefix']
    )
    for page in response_iterator:
        archives.extend(page.get('Contents', []))
    archives_to_prune = get_archives_to_prune(
        archives,
        context.hook_data['staticsite']
    )
    # Iterate in chunks of 1000 to match delete_objects limit
    for objects in [archives_to_prune[i:i + 1000]
                    for i in range(0, len(archives_to_prune), 1000)]:
        s3_client.delete_objects(
            Bucket=context.hook_data['staticsite']['artifact_bucket_name'],
            Delete={'Objects': [{'Key': i} for i in objects]}
        )
    return True
593136
def stream_index(self, bucket, index, startkey, endkey=None,
                     return_terms=None, max_results=None, continuation=None,
                     timeout=None):
        
        raise NotImplementedError
847120
def _setup_source_conn(self, source_conn_id, source_bucket_name=None):
        
        self.source_conn = BaseHook.get_hook(source_conn_id)
        self.source_conn_id = source_conn_id

        # Workaround for getting hook in case of s3 connection
        # This is needed because get_hook silently returns None for s3 connections
        # See https://issues.apache.org/jira/browse/AIRFLOW-2316 for more info
        connection = BaseHook._get_connection_from_env(source_conn_id)
        self.log.info(connection.extra_dejson)
        if connection.conn_type == 's3':
            self.log.info("Setting up s3 connection {0}".format(source_conn_id))
            self.source_conn = S3Hook(aws_conn_id=source_conn_id)
            # End Workaround
            if source_bucket_name is None:
                raise AttributeError("Missing source bucket for s3 connection")
            self.source_bucket_name = source_bucket_name

        if not isinstance(self.source_conn, DbApiHook) and not isinstance(self.source_conn, S3Hook):
            raise AttributeError(
                "Only s3_csv, local and sql connection types are allowed, not {0}".format(type(self.source_conn)))
1270609
def run(self, *args, **kwargs):
        
        accounts = list(AWSAccount.get_all(include_disabled=False).values())

        # S3 Bucket config
        s3_acl = get_template('cloudtrail_s3_bucket_policy.json')
        s3_bucket_name = self.dbconfig.get('bucket_name', self.ns)
        s3_bucket_region = self.dbconfig.get('bucket_region', self.ns, 'us-west-2')
        s3_bucket_account = AWSAccount.get(self.dbconfig.get('bucket_account', self.ns))
        CloudTrail.create_s3_bucket(s3_bucket_name, s3_bucket_region, s3_bucket_account, s3_acl)

        self.validate_sqs_policy(accounts)

        for account in accounts:
            ct = CloudTrail(account, s3_bucket_name, s3_bucket_region, self.log)
            ct.run()
78901
def _get_s3_files(local_dir, file_info, params):
    
    assert len(file_info) == 1
    files = file_info.values()[0]
    fnames = []
    for k in ["1", "2"]:
        if files[k] not in fnames:
            fnames.append(files[k])
    out = []
    for fname in fnames:
        bucket, key = fname.replace("s3://", "").split("/", 1)
        if params["access_key_id"] == "TEST":
            out.append(os.path.join(local_dir, os.path.basename(key)))
        else:
            out.append(s3.get_file(local_dir, bucket, key, params))
    return out
448092
def _global_indexes_present(provisioned_indexes, global_indexes, changes_old,
                            changes_new, comments, name, region, key, keyid,
                            profile):
    ''''''
    ret = {'result': True}
    if provisioned_indexes:
        provisioned_gsi_config = dict((index['IndexName'], index) for index in provisioned_indexes)
    else:
        provisioned_gsi_config = {}
    provisioned_index_names = set(provisioned_gsi_config.keys())

    # Map of index name to given Salt config for this run. This loop is complicated
    # because global_indexes is made up of OrderedDicts and lists.
    gsi_config = {}
    if global_indexes:
        for index in global_indexes:
            # Each index config is a key that maps to a list of OrderedDicts.
            index_config = index.values()[0]
            index_name = None
            for entry in index_config:
                # Key by the name field in the index config.
                if entry.keys() == ['name']:
                    index_name = entry.values()[0]
            if not index_name:
                ret['result'] = False
                ret['comment'] = 'Index name not found for table {0}'.format(name)
                return ret
            gsi_config[index_name] = index

    existing_index_names, new_index_names, index_names_to_be_deleted = _partition_index_names(
        provisioned_index_names, set(gsi_config.keys()))

    if index_names_to_be_deleted:
        ret['result'] = False
        ret['comment'] = ('Deletion of GSIs ({0}) is not supported! Please do this '
                          'manually in the AWS console.'.format(', '.join(index_names_to_be_deleted)))
        return ret
    elif len(new_index_names) > 1:
        ret['result'] = False
        ret['comment'] = ('Creation of multiple GSIs ({0}) is not supported due to API '
                          'limitations. Please create them one at a time.'.format(new_index_names))
        return ret

    if new_index_names:
        # Given the length check above, new_index_names should have a single element here.
        index_name = next(iter(new_index_names))
        _add_global_secondary_index(ret, name, index_name, changes_old, changes_new, comments,
                                    gsi_config, region, key, keyid, profile)
        if not ret['result']:
            return ret

    if existing_index_names:
        _update_global_secondary_indexes(ret, changes_old, changes_new, comments,
                                         existing_index_names, provisioned_gsi_config, gsi_config,
                                         name, region, key, keyid, profile)
        if not ret['result']:
            return ret

    if 'global_indexes' not in changes_old and 'global_indexes' not in changes_new:
        comments.append('All global secondary indexes match')
    return ret
713887
def _setup_dest_conn(self, dest_conn_id, results_bucket_name, results_dest_name):
        
        conn = BaseHook._get_connection_from_env(dest_conn_id)
        if conn.conn_type != 's3':
            raise AttributeError(
                "Only s3 is allowed as a results destination, not {0}".format(conn.conn_type))

        self.dest_conn = S3Hook(aws_conn_id=dest_conn_id)
        self.dest_conn_id = dest_conn_id

        if results_bucket_name is None or results_dest_name is None:
            raise AttributeError("Specify bucket name and key name to store results")

        self.results_bucket_name = results_bucket_name
        self.results_dest_name = results_dest_name