Retrieve remote resources like GATK/MuTect jars present in S3.
546766
def _add_remote_resources(resources):
    
    out = copy.deepcopy(resources)
    for prog, info in resources.items():
        for key, val in info.items():
            if key == "jar" and objectstore.is_remote(val):
                store_dir = utils.safe_makedir(os.path.join(os.getcwd(), "inputs", "jars", prog))
                fname = objectstore.download(val, store_dir, store_dir)
                version_file = os.path.join(store_dir, "version.txt")
                if not utils.file_exists(version_file):
                    version = install.get_gatk_jar_version(prog, fname)
                    with open(version_file, "w") as out_handle:
                        out_handle.write(version)
                else:
                    with open(version_file) as in_handle:
                        version = in_handle.read().strip()
                del out[prog][key]
                out[prog]["dir"] = store_dir
                out[prog]["version"] = version
    return out
331346
def _get_client(self):
        
        client_kwargs = self._storage_parameters.get('client', dict())

        # Handles unsecure mode
        if self._unsecure:
            client_kwargs = client_kwargs.copy()
            client_kwargs['use_ssl'] = False

        return self._get_session().client("s3", **client_kwargs)
837970
def get_file_client(opts, pillar=False):
    '''
    
    '''
    client = opts.get('file_client', 'remote')
    if pillar and client == 'local':
        client = 'pillar'
    return {
        'remote': RemoteClient,
        'local': FSClient,
        'pillar': PillarClient,
    }.get(client, RemoteClient)(opts)
144721
def get_s3_client():
    
    session_kwargs = {}
    if hasattr(settings, 'AWS_ACCESS_KEY_ID'):
        session_kwargs['aws_access_key_id'] = settings.AWS_ACCESS_KEY_ID

    if hasattr(settings, 'AWS_SECRET_ACCESS_KEY'):
        session_kwargs['aws_secret_access_key'] = settings.AWS_SECRET_ACCESS_KEY
    boto3.setup_default_session(**session_kwargs)

    s3_kwargs = {}
    if hasattr(settings, 'AWS_S3_ENDPOINT'):
        s3_kwargs['endpoint_url'] = settings.AWS_S3_ENDPOINT
    elif hasattr(settings, 'AWS_S3_HOST'):
        if hasattr(settings, 'AWS_S3_USE_SSL') and settings.AWS_S3_USE_SSL is False:
            protocol = "http://"
        else:
            protocol = "https://"
        s3_kwargs['endpoint_url'] = "{}{}".format(
            protocol,
            settings.AWS_S3_HOST
        )
    if hasattr(settings, "AWS_REGION"):
        s3_kwargs['region_name'] = settings.AWS_REGION
    s3_client = boto3.client('s3', **s3_kwargs)
    s3_resource = boto3.resource('s3', **s3_kwargs)
    return s3_client, s3_resource
921915
def get_resource(self, request, filename):
        """"""
        filename = join("shared", basename(filename))
        try:
            data = pkgutil.get_data(__package__, filename)
        except OSError:
            data = None
        if data is not None:
            mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
            return Response(data, mimetype=mimetype)
        return Response("Not Found", status=404)
1065459
def collect_s3(self):
        
        print('Collecting artifacts matching tag/sha %s from S3 bucket %s' % (self.gitref, s3_bucket))
        self.s3 = boto3.resource('s3')
        self.s3_bucket = self.s3.Bucket(s3_bucket)
        self.s3.meta.client.head_bucket(Bucket=s3_bucket)
        for key in self.s3_bucket.objects.all():
            self.collect_single_s3(key.key)

        for a in self.artifacts:
            a.download(self.dlpath)
1234947
def _instantiate_remote_sources(self, kwargs):
    """"""
    kwargs['dest'] = _DestWrapper((self._target_types[kwargs['dest']],))
    return RemoteSources(build_graph=self, **kwargs)
450307
def known_remotes(self):
        """"""
        objects = []
        output = self.context.capture(
            'bzr', 'config', 'parent_location',
            check=False, silent=True,
        )
        if output and not output.isspace():
            location = output.strip()
            # The `bzr branch' command has the unusual habit of converting
            # absolute pathnames into relative pathnames. Although I get why
            # this can be preferred over the use of absolute pathnames I
            # nevertheless want vcs-repo-mgr to communicate to its callers as
            # unambiguously as possible, so if we detect a relative pathname
            # we convert it to an absolute pathname.
            if location.startswith('../'):
                location = os.path.normpath(os.path.join(self.local, location))
            objects.append(Remote(
                default=True,
                location=location,
                repository=self,
                roles=['push', 'pull'],
            ))
        return objects
441235
def get_s3_buckets(api_client, s3_info, s3_params):
    
    manage_dictionary(s3_info, 'buckets', {})
    buckets = api_client[get_s3_list_region(s3_params['selected_regions'])].list_buckets()['Buckets']
    targets = []
    for b in buckets:
        # Abort if bucket is not of interest
        if (b['Name'] in s3_params['skipped_buckets']) or (len(s3_params['checked_buckets']) and b['Name'] not in s3_params['checked_buckets']):
            continue
        targets.append(b)
    s3_info['buckets_count'] = len(targets)
    s3_params['api_clients'] = api_client
    s3_params['s3_info'] = s3_info
    thread_work(targets, get_s3_bucket, params = s3_params, num_threads = 30)
    show_status(s3_info)
    s3_info['buckets_count'] = len(s3_info['buckets'])
    return s3_info
287616
def _retrieve_remote(fnames):
    
    for fname in fnames:
        if objectstore.is_remote(fname):
            inputs = []
            regions = []
            remote_base = os.path.dirname(fname)
            for rfname in objectstore.list(remote_base):
                if rfname.endswith(tuple(KNOWN_EXTS.keys())):
                    inputs.append(rfname)
                elif rfname.endswith((".bed", ".bed.gz")):
                    regions.append(rfname)
            return {"base": remote_base,
                    "inputs": inputs,
                    "region": regions[0] if len(regions) == 1 else None}
    return {}
687087
def get_s3store_instance(bucket):
    """"""
    global _s3store_instances

    key = "%s" % bucket

    try:
        instance = _s3store_instances[key]
    except KeyError:
        instance = S3Store(
            bucket=bucket
        )
        _s3store_instances[key] = instance

    return instance
41063
def storage_uri_for_key(key):
    
    if not isinstance(key, boto.s3.key.Key):
        raise InvalidUriError('Requested key (%s) is not a subclass of '
                              'boto.s3.key.Key' % str(type(key)))
    prov_name = key.bucket.connection.provider.get_provider_name()
    uri_str = '%s://%s/%s' % (prov_name, key.bucket.name, key.name)
    return storage_uri(uri_str)
1063811
def _get_cur_remotes(path):
    
    cur_remotes = set([])
    if isinstance(path, (list, tuple)):
        for v in path:
            cur_remotes |= _get_cur_remotes(v)
    elif isinstance(path, dict):
        for v in path.values():
            cur_remotes |= _get_cur_remotes(v)
    elif path and isinstance(path, six.string_types):
        if path.startswith(tuple(INTEGRATION_MAP.keys())):
            cur_remotes.add(INTEGRATION_MAP.get(path.split(":")[0] + ":"))
    return cur_remotes
1109607
def _get_resource_programs(progs, algs):
    
    checks = {"gatk-vqsr": config_utils.use_vqsr,
              "snpeff": config_utils.use_snpeff,
              "bcbio-variation-recall": config_utils.use_bcbio_variation_recall}
    parent_child = {"vardict": _parent_prefix("vardict")}
    out = set([])
    for p in progs:
        if p == "aligner":
            for alg in algs:
                aligner = alg.get("aligner")
                if aligner and not isinstance(aligner, bool):
                    out.add(aligner)
        elif p in ["variantcaller", "svcaller", "peakcaller"]:
            if p == "variantcaller":
                for key, fn in parent_child.items():
                    if fn(algs):
                        out.add(key)
            for alg in algs:
                callers = alg.get(p)
                if callers and not isinstance(callers, bool):
                    if isinstance(callers, dict):
                        callers = reduce(operator.add, callers.values())
                    if isinstance(callers, (list, tuple)):
                        for x in callers:
                            out.add(x)
                    else:
                        out.add(callers)
        elif p in checks:
            if checks[p](algs):
                out.add(p)
        else:
            out.add(p)
    return sorted(list(out))
139109
def jarSources(target, source, env, for_signature):
    """"""
    try:
        env['JARCHDIR']
    except KeyError:
        jarchdir_set = False
    else:
        jarchdir_set = True
        jarchdir = env.subst('$JARCHDIR', target=target, source=source)
        if jarchdir:
            jarchdir = env.fs.Dir(jarchdir)
    result = []
    for src in source:
        contents = src.get_text_contents()
        if contents[:16] != "Manifest-Version":
            if jarchdir_set:
                _chdir = jarchdir
            else:
                try:
                    _chdir = src.attributes.java_classdir
                except AttributeError:
                    _chdir = None
            if _chdir:
                # If we are changing the dir with -C, then sources should
                # be relative to that directory.
                src = SCons.Subst.Literal(src.get_path(_chdir))
                result.append('-C')
                result.append(_chdir)
            result.append(src)
    return result
361010
def _get_resource(resource, name=None, resource_id=None, region=None,
                  key=None, keyid=None, profile=None):
    '''
    
    '''

    if not _exactly_one((name, resource_id)):
        raise SaltInvocationError('One (but not both) of name or id must be '
                                  'provided.')

    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)

    f = 'get_all_{0}'.format(resource)
    if not f.endswith('s'):
        f = f + 's'
    get_resources = getattr(conn, f)
    filter_parameters = {}

    if name:
        filter_parameters['filters'] = {'tag:Name': name}
    if resource_id:
        filter_parameters['{0}_ids'.format(resource)] = resource_id

    try:
        r = get_resources(**filter_parameters)
    except BotoServerError as e:
        if e.code.endswith('.NotFound'):
            return None
        raise

    if r:
        if len(r) == 1:
            if name:
                _cache_id(name, sub_resource=resource,
                          resource_id=r[0].id,
                          region=region,
                          key=key, keyid=keyid,
                          profile=profile)
            return r[0]
        else:
            raise CommandExecutionError('Found more than one '
                                        '{0} named "{1}"'.format(
                                            resource, name))
    else:
        return None
488995
def get_package_sources(self):
        """"""
        # Checkout local modules
        for config in self.sources.get('local', []):
            self.fetch_local_package(config=config)
        # Checkout S3 repositories specified in config
        for config in self.sources.get('s3', []):
            self.fetch_s3_package(config=config)
        # Checkout git repositories specified in config
        for config in self.sources.get('git', []):
            self.fetch_git_package(config=config)