Amazon S3 downloader
186728
def amazon_s3(self, scene, bands):
        

        sat = self.scene_interpreter(scene)

        # Always grab MTL.txt and QA band if bands are specified
        if 'BQA' not in bands:
            bands.append('QA')

        if 'MTL' not in bands:
            bands.append('MTL')

        urls = []

        for band in bands:
            # get url for the band
            url = self.amazon_s3_url(sat, band)

            # make sure it exist
            self.remote_file_exists(url)
            urls.append(url)

        # create folder
        path = check_create_folder(join(self.download_dir, scene))

        self.output('Source: AWS S3', normal=True, arrow=True)
        for url in urls:
            self.fetch(url, path)

        return path
934169
function download(url) {
  return request('GET', url).then(function (res) {
    var out = {
      url: url,
      body: res.body,
      statusCode: res.statusCode,
      headers: res.headers,
      dependencies: []
    };

    try {
      if (out.statusCode === 302) {
        out.dependencies = [out.headers.location];
      } else if (!out.headers['content-type']) {
        out.dependencies = [];
      } else if (out.headers['content-type'].indexOf('text/html') !== -1) {
        out.dependencies = htmldeps(out.body.toString());
      } else if (out.headers['content-type'].indexOf('text/css') !== -1) {
        out.dependencies = cssdeps(out.body.toString());
      } else {
        out.dependencies = [];
      }
    } catch (ex) {
      ex.message = 'Error parsing deps for "' + url + '": ' + ex.message;
      throw ex;
    }
    return out;
  });
}
830075
def s3_download(url, dst):  # type: (str, str) -> None
    
    url = parse.urlparse(url)

    if url.scheme != 's3':
        raise ValueError("Expecting 's3' scheme, got: %s in %s" % (url.scheme, url))

    bucket, key = url.netloc, url.path.lstrip('/')

    region = os.environ.get('AWS_REGION', os.environ.get(_params.REGION_NAME_ENV))
    s3 = boto3.resource('s3', region_name=region)

    s3.Bucket(bucket).download_file(key, dst)
640222
def download(self, url, output_path):
        
        request_failed_message = self.tr(
            "Can't access PetaBencana API: {source}").format(
            source=url)
        downloader = FileDownloader(url, output_path)
        result, message = downloader.download()
        if not result:
            display_warning_message_box(
                self,
                self.tr('Download error'),
                self.tr(request_failed_message + '\n' + message))

        if result == QNetworkReply.OperationCanceledError:
            display_warning_message_box(
                self,
                self.tr('Download error'),
                self.tr(message))
914373
def _thumbnail_s3(self, original_filename, thumb_filename,
                      thumb_size, thumb_url, bucket_name,
                      crop=None, bg=None, quality=85):
        """"""

        scheme = self.app.config.get('THUMBNAIL_S3_USE_HTTPS') and 'https' or 'http'

        thumb_url_full = url_for_s3(
            'static',
            bucket_name=self.app.config.get('THUMBNAIL_S3_BUCKET_NAME'),
            filename=thumb_url,
            scheme=scheme)
        original_url_full = url_for_s3(
            'static',
            bucket_name=bucket_name,
            filename=self._get_s3_path(original_filename).replace('static/', ''),
            scheme=scheme)

        # Return the thumbnail URL now if it already exists on S3.
        # HTTP HEAD request saves us actually downloading the image
        # for this check.
        # Thanks to:
        # http://stackoverflow.com/a/16778749/2066849
        try:
            resp = httplib2.Http().request(thumb_url_full, 'HEAD')
            resp_status = int(resp[0]['status'])
            assert(resp_status < 400)
            return thumb_url_full
        except Exception:
            pass

        # Thanks to:
        # http://stackoverflow.com/a/12020860/2066849
        try:
            fd = urllib.urlopen(original_url_full)
            temp_file = BytesIO(fd.read())
            image = Image.open(temp_file)
        except Exception:
            return ''

        img = self._thumbnail_resize(image, thumb_size, crop=crop, bg=bg)

        temp_file = BytesIO()
        img.save(temp_file, image.format, quality=quality)

        conn = S3Connection(self.app.config.get('THUMBNAIL_S3_ACCESS_KEY_ID'), self.app.config.get('THUMBNAIL_S3_ACCESS_KEY_SECRET'))
        bucket = conn.get_bucket(self.app.config.get('THUMBNAIL_S3_BUCKET_NAME'))

        path = self._get_s3_path(thumb_filename)
        k = bucket.new_key(path)

        try:
            k.set_contents_from_string(temp_file.getvalue())
            k.set_acl(self.app.config.get('THUMBNAIL_S3_ACL', 'public-read'))
        except S3ResponseError:
            return ''

        return thumb_url_full
1287412
def s3(self, url, account_acessor=None, access=None, secret=None):
        """"""
        from ambry.util.ambrys3 import AmbryS3FS
        from ambry.util import parse_url_to_dict
        import ssl

        pd = parse_url_to_dict(url)

        if account_acessor:
            account = account_acessor(pd['hostname'])

            assert account['account_id'] == pd['hostname']
            aws_access_key = account['access_key'],
            aws_secret_key = account['secret']
        else:
            aws_access_key = access
            aws_secret_key = secret

        assert access, url
        assert secret, url

        s3 = AmbryS3FS(
            bucket=pd['netloc'],
            prefix=pd['path'].strip('/')+'/',
            aws_access_key=aws_access_key,
            aws_secret_key=aws_secret_key,

        )

        return s3
1209396
def s3_etag(url: str) -> Optional[str]:
    """"""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag
763964
function deployS3(config) {
  if (!config.credentials.accessKeyId || !config.credentials.secretAccessKey) {
    gutil.log(
      gutil.colors.red('AWS S3 credentials missing, skipping upload...'),
    );
    return;
  }

  // create a new publisher
  const publisher = aws.create(config.credentials);

  // define custom headers
  const headers = {
    'Cache-Control': 'max-age=315360000, no-transform, public',
  };
  return (
    gulp
      .src(config.assetsPath)
      .pipe(
        rename(function(filepath) {
          filepath.dirname = path.join(config.dirname, filepath.dirname);
        }),
      )
      // gzip, Set Content-Encoding headers and add .gz extension
      .pipe(gulpIf(file => !file.path.match(/\.(mp4|ogg|webm|avi|mov|mkv)$/), aws.gzip()))
      // publisher will add Content-Length, Content-Type and headers specified above
      // If not specified it will set x-amz-acl to public-read by default
      .pipe(publisher.publish(headers))
      // create a cache file to speed up consecutive uploads
      .pipe(publisher.cache())
      // print upload updates to console
      .pipe(aws.reporter())
  );
}
708474
function download(target, dest, opts, fn) {
	if (typeof opts === 'function') {
		fn = opts;
		opts = null;
	}
	opts = opts || {};
	let clone = opts.clone || false;

	const repo = normalize(target);
	let url = getUrl(repo, clone);
	if (url && typeof url === 'object' && url.url) {
		if (typeof url.clone === 'boolean') {
			clone = url.clone;
		}
		url = url.url;
	}

	return PromiseA.try(() => {
		if (clone) {
			const checkout = repo.checkout && repo.checkout !== 'master' ? repo.checkout : null;
			debug('Cloning from:', url, checkout ? '#' + checkout : '');
			return PromiseA.fromCallback(cb => gitclone(url, dest, {checkout}, cb)).then(() => rm(dest + '/.git'));
		} else {
			debug('Downloading from:', url);
			return PromiseA.resolve(downloadUrl(url, dest, {extract: true, strip: 1, mode: '666', headers: {accept: 'application/zip'}}));
		}
	}).asCallback(fn);
}
818153
def download_object(self, instance, bucket_name, object_name):
        
        url = '/buckets/{}/{}/{}'.format(instance, bucket_name, object_name)
        response = self._client.get_proto(path=url)
        return response.content
322190
def generate_s3_bucket():
    """"""
    logger.debug("[#] Setting up S3 bucket")
    client = boto3.client("s3", region_name=PRIMARY_REGION)
    buckets = client.list_buckets()
    matches = [x for x in buckets.get('Buckets', list())
               if x['Name'].startswith(S3_BUCKET)]
    if len(matches) > 0:
        logger.debug("[*] Bucket already exists")
        return matches.pop()

    response = client.create_bucket(
        Bucket=S3_BUCKET,
        CreateBucketConfiguration={
            'LocationConstraint': PRIMARY_REGION
        }
    )
    logger.info("[#] Successfully setup the S3 bucket")

    return response
1177089
public static boolean isDownloadParallelizable(final AmazonS3 s3, final GetObjectRequest getObjectRequest,
            Integer partCount) {
        ValidationUtils.assertNotNull(s3, "S3 client");
        ValidationUtils.assertNotNull(getObjectRequest, "GetObjectRequest");

        if (s3 instanceof AmazonS3Encryption || getObjectRequest.getRange() != null
                || getObjectRequest.getPartNumber() != null || partCount == null) {
            return false;
        }
        return true;
    }
1042124
def fetch_file(uri, file=None, username=None, password=None):
    
    boto.log.info('Fetching %s' % uri)
    if file == None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = boto.connect_s3(aws_access_key_id=username, aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib2.HTTPBasicAuthHandler(passman)
                opener = urllib2.build_opener(authhandler)
                urllib2.install_opener(opener)
            s = urllib2.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        boto.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file
484508
function (options) {
        var s3 = {};
        s3.client = connectToS3(options);
        s3.bucket = options.bucket || '';
        return s3;
    }
359615
def _read_s3_config(self):
        
        try:
            import boto3
            import botocore.exceptions
        except ImportError:
            boto3, botocore = None, None

        if not boto3:
            raise ValueError(
                's3 URL specified for configuration but boto3 not installed')
        parsed = parse.urlparse(self._file_path)
        try:
            response = boto3.client(
                's3', endpoint_url=os.environ.get('S3_ENDPOINT')).get_object(
                    Bucket=parsed.netloc, Key=parsed.path.lstrip('/'))
        except botocore.exceptions.ClientError as e:
            raise ValueError(
                'Failed to download configuration from S3: {}'.format(e))
        return response['Body'].read().decode('utf-8')
935866
def download(self, bucket, key, fileobj, transfer_config=None, subscribers=None):
        '''  '''
        check_io_access(os.path.dirname(fileobj), os.W_OK)
        return self._queue_task(bucket, [FilePair(key, fileobj)], transfer_config,
                                subscribers, enumAsperaDirection.RECEIVE)
568947
private ObjectMetadata getObjectMetadataUsingRange(final PresignedUrlDownloadRequest request) {
        PresignedUrlDownloadRequest copy = request.clone();

        try {
            return s3.download(copy.withRange(0, 1))
                     .getS3Object()
                     .getObjectMetadata();
        } catch (AmazonS3Exception exception) {
            // This handles error case when trying a range GET on object with zero size
            if (exception.getStatusCode() == 416 && "InvalidRange".equals(exception.getErrorCode())) {
                return null;
            }
            throw exception;
        }
    }