downloads a web response in pieces
426720
def _chunk(self, response, size=4096):
        
        method = response.headers.get("content-encoding")
        if method == "gzip":
            d = zlib.decompressobj(16+zlib.MAX_WBITS)
            b = response.read(size)
            while b:
                data = d.decompress(b)
                yield data
                b = response.read(size)
                del data
        else:
            while True:
                chunk = response.read(size)
                if not chunk: break
                yield chunk
367759
function download(url, hostHeader, destinationFilename, callback) {
	if (typeof destinationFilename === 'function') {
		callback = destinationFilename;
		destinationFilename = null;
	}

	let options = require('url').parse(url);
	options.method = "GET";
	options.headers = {
		"Host": hostHeader,
		"Accept": "text/html,*/*;q=0.9",
		"Accept-Encoding": "gzip,identity,*;q=0",
		"Accept-Charset": "ISO-8859-1,utf-8,*;q=0.7",
		"User-Agent": "Valve/Steam HTTP Client 1.0"
	};

	let req = require('http').request(options, (res) => {
		if (res.statusCode != 200) {
			callback(new Error("HTTP error " + res.statusCode));
			return;
		}

		res.setEncoding('binary'); // apparently using null just doesn't work... thanks node
		let stream = res;

		if (res.headers['content-encoding'] && res.headers['content-encoding'] == 'gzip') {
			stream = require('zlib').createGunzip();
			stream.setEncoding('binary');
			res.pipe(stream);
		}

		let totalSizeBytes = parseInt(res.headers['content-length'] || 0, 10);
		let receivedBytes = 0;
		let dataBuffer = new Buffer(0);

		if (destinationFilename) {
			stream.pipe(require('fs').createWriteStream(destinationFilename));
		}

		stream.on('data', (chunk) => {
			if (typeof chunk === 'string') {
				chunk = new Buffer(chunk, 'binary');
			}

			receivedBytes += chunk.length;

			if (!destinationFilename) {
				dataBuffer = Buffer.concat([dataBuffer, chunk]);
			}

			callback(null, {"type": "progress", "receivedBytes": receivedBytes, "totalSizeBytes": totalSizeBytes});
		});

		stream.on('end', () => {
			callback(null, {"type": "complete", "data": dataBuffer});
		});
	});

	req.on('error', (err) => {
		callback(err);
	});

	req.end();
}
174269
def stream_download(self, chunk_size=None, callback=None):
        # type: (Optional[int], Optional[Callable]) -> Iterator[bytes]
        
        chunk_size = chunk_size or CONTENT_CHUNK_SIZE
        with contextlib.closing(self.internal_response) as response:
            # https://github.com/PyCQA/pylint/issues/1437
            for chunk in response.iter_content(chunk_size):  # pylint: disable=no-member
                if not chunk:
                    break
                if callback and callable(callback):
                    callback(chunk, response=response)
                yield chunk
747689
def download(url, target, headers=None, trackers=()):
    
    if headers is None:
        headers = {}
    headers.setdefault('user-agent', 'requests_download/'+__version__)
    r = requests.get(url, headers=headers, stream=True)
    r.raise_for_status()
    for t in trackers:
        t.on_start(r)


    with open(target, 'wb') as f:
        for chunk in r.iter_content(chunk_size=8192):
            if chunk:
                f.write(chunk)
                for t in trackers:
                    t.on_chunk(chunk)

    for t in trackers:
        t.on_finish()
317817
function download (data, filename) {
  const element = getElement();
  const url = getObjectUrl(data);

  element.setAttribute('href', url);
  element.setAttribute('download', filename);
  document.body.appendChild(element);
  element.click();
  document.body.removeChild(element);

  setTimeout(function () {
    URL.revokeObjectURL(url);
  }, 100);
}
318839
def process_response(self, request, response):
        """"""

        # For debug only.
        if not settings.DEBUG:
            return response

        # Check for responses where the data can't be inserted.
        content_encoding = response.get('Content-Encoding', '')
        content_type = response.get('Content-Type', '').split(';')[0]
        if any((getattr(response, 'streaming', False),
                'gzip' in content_encoding,
                content_type not in _HTML_TYPES)):
            return response

        content = force_text(response.content, encoding=settings.DEFAULT_CHARSET)

        pattern = re.escape('</body>')
        bits = re.split(pattern, content, flags=re.IGNORECASE)

        if len(bits) > 1:
            bits[-2] += debug_payload(request, response, self.view_data)
            response.content = "</body>".join(bits)
            if response.get('Content-Length', None):
                response['Content-Length'] = len(response.content)

        return response
245206
def stream_download(url, target_path, verbose=False):
    
    response = requests.get(url, stream=True)
    handle = open(target_path, "wb")
    if verbose:
        print("Beginning streaming download of %s" % url)
        start = datetime.now()
        try:
            content_length = int(response.headers['Content-Length'])
            content_MB = content_length/1048576.0
            print("Total file size: %.2f MB" % content_MB)
        except KeyError:
            pass      # allow Content-Length to be missing
    for chunk in response.iter_content(chunk_size=512):
        if chunk:     # filter out keep-alive new chunks
            handle.write(chunk)
    if verbose:
        print(
            "Download completed to %s in %s" %
            (target_path, datetime.now() - start))
887727
def download(self, download_key, raise_exception_on_failure=False):
        """"""
        query = {"output": "json", "user_credentials": self.api_key}
        resp = requests.get(
            "%sdownload/%s" % (self._url, download_key),
            params=query,
            timeout=self._timeout,
        )

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentDownloadFailure(resp.content, resp.status_code)
        return resp
1257744
def _request_process_octet(response):
        
        status = 'Failure'
        # Handle document download
        data = response.content
        if data:
            status = 'Success'

        return data, status
864958
def download_chunk(self, url, headers):
        
        response = requests.get(url, headers=headers, stream=True)
        if response.status_code == SWIFT_EXPIRED_STATUS_CODE \
                or response.status_code == S3_EXPIRED_STATUS_CODE:
            raise DownloadInconsistentError(response.text)
        response.raise_for_status()
        self.actual_bytes_read = 0
        self._write_response_to_file(response)
        self._verify_download_complete()
555195
def _download(self):
        """"""
        print("Downloading!")
        def parse(result):
            print("Got %r back from Yahoo." % (result,))
            values = result.strip().split(",")
            self._value = float(values[1])
        d = getPage(
            "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X"
            % (self._name,))
        d.addCallback(parse)
        d.addErrback(log.err)
        return d
1171508
def file_response(data: Union[bytes, str],  # HttpResponse encodes str if req'd
                  content_type: str,
                  filename: str) -> HttpResponse:
    
    response = HttpResponse(data, content_type=content_type)
    add_download_filename(response, filename)
    return response
485815
function copyResponseData(response, link, options)
{
	if (response instanceof Error === false)
	{
		if (response.statusCode !== 200)
		{
			link.broken = true;
			link.brokenReason = "HTTP_" + response.statusCode;
		}
		else
		{
			link.broken = false;
		}
		
		link.http.response = response;
		
		if (link.url.resolved !== response.url)
		{
			link.url.redirected = response.url;
			
			if (link.base.resolved !== null)
			{
				// TODO :: this needs a test
				linkObj.relation(link, link.url.redirected);
			}
		}
	}
	else
	{
		link.broken = true;
		
		if (reasons["ERRNO_"+response.code] != null)
		{
			link.brokenReason = "ERRNO_" + response.code;
		}
		/*else if (response.message === "Invalid URL")
		{
			link.brokenReason = "BLC_INVALID";
		}*/
		else
		{
			link.brokenReason = "BLC_UNKNOWN";
		}
	}
	
	linkObj.clean(link);
}
573789
def as_html(self):
        
        url = '{prefix}{code}.html?download'.format(prefix=EPSG_IO_URL,
                                                    code=self.id)
        return requests.get(url).text
1217947
public static void downloadDataToSink( CResponse response, ByteSink byteSink )
    {
        InputStream is = null;
        ByteSinkStream bss = null;
        boolean success = false;
        try {

            long contentLength = response.getContentLength();
            if ( contentLength >= 0 ) {
                // content length is known: inform any listener
                byteSink.setExpectedLength( contentLength );
            }

            long current = 0;
            is = response.openStream();
            bss = byteSink.openStream();

            byte[] buffer = new byte[ 8 * 1024 ];
            int len;
            while ( ( len = is.read( buffer ) ) != -1 ) {
                current += len;
                bss.write( buffer, 0, len );
            }

            if ( contentLength < 0 ) {
                // content length was unknown; inform listener operation is terminated
                byteSink.setExpectedLength( current );
            }

            bss.flush();
            bss.close();
            bss = null;

            success = true;

        } catch ( IOException ex ) {
            throw new CStorageException( ex.getMessage(), ex );
        } finally {
            if ( bss != null && !success ) {
                bss.abort();
            }

            PcsUtils.closeQuietly( is );
            PcsUtils.closeQuietly( bss );
        }
    }
943198
def download_as_json(url):
    
    try:
        return Response('application/json', request(url=url)).read()
    except HTTPError as err:
        raise ResponseException('application/json', err)
444476
def save_response_content(response, filename='data.csv', destination=os.path.curdir, chunksize=32768):
    
    chunksize = chunksize or 32768
    if os.path.sep in filename:
        full_destination_path = filename
    else:
        full_destination_path = os.path.join(destination, filename)
    full_destination_path = expand_filepath(full_destination_path)
    with open(full_destination_path, "wb") as f:
        for chunk in tqdm(response.iter_content(CHUNK_SIZE)):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
    return full_destination_path