Download http response and save to disk.

@param {string} url to fetch.
@param {string} dest where to write the tarball.
@param {Function} callback invoke when done.
972716
function(url, dest, callback, count) {
    if (isNaN(count)) {
      count = 10;
    }

    if (this.cache[url]) {
      return fs.writeFile(dest, this.cache[url], callback);
    }

    fs.exists(dest, function(exists) {
      if (exists) {
        // No need to download :).
        return callback && callback();
      }

      var stream = fs.createWriteStream(dest);
      debug('GET ' + url);
      var protocol = this._getProtocol(url);
      protocol.get(url, function(res) {
        if (res.statusCode !== 200) {
          debug('Bad status ' + res.statusCode + ' for ' + url + '... ' +
                'will try ' + count + ' more times');
          if (count === 0) {
            return callback && callback(
              new Error('Bad status ' + res.statusCode + ' for ' + url));
          }

          return Download.inst.downloadToDisk(url, dest, callback, count - 1);
        }

        res.pipe(stream);
        stream.on('finish', callback);
      });
    }.bind(this));
  }
335284
function requestUrl(url, callback) {
  var req = new XMLHttpRequest();
  req.addEventListener("load", function () {
    callback(req.responseText);
  });
  req.open("GET", url);
  req.send();
}
182514
function(callback) {
        request.get(this.getStatusUrl(), function(err, res, body) {
            if (err) {
                return callback(err);
            }

            body = body || {};
            body.status = res.statusCode;
            body.date = new Date(body.date);

            callback(err, body, res);
        });

        return this;
    }
1260979
function(base_url, callback, cors) {
    var recv = new XhrReceiver(base_url + '/chunking_test', {cors: cors});
    var result = 0;
    recv.onmessage = function(e) {
        // Now a cool hack: we can stop receiving after we got at least
        // one chunk, contains some data, but not everyting.
        var l = e.responsetext.split('h\n').length;
        if(e.readystate === 3 && l > 0 && l < 6 ) {
            result = l;
            recv.abort();
        }
    };
    recv.onclose = function(e) {
        recv = recv.onmessage = recv.onclose = null;
        utils.log('Chunking test: ' + (result ? 'passed' : 'failed')
                  + ' (' + result + ' chunk received)');
        callback(!!result);
    };
}
709703
function download(downloadUrl, filename, callback, ondata) {
    var url = require("url").parse(downloadUrl);
    var out = require("fs").createWriteStream(filename, { flags: 'w', encoding: null, mode: 0666 });
    var bytes = 0, total = -1;
    var options = {
        "hostname" : url["host"],
        "method" : "GET",
        "path" : url["path"],
        "agent" : false
    }

    // Reconfigure request options if an HTTP proxy setting exists
    if (process.env.http_proxy) {
        var http_proxy = require("url").parse(process.env.http_proxy);
        options["hostname"] = http_proxy["hostname"];
        options["port"] = http_proxy["port"];
        options["path"] = url["href"];
        options["headers"] = {
            "Host" : url["host"]
        };
    }

    var req = require(path.join(__dirname, "..", "lib", "follow-redirects.js")).http.request(options,
                                                                                             function(res) {
        if (res.headers["content-length"]) {
            total = parseInt(res.headers["content-length"], 10);
        }
        if (res.statusCode != 200) {
            res.setEncoding(null);
            callback(new Error("Download failed: HTTP status code "+res.statusCode), -1);
            return;
        }
        res.on("data", function(chunk) {
            bytes += chunk.length;
            if (ondata) ondata(chunk.length, total);
            out.write(chunk);
        });

        res.on("end", function() {
            callback(null, bytes);
        });
    });
    req.on("error", function(e) {
        callback(e, -1);
    });
    req.end();
}
882442
function download(url, dest, cb) {
  var reportError = function(err) {
    var timeoutMessge;

    if (err.code === 'ETIMEDOUT') {
      if (err.connect === true) {
        // timeout is hit while your client is attempting to establish a connection to a remote machine
        timeoutMessge = 'Timed out attemping to establish a remote connection';
      } else {
        timeoutMessge = 'Timed out whilst downloading the prebuilt binary';
        // occurs any time the server is too slow to send back a part of the response
      }

    }
    cb(['Cannot download "', url, '": ', eol, eol,
      typeof err.message === 'string' ? err.message : err, eol, eol,
      timeoutMessge ? timeoutMessge + eol + eol : timeoutMessge,
      'Hint: If github.com is not accessible in your location', eol,
      '      try setting a proxy via HTTP_PROXY, e.g. ', eol, eol,
      '      export HTTP_PROXY=http://example.com:1234',eol, eol,
      'or configure npm proxy via', eol, eol,
      '      npm config set proxy http://example.com:8080'].join(''));
  };

  var successful = function(response) {
    return response.statusCode >= 200 && response.statusCode < 300;
  };

  console.log('Downloading binary from', url);

  try {
    request(url, downloadOptions(), function(err, response, buffer) {
      if (err) {
        reportError(err);
      } else if (!successful(response)) {
        reportError(['HTTP error', response.statusCode, response.statusMessage].join(' '));
      } else {
        console.log('Download complete');

        if (successful(response)) {
          fs.createWriteStream(dest)
            .on('error', cb)
            .end(buffer, cb);
        } else {
          cb();
        }
      }
    })
    .on('response', function(response) {
      var length = parseInt(response.headers['content-length'], 10);
      var progress = log.newItem('', length);

      // The `progress` is true by default. However if it has not
      // been explicitly set it's `undefined` which is considered
      // as far as npm is concerned.
      if (process.env.npm_config_progress === 'true') {
        log.enableProgress();

        response.on('data', function(chunk) {
          progress.completeWork(chunk.length);
        })
        .on('end', progress.finish);
      }
    });
  } catch (err) {
    cb(err);
  }
}
367759
function download(url, hostHeader, destinationFilename, callback) {
	if (typeof destinationFilename === 'function') {
		callback = destinationFilename;
		destinationFilename = null;
	}

	let options = require('url').parse(url);
	options.method = "GET";
	options.headers = {
		"Host": hostHeader,
		"Accept": "text/html,*/*;q=0.9",
		"Accept-Encoding": "gzip,identity,*;q=0",
		"Accept-Charset": "ISO-8859-1,utf-8,*;q=0.7",
		"User-Agent": "Valve/Steam HTTP Client 1.0"
	};

	let req = require('http').request(options, (res) => {
		if (res.statusCode != 200) {
			callback(new Error("HTTP error " + res.statusCode));
			return;
		}

		res.setEncoding('binary'); // apparently using null just doesn't work... thanks node
		let stream = res;

		if (res.headers['content-encoding'] && res.headers['content-encoding'] == 'gzip') {
			stream = require('zlib').createGunzip();
			stream.setEncoding('binary');
			res.pipe(stream);
		}

		let totalSizeBytes = parseInt(res.headers['content-length'] || 0, 10);
		let receivedBytes = 0;
		let dataBuffer = new Buffer(0);

		if (destinationFilename) {
			stream.pipe(require('fs').createWriteStream(destinationFilename));
		}

		stream.on('data', (chunk) => {
			if (typeof chunk === 'string') {
				chunk = new Buffer(chunk, 'binary');
			}

			receivedBytes += chunk.length;

			if (!destinationFilename) {
				dataBuffer = Buffer.concat([dataBuffer, chunk]);
			}

			callback(null, {"type": "progress", "receivedBytes": receivedBytes, "totalSizeBytes": totalSizeBytes});
		});

		stream.on('end', () => {
			callback(null, {"type": "complete", "data": dataBuffer});
		});
	});

	req.on('error', (err) => {
		callback(err);
	});

	req.end();
}
804535
def download(url, dest):
    """"""
    path = os.path.join(dest, url.split('/')[-1])
    r = requests.get(url, stream=True)
    r.raise_for_status()
    with open(path, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
    return path
719121
function (url, data, onDone) {

    SR.Log('url to post: ' + url);

    // construct an HTTP request
    var xmlHttp = new XMLHttpRequest();
    xmlHttp.open("POST", url, true);
    xmlHttp.setRequestHeader('Content-Type', 'application/json; charset=UTF-8');
    
    // set callback when response is returned
    xmlHttp.onloadend = onHttpResponse(xmlHttp, onDone);
	//console.log('data to send:');
	//console.log(data);
	
    // send the collected data as JSON
	// TODO: may be slow in Firefox?
	xmlHttp.send(Object.keys(data).length > 0 ? JSON.stringify(data) : undefined);
}
648362
public static void downloadAsync(String url, String file, HttpCallback<File> callback) {
        downloadAsync(url, file, null, callback, new StrParam[0]);
    }
373890
function( callback ) {
		var self = this;
		if (!callback) callback = function(){};

		// If there are no queued requests, fire callback as-is
		if (this.queuedRequests.length === 0) {
			callback( null, this.database );
			return;
		}

		// First make sure that there are no bundles pending loading
		var pendingLoading = false;
		for (var i=0; i<this.queuedRequests.length; ++i) {
			if (this.queuedRequests[i].status === PBUND_REQUESTED) {
				pendingLoading = true;
				break;
			}
		}

		////////////////////////////////////////////////////////
		// Iteration 1 - PBUND_REQUESTED -> PBUND_LOADED
		// ----------------------------------------------------
		// Download all bundles in pending state.
		////////////////////////////////////////////////////////

		if (pendingLoading) {

			// Prepare the callbacks for when this is finished
			var state = { 'counter': 0 }
			var continue_callback = (function() {
				// When reached 0, continue loading
				if (--this.counter === 0)
					self.__process( callback );
			}).bind(state);

			// Place all requests in parallel
			var triggeredError = false;
			for (var i=0; i<this.queuedRequests.length; ++i) {
				var req = this.queuedRequests[i], part = this.progressManager.part();
				if (req.status === PBUND_REQUESTED) {

					// Download bundle from URL(s)
					state.counter++;
					req.buffer = downloadArrayBuffers(req.url, part,
						(function(req) {
							return function( err ) {

								// Handle errors
								if (err) {
									if (triggeredError) return;
									var errMsg = "Error downloading bundle: "+err;
									if (req.callback) req.callback( errMsg, null);
									if (callback) callback(errMsg, null);
									triggeredError = true;
									return;
								}

								// Keep buffer and mark as loaded
								req.status = PBUND_LOADED;
								// Continue
								continue_callback();
							}
						})(req)
					);

				}
			}

			// Do not continue, we ar asynchronous
			return;
		}

		////////////////////////////////////////////////////////
		// Iteration 2 - PBUND_LOADED -> PBUND_PARSED
		// ----------------------------------------------------
		// Parse all loaded bundles (synchronous)
		////////////////////////////////////////////////////////

		for (var i=0; i<this.queuedRequests.length; ++i) {
			var req = this.queuedRequests[i];
			if (req.status === PBUND_LOADED) {
				// try {

				// Create bundle from sparse or compact format
				var bundle;
				if (req.buffer.length === 1) {
					bundle = new BinaryBundle( req.buffer[0], self.profile );
				} else {
					bundle = new BinaryBundle( req.buffer, self.profile );
				}

				// Parse bundle
				parseBundle( bundle, self.database );

				// Trigger bundle callback
				if (req.callback) req.callback( null, req.bundle );

				// } catch (e) {

				// 	// Update bundle status
				// 	req.status = PBUND_ERROR;

				// 	// Fire error callbacks and exit
				// 	var errMsg = "Error parsing bundle: "+e.toString();
				// 	if (req.callback) req.callback( errMsg, null);
				// 	if (callback) callback(errMsg, null);
				// 	return;

				// }
			}
		}

		// We are ready
		this.queuedRequests = [];
		callback( null, this.database );

	}
137573
function downloadDistantOrLoad(fileUrl, f) {
    if (!fileUrl || !_.isString(fileUrl)) {
      return f();
    }

    var filename = path.basename(url.parse(fileUrl).path);

    debug('downloading %s -> %s', fileUrl, filename);

    request({
      url: fileUrl
    }, function (err, resp, body) {
      if (err) {
        return f(err);
      }

      if (resp.statusCode < 200 || resp.statusCode >= 300) {
        return f(new Error('Invalid HTTP Status code "' + resp.statusCode + '" for "' + fileUrl + '"'));
      }

      fs.writeFileSync(path.resolve(process.cwd(), filename), body);

      f();
    });
  }
1089743
function downloadVisualizationData(_visJson, destDir, callback) {
    if (destDir === undefined) destDir = '.';

    withVisJson(_visJson, function (err, visJson) {
        if (err && callback) return callback(err);
        _async2['default'].forEachOf(visJson.layers, function (layer, layerIndex, callback) {
            if (layer.type === 'namedmap') {
                console.error("Layer is not public, cannot download its data. If you own the map and datasets, please change the privacy settings to 'public' and try again.");
                return callback();
            }
            if (layer.type !== 'layergroup') {
                if (callback) callback();
                return;
            }
            _async2['default'].forEachOf(layer.options.layer_definition.layers, function (sublayer, sublayerIndex, callback) {
                var dest = _path2['default'].join(sublayerDir(destDir, layerIndex, sublayerIndex), 'layer.geojson');
                downloadSublayerData(visJson, layerIndex, sublayerIndex, dest, callback);
            }, callback);
        }, callback);
    });
}
142276
function(callback)
    {
        var params = util.extend({count:1}, this.params);        
        this.api.doRequest("GET", this.path, params, function(err, res) {
            if (err)
            {
                return callback(err);
            }                    
            callback(null, res.count);
        });
    }
830615
function (url) {
        var self = this;
        this.searching = true;
        this.loadingError = false;
        return $http.get(url)
          .success(function (res) {
            self.searching = false;
            return res.data;
          })
          .error(function () {
            self.searching = false;
            self.loadingError = true;
            return false;
          });
      }
823385
function(url, callback) {
      var t = this,
          originalUrl = url,
          page = null;

      // Change urls that end in / to /index
      if (url.substr(-1) === '/') {
        url = url + 'index';
      }

      // Return if it's in cache
      page = pageCache.get(url);
      if (page) {
        return callback(null, page);
      }

      // Read from storage
      page = new Page({id: url});
      page.fetch({liveSync: true, silenceErrors: true}, function(error) {

        // Process a 404.  This returns a transient page copied from
        // the default 404 page, with the id replaced by the specified url.
        if (error && error.code === 'NOTFOUND' && url !== '/app/core/404') {

          // Default the home page if notfound
          if (originalUrl === '/') {
            return t._getPage('/app/core/index', callback);
          }

          // Default the 404 page if notfound
          if (originalUrl === '/404') {
            return t._getPage('/app/core/404', callback);
          }

          // Otherwise it's a new page.  Create it.
          t._getPage('/404', function(error, page404) {
            if (error) {
              console.error("Error loading the 404 page", error);
              return callback('404 page load error');
            }

            // Copy the 404 page into a new page
            var newPage = new Page(JSON.parse(JSON.stringify(page404)));

            // Build a sane starting title.  TitleCase the last url element, separate words, replace underscores
            var title = $.titleCase(url.split('/').pop(), true).replace(/([A-Z])/g," $1").replace(/^ /,'').replace(/_/g,' ');
            var title = url.split('/').pop().replace(/([A-Z])/g," $1").replace(/^ /,'').replace(/_/g,' ');
            newPage.set({id:url, title:title, is404page:true});
            callback(null, newPage);
          });
          return;
        }

        // Process other errors
        if (error) {
          return callback(error);
        }

        // Assure the page model ID is correct on disk
        if (url !== page.get('id')) {
          page.set('id', url);
        }

        // Put the page into cache and return it
        pageCache.add(page);
        return callback(null, page);
      });
    }
121883
function downloader(url, saveAtPath, callback) {
  // Handle the parameter errors
  if (!url || !saveAtPath) {
    return callback(new Error("You need to provide the URL and the location to save the file!"));
  }
  if (url && typeof url !== "string") {
    return callback(new Error("The parameter 'url' must be a string"));
  }
  if (saveAtPath && typeof saveAtPath !== "string") {
    return callback(new Error("The parameter 'saveAtPath' must be a string"));
  }
  if (callback && typeof callback !== "function") {
    throw new Error("The callback provided must be a function");
  }

  var httpClient = url.indexOf("https://") === 0 ? https : http;

  // The directory to save the file.
  var dirToSave = path.dirname(saveAtPath);

  // Verify if the directory to save exists before downloading.
  fs.stat(dirToSave, function gettingStats(err) {
    if (err) {
      return callback(new Error("The path '" + dirToSave + "' does NOT exist!"));
    }

    var req = httpClient.get(url, function requestHandler(res) {
      if (res.statusCode === 404) {
        return callback(new Error("The resource at the given URL " + url + " does NOT exist"));
      }

      var chunks = [];

      res.on("data", function(chunk) {
        chunks.push(chunk);
      });

      res.on("end", function() {
        var buffer = Buffer.concat(chunks);

        // Finished downloading and getting the buffer to be saved.
        fs.writeFile(saveAtPath, buffer, function(err) {
          if (err) {
            var error = new Error("Error while saving image: " + err.message);
            if (err.code === "EISDIR") {
              error = new Error("Cannot save the url " + url + " as a directory " + saveAtPath);
            }
            callback(error);
          }

          fs.stat(saveAtPath, function(err, stat) {
            if (err) {
              callback(new Error("Error while getting image stats: " + err.message));
            }

            // Creating the response object.
            var response = {
              downloaded: true,
              filePath: saveAtPath,
              size: stat.size
            };

            // The request to the URL was successful and the file was saved.
            callback(null, response);
          });
        });
      });

    });

    // Handle errors during the request.
    req.on("error", function(error) {
      if (error) {
        if (error.code === "ENOTFOUND") {
          return callback(new Error("There's no internet connectivity to " + url));

        } else {
          var unknownError = new Error("Unknown error to save " + url + " as " + saveAtPath);
          unknownError.stack = error.stack;
          return callback(unknownError);
        }
      }
    });
  });
}