Enqueue the download of the given foreign resource.
431185
async def async_enqueue_download(self, resource):
        '''
        
        '''
        worker = self.pick_sticky(resource.url_string)
        await worker.enqueue(enums.Task.DOWNLOAD, (resource,))
1363338
function downloadFile(url, outputPath) {
	const dir = path.dirname(outputPath);
	if (!fs.existsSync(dir)) {
		fs.mkdirSync(dir);
	}
	return new Promise((resolve, reject) => {
		const file = fs.createWriteStream(outputPath);
		const sendReq = request.get(url);
		sendReq.pipe(file);

		sendReq
		.on("response", (response) => {
			if (response.statusCode !== 200) {
				fs.unlink(outputPath);
				const error = new Error("Response status code was " + response.statusCode + ".");
				console.trace(error);
				reject(error);
			}
		})
		.on("error", (error) => {
			fs.unlink(outputPath);
			console.trace(error);
			reject(error);
		});

		file
		.on("finish", () => {
			file.close(() => {
				resolve();
			});
		})
		.on("error", (error) => {
			fs.unlink(outputPath);
			console.trace(error);
			reject(error);
		});
	});
}
1167686
def download(source, sink=None):
    
    try:
        from urllib.request import urlretrieve  # Python 3
    except ImportError:
        from urllib import urlretrieve  # Python 2
    if sink is None:
        sink = os.path.abspath(os.path.split(source)[1])
    urlretrieve(source, sink)
    return sink
585758
function download(input, output) {
	if (!output) output = input.split("/").pop();
	var file = fs.createWriteStream(output);
	var request = http.get(input, response => { response.pipe(file); });
}
710376
def enqueue(self, stream_url, offset=0, opaque_token=None):
        """"""
        directive = self._play_directive('ENQUEUE')
        audio_item = self._audio_item(stream_url=stream_url,
                                      offset=offset,
                                      push_buffer=False,
                                      opaque_token=opaque_token)
        audio_item['stream']['expectedPreviousToken'] = current_stream.token

        directive['audioItem'] = audio_item
        self._response['directives'].append(directive)
        return self
1217587
def download_and_run(config):
    '''
    
    '''
    job = fetch_job(config)
    if job:
        job._run_validate()
        return True
    else:
        return False
1147650
function addToQueueCopy(sources, dest, options) {
      if(sources.length === 1) {
        // Simple copy
        queueCopy.push({
          src:      sources[0],
          dest:     dest,
          options:  _.extend({}, options, { realDest: dest, isConcat: false })
        });
      } else {
        // Concat
        sources.forEach(function(src) {

          var def       = Q.defer();
          var cacheDest = join(options.cache, fileUID(src));

          if(!concatPromises[dest]) {
            concatPromises[dest] = [];
          }
          concatPromises[dest].push(def.promise);

          if(!concatSources[dest]) {
            concatSources[dest] = [];
          }
          concatSources[dest].push(cacheDest);

          queueCopy.push({
            src:      src,
            dest:     cacheDest,
            options:  _.extend({}, options, { realDest: dest, isConcat: true })
          }, function(err, changed) {
            if(err) {
              return def.reject(err);
            }
            def.resolve(changed);
          });
        });
      }
    }
1298101
def _download(self, resource):
    """"""
    if isinstance(resource, six.string_types):
      resource = resource_lib.Resource(url=resource)
    url = resource.url
    if url in self._sizes_checksums:
      expected_sha256 = self._sizes_checksums[url][1]
      download_path = self._get_final_dl_path(url, expected_sha256)
      if not self._force_download and resource.exists_locally(download_path):
        logging.info('URL %s already downloaded: reusing %s.',
                     url, download_path)
        self._recorded_sizes_checksums[url] = self._sizes_checksums[url]
        return promise.Promise.resolve(download_path)
    # There is a slight difference between downloader and extractor here:
    # the extractor manages its own temp directory, while the DownloadManager
    # manages the temp directory of downloader.
    download_dir_path = os.path.join(
        self._download_dir,
        '%s.tmp.%s' % (resource_lib.get_dl_dirname(url), uuid.uuid4().hex))
    tf.io.gfile.makedirs(download_dir_path)
    logging.info('Downloading %s into %s...', url, download_dir_path)

    def callback(val):
      checksum, dl_size = val
      return self._handle_download_result(
          resource, download_dir_path, checksum, dl_size)
    return self._downloader.download(url, download_dir_path).then(callback)
1157580
def stream_download(self, chunk_size: Optional[int] = None, callback: Optional[Callable] = None) -> AsyncIterator[bytes]:
        
        chunk_size = chunk_size or CONTENT_CHUNK_SIZE
        async def async_gen(resp):
            while True:
                chunk = await resp.content.read(chunk_size)
                if not chunk:
                    break
                callback(chunk, resp)
        return async_gen(self.internal_response)
581255
def _harvest_lost_resources(self):
        """"""
        with self._lock:
            for i in self._unavailable_range():
                rtracker = self._reference_queue[i]
                if rtracker is not None and rtracker.available():
                    self.put_resource(rtracker.resource)
304710
async def convert_endpoint(url_string, ts, is_just_checking):
    '''
    
    '''
    response = singletons.server.response

    # Prep ForeignResource and ensure does not validate security settings
    singletons.settings
    foreign_res = ForeignResource(url_string)

    target_ts = TypeString(ts)
    target_resource = TypedResource(url_string, target_ts)

    # Send back cache if it exists
    if target_resource.cache_exists():
        if is_just_checking:
            return _just_checking_response(True, target_resource)
        return await response.file(target_resource.cache_path, headers={
            'Content-Type': target_ts.mimetype,
        })

    # Check if already downloaded. If not, queue up download.
    if not foreign_res.cache_exists():
        singletons.workers.enqueue_download(foreign_res)

    # Queue up a single function that will in turn queue up conversion
    # process
    singletons.workers.enqueue_sync(
        enqueue_conversion_path,
        url_string,
        str(target_ts),
        singletons.workers.enqueue_convert
    )

    if is_just_checking:
        return _just_checking_response(False, target_resource)

    # Respond with placeholder
    return singletons.placeholders.stream_response(target_ts, response)
936459
public void enqueue(String resourceFileName) throws IOException {
		File tmp = new File(path,resourceFileName);
		if(!tmp.isFile()) {
			tmp.createNewFile();
		}
	}
887727
def download(self, download_key, raise_exception_on_failure=False):
        """"""
        query = {"output": "json", "user_credentials": self.api_key}
        resp = requests.get(
            "%sdownload/%s" % (self._url, download_key),
            params=query,
            timeout=self._timeout,
        )

        if raise_exception_on_failure and resp.status_code != 200:
            raise DocumentDownloadFailure(resp.content, resp.status_code)
        return resp
919961
def _enqueue_capture(build, release, run, url, config_data, baseline=False):
    """"""
    # Validate the JSON config parses.
    try:
        config_dict = json.loads(config_data)
    except Exception, e:
        abort(utils.jsonify_error(e))

    # Rewrite the config JSON to include the URL specified in this request.
    # Blindly overwrite anything that was there.
    config_dict['targetUrl'] = url
    config_data = json.dumps(config_dict)

    config_artifact = _save_artifact(build, config_data, 'application/json')
    db.session.add(config_artifact)
    db.session.flush()

    suffix = ''
    if baseline:
        suffix = ':baseline'

    task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix)
    logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline)

    work_queue.add(
        constants.CAPTURE_QUEUE_NAME,
        payload=dict(
            build_id=build.id,
            release_name=release.name,
            release_number=release.number,
            run_name=run.name,
            url=url,
            config_sha1sum=config_artifact.id,
            baseline=baseline,
        ),
        build_id=build.id,
        release_id=release.id,
        run_id=run.id,
        source='request_run',
        task_id=task_id)

    # Set the URL and config early to indicate to report_run that there is
    # still data pending even if 'image' and 'ref_image' are unset.
    if baseline:
        run.ref_url = url
        run.ref_config = config_artifact.id
    else:
        run.url = url
        run.config = config_artifact.id
811429
def _handle_download_result(self, resource, tmp_dir_path, sha256, dl_size):
    """"""
    fnames = tf.io.gfile.listdir(tmp_dir_path)
    if len(fnames) > 1:
      raise AssertionError('More than one file in %s.' % tmp_dir_path)
    original_fname = fnames[0]
    tmp_path = os.path.join(tmp_dir_path, original_fname)
    self._recorded_sizes_checksums[resource.url] = (dl_size, sha256)
    if self._register_checksums:
      self._record_sizes_checksums()
    elif (dl_size, sha256) != self._sizes_checksums.get(resource.url, None):
      raise NonMatchingChecksumError(resource.url, tmp_path)
    download_path = self._get_final_dl_path(resource.url, sha256)
    resource_lib.write_info_file(resource, download_path, self._dataset_name,
                                 original_fname)
    # Unconditionally overwrite because either file doesn't exist or
    # FORCE_DOWNLOAD=true
    tf.io.gfile.rename(tmp_path, download_path, overwrite=True)
    tf.io.gfile.rmtree(tmp_dir_path)
    return download_path
1188379
def download(self, local, remote):
        
        self.sync(RemoteFile(remote, self.api), LocalFile(local))
158980
def fetch_model(self, source: str, file: Union[str, BinaryIO],
                    chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
        """"""
        download_http(source, file, self._log, chunk_size)