Divides file_url size into an array of ranges to be downloaded by workers.
        :param: file_url: ProjectFileUrl: file url to download
        :return: [(int,int)]: array of (start, end) tuples
195484
def make_ranges(self, file_url):
        
        size = file_url.size
        bytes_per_chunk = self.determine_bytes_per_chunk(size)
        start = 0
        ranges = []
        while size > 0:
            amount = bytes_per_chunk
            if amount > size:
                amount = size
            ranges.append((start, start + amount - 1))
            start += amount
            size -= amount
        return ranges
747689
def download(url, target, headers=None, trackers=()):
    
    if headers is None:
        headers = {}
    headers.setdefault('user-agent', 'requests_download/'+__version__)
    r = requests.get(url, headers=headers, stream=True)
    r.raise_for_status()
    for t in trackers:
        t.on_start(r)


    with open(target, 'wb') as f:
        for chunk in r.iter_content(chunk_size=8192):
            if chunk:
                f.write(chunk)
                for t in trackers:
                    t.on_chunk(chunk)

    for t in trackers:
        t.on_finish()
1187753
def download(odir: Path, source_url: str, irng: Sequence[int]):
    
    assert len(irng) == 2, 'specify start, stop indices'

    odir = Path(odir).expanduser()
    odir.mkdir(parents=True, exist_ok=True)

    ri = int(source_url.split('/')[-2][:2])

    for i in range(*irng):
        fn = f'index-{ri:2d}{i:02d}.fits'
        url = f'{source_url}{fn}'
        ofn = odir / fn
        if ofn.is_file():  # no clobber
            print('skipping', ofn)
            continue
        print(f'{url} => {ofn}', end='\r')

        urlretrieve(url, ofn)
274719
def enumerateURL(urlDict, outputFolder, startIndex= 0, maxErrors = 100):
    
    for i, url in enumerate(urlDict.keys()):
        # Grabbing domain name:
        domain = re.findall("://(.*)/", url)[0]

        # Defining the starting index
        index = startIndex

        # The app will stop when this value reaches maxErrors
        consecutiveErrors = 0

        i3Browser = browser.Browser()

        # Main loop that checks if the maximum number of errors has been reached
        while consecutiveErrors <= maxErrors:
            # creating the new URL to download
            newQuery = url.replace("<INDEX>", str(index))
            print(newQuery)
            # Downloading the file
            try:
                data = i3Browser.recoverURL(newQuery)

                filename = domain.replace("/", "|") + "_" + "-profile_" + str(index).rjust(10, "0") +".html"

                if urlDict[url] != None:
                    if urlDict[url] in data:
                        print(general.info("Storing resource as:\t" + filename + "..."))
                        # The profile was found  so we will store it:
                        with open( outputFolder + "/" + filename, "w") as oF:
                            oF.write(data)
                else:
                    # The profile was found  so we will store it:
                    print(general.info("Storing resource as:\t" + filename + "..."))
                    with open( outputFolder + "/" + filename, "w") as oF:
                        oF.write(data)
            except:
                pass
                #logger.error("The resource could not be downloaded.")

            index+=1
347942
def download_http(source: str, file: Union[str, BinaryIO], log: logging.Logger,
                  chunk_size: int=DEFAULT_DOWNLOAD_CHUNK_SIZE) -> None:
    
    log.info("Fetching %s...", source)
    r = requests.get(source, stream=True)
    if r.status_code != 200:
        log.error(
            "An error occurred while fetching the model, with code %s" % r.status_code)
        raise ValueError
    if isinstance(file, str):
        os.makedirs(os.path.dirname(file), exist_ok=True)
        f = open(file, "wb")
    else:
        f = file
    try:
        total_length = int(r.headers.get("content-length"))
        num_chunks = math.ceil(total_length / chunk_size)
        if num_chunks == 1:
            f.write(r.content)
        else:
            for chunk in progress_bar(
                    r.iter_content(chunk_size=chunk_size),
                    log,
                    expected_size=num_chunks):
                if chunk:
                    f.write(chunk)
    finally:
        if isinstance(file, str):
            f.close()
75485
def download_links(self, dir_path):
    
    links = self.links
    if not path.exists(dir_path):
      makedirs(dir_path)
    for i, url in enumerate(links):
      if 'start' in self.cseargs:
        i += int(self.cseargs['start'])
      ext = self.cseargs['fileType']
      ext = '.html' if ext == '' else '.' + ext
      file_name = self.cseargs['q'].replace(' ', '_') + '_' + str(i) + ext
      file_path = path.join(dir_path, file_name)
      r = requests.get(url, stream=True)
      if r.status_code == 200:
        with open(file_path, 'wb') as f:
          r.raw.decode_content = True
          shutil.copyfileobj(r.raw, f)
440150
def downloadFileFromUrl(self, url):
		""""""
		fullurl = self.baseUrl + url
		try:
			urlobj = urllib2.urlopen(fullurl)
			contents = urlobj.read()
		except urllib2.HTTPError, e:
			self.printd("HTTP error:", e.code, url)
			return None
		except urllib2.URLError, e:
			self.printd("URL error:", e.code, url)
			return None
		self.printd("Fetched '%s' (size %d bytes)" % (fullurl, len(contents)))
		return contents
1081604
def download(url, filename=None):
	
	# requirements os, shutil, urllib.parse, urllib.request
	if not filename:
		url_parts = urllib.parse.urlparse(url)
		filename = os.path.basename(url_parts.path)
	url_h = urllib.request.urlopen(url)
	with open(filename, 'wb') as file_h:
		shutil.copyfileobj(url_h, file_h)
	url_h.close()
	return
308957
def download_yield_bytes(url, chunk_size=1024 * 1024 * 10):
    
    response = requests.get(url, stream=True)
    # raise error if download was unsuccessful
    response.raise_for_status()

    total_length = response.headers.get('content-length')
    if total_length is not None:
        total_length = float(total_length)
        length_str = "{0:.2f}Mb ".format(total_length / (1024 * 1024))
    else:
        length_str = ""

    print("Yielding {0:s} {1:s}".format(url, length_str))
    for chunk in response.iter_content(chunk_size=chunk_size):
        yield chunk
    response.close()
1025770
def getfile(url, outdir=None):
    
    fn = os.path.split(url)[-1]
    if outdir is not None:
        fn = os.path.join(outdir, fn)
    if not os.path.exists(fn):
        #Find appropriate urlretrieve for Python 2 and 3
        try:
            from urllib.request import urlretrieve
        except ImportError:
            from urllib import urlretrieve 
        print("Retrieving: %s" % url)
        #Add progress bar
        urlretrieve(url, fn)
    return fn
1283406
def download_images(urls:Collection[str], dest:PathOrStr, max_pics:int=1000, max_workers:int=8, timeout=4):
    ""
    urls = open(urls).read().strip().split("\n")[:max_pics]
    dest = Path(dest)
    dest.mkdir(exist_ok=True)
    parallel(partial(_download_image_inner, dest, timeout=timeout), urls, max_workers=max_workers)
1069015
def from_url(cls, url, filename=None, chunk_size=CHUNK_SIZE):
        
        pipe = _WebPipe(url, chunk_size=chunk_size)
        if filename is None:
            filename = pipe.name

        return cls(pipe, filename, chunk_size)
44720
def get_files(self, file_paths):
    

    results = []

    def get_file_thunk(path, interface):
      result = error = None 

      try:
        result = interface.get_file(path)
      except Exception as err:
        error = err
        # important to print immediately because 
        # errors are collected at the end
        print(err) 
      
      content, encoding = result
      content = compression.decompress(content, encoding)

      results.append({
        "filename": path,
        "content": content,
        "error": error,
      })

    for path in file_paths:
      if len(self._threads):
        self.put(partial(get_file_thunk, path))
      else:
        get_file_thunk(path, self._interface)

    desc = 'Downloading' if self.progress else None
    self.wait(desc)

    return results
354651
def get_remote_file_size(self, url):
        
        headers = requests.head(url).headers
        return int(headers['content-length'])
659443
def download_file(self, project, path):
        

        url = self._getUrl("/projects/{}/files/{}".format(project.id, path))
        response = yield from self._session().request("GET", url, auth=self._auth)
        if response.status == 404:
            raise aiohttp.web.HTTPNotFound(text="{} not found on compute".format(path))
        return response
438282
def download(url, target_file, chunk_size=4096):
    
    r = requests.get(url, stream=True)

    with open(target_file, 'w+') as out:
        # And this is why I love Armin Ronacher:
        with click.progressbar(r.iter_content(chunk_size=chunk_size),
                               int(r.headers['Content-Length'])/chunk_size,
                               label='Downloading...') as chunks:
            for chunk in chunks:
                out.write(chunk)
120773
def download_content(**args):
	
	args = validate_args(**args)

	if not args['directory']:
		args['directory'] = args['query'].replace(' ', '-')

	print("Downloading {0} {1} files on topic {2} from {3} and saving to directory: {4}"
		.format(args['limit'], args['file_type'], args['query'], args['website'], args['directory']))
		

	links = search(args['query'], args['engine'], args['website'], args['file_type'], args['limit'])

	if args['parallel']:
		download_parallel(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])
	else:
		download_series(links, args['directory'], args['min_file_size'], args['max_file_size'], args['no_redirects'])