Initiate a streaming download

        Args:
            url (str): The url to download

        Returns:
            A tuple of the content length and the streaming response
966556
def download_file(self, url):
        
        response = requests.get(url, stream=True)
        response.raise_for_status()
        return (int(response.headers.get('content-length', 0)), response)
332674
def download(self, url):
        
        store = DocumentStore()
        fp = store.open(url)
        if fp is None:
            fp = self.options.transport.open(Request(url))
        content = fp.read()
        fp.close()
        ctx = self.plugins.document.loaded(url=url, document=content)
        content = ctx.document
        sax = Parser()
        return sax.parse(string=content)
1047725
def download_file(self, url, local_path):
        
        response = requests.get(url, stream=True)

        with open(local_path, 'wb') as f:
            for chunk in tqdm(response.iter_content(chunk_size=1024), unit='KB'):
                if chunk: # filter out keep-alive new chunks
                    f.write(chunk)
                    f.flush()
895907
def download(self, url, filename, relative=False, headers=None, timeout=5):
        
        request_url = self.base_url + url if relative else url
        floyd_logger.debug("Downloading file from url: {}".format(request_url))

        # Auth headers if present
        request_headers = {}
        if self.auth_header:
            request_headers["Authorization"] = self.auth_header
        # Add any additional headers
        if headers:
            request_headers.update(headers)

        try:
            response = requests.get(request_url,
                                    headers=request_headers,
                                    timeout=timeout,
                                    stream=True)
            self.check_response_status(response)
            with open(filename, 'wb') as f:
                # chunk mode response doesn't have content-length so we are
                # using a custom header here
                content_length = response.headers.get('x-floydhub-content-length')
                if not content_length:
                    content_length = response.headers.get('content-length')
                if content_length:
                    for chunk in progress.bar(response.iter_content(chunk_size=1024),
                                              expected_size=(int(content_length) / 1024) + 1):
                        if chunk:
                            f.write(chunk)
                else:
                    for chunk in response.iter_content(chunk_size=1024):
                        if chunk:
                            f.write(chunk)
            return filename
        except requests.exceptions.ConnectionError as exception:
            floyd_logger.debug("Exception: {}".format(exception))
            sys.exit("Cannot connect to the Floyd server. Check your internet connection.")
202660
def stream_download(cookie, tokens, path):
    '''
    '''
    url = ''.join([
        const.PCS_URL_D,
        'file?method=download',
        '&path=', encoder.encode_uri_component(path),
        '&app_id=250528',
    ])
    req = net.urlopen_without_redirect(url, headers=
            {'Cookie': cookie.header_output()})
    if req:
        return req
    else:
        return None
834384
def _load_url(url):
    
    try:
        response = requests.get(url)
        return BytesIO(response.content)
    except IOError as ex:
        parser.error("{url} could not be loaded remotely! ({ex})".format(url=url, ex=ex))
1146282
def _download_file_from_drive(filename, url):  # pragma: no cover
    
    confirm_token = None

    # Since the file is big, drive will scan it for virus and take it to a
    # warning page. We find the confirm token on this page and append it to the
    # URL to start the download process.
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token

    logger.info("Downloading %s to %s" % (url, filename))

    response = session.get(url, stream=True)
    # Now begin the download.
    chunk_size = 16 * 1024
    with open(filename, "wb") as f:
        for chunk in response.iter_content(chunk_size):
            if chunk:
                f.write(chunk)

    # Print newline to clear the carriage return from the download progress
    statinfo = os.stat(filename)
    logger.info("Successfully downloaded %s, %s bytes." % (filename, statinfo.st_size))
639413
def _download_from_url(self, url):
        
        ext = get_file_extension(url)
        if "?" in url:
            ext = get_file_extension(os.path.splitext(url.split("?")[0]))
        filepath = "/tmp/%s.%s" % (uuid.uuid4().hex, ext)
        request.urlretrieve(url, filepath)
        return filepath
335822
def download_from_url(url, path):
    """"""
    def process_response(r):
        chunk_size = 16 * 1024
        total_size = int(r.headers.get('Content-length', 0))
        with open(path, "wb") as file:
            with tqdm(total=total_size, unit='B',
                      unit_scale=1, desc=path.split('/')[-1]) as t:
                for chunk in r.iter_content(chunk_size):
                    if chunk:
                        file.write(chunk)
                        t.update(len(chunk))

    if 'drive.google.com' not in url:
        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}, stream=True)
        process_response(response)
        return

    print('downloading from Google Drive; may take a few minutes')
    confirm_token = None
    session = requests.Session()
    response = session.get(url, stream=True)
    for k, v in response.cookies.items():
        if k.startswith("download_warning"):
            confirm_token = v

    if confirm_token:
        url = url + "&confirm=" + confirm_token
        response = session.get(url, stream=True)

    process_response(response)
846037
def download_to_bytesio(url):
    
    logger.info("Downloading url: {0}".format(url))
    r = cleaned_request('get', url, stream=True)
    stream = io.BytesIO()
    total_length = int(r.headers.get('content-length'))
    for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1):
        if chunk:
            stream.write(chunk)
    stream.seek(0)
    return stream
1246003
def download_url_to_path(self, url, path, force=False):
        

        path_exists = isfile(path)
        if not path_exists or force:

            dir = dirname(path)
            if not exists(dir):
                if self.verbose:
                    print("CREATE %s" % dir)
                makedirs(dir)

            try:
                u = urlopen(url)
            except HTTPError as e:
                u = None
                print("HTTP error code %r.  Please check you ~/.netrc has the correct authorization" % e.code)

            if u:
                with open(path, 'wb') as file:
                    meta = u.info()
                    meta_func = meta.getheaders \
                        if hasattr(meta, 'getheaders') else meta.get_all
                    meta_length = meta_func("Content-Length")
                    file_size = None
                    if meta_length:
                        file_size = int(meta_length[0])
                    if self.verbose:
                        print("Downloading: {0} Bytes: {1}".format(url, file_size))

                    file_size_dl = 0
                    block_sz = 8192
                    while True:
                        buffer = u.read(block_sz)
                        if not buffer:
                            break
                        file_size_dl += len(buffer)
                        file.write(buffer)

                if self.verbose:
                    if path_exists:
                        print("OVERWRITING %s" % path)
                    else:
                        print("CREATE %s" % path)

        elif self.verbose:
            print("FOUND %s (already downloaded)" % path)
673708
def _grabContentFromUrl(self, url):
        
        # Defining an empty object for the response
        info = {}
                
        # This part has to be modified...        
        try:            
            # Configuring the socket
            queryURL = "http://" + self.info["host"] + ":" + self.info["port"] + "/" + url
            response = urllib2.urlopen(queryURL)

            # Rebuilding data to be processed
            data = str(response.headers) + "\n"

            data += response.read()

            # Processing data as expected
            info = self._createDataStructure(data)     
            
        # Try to make the errors clear for other users
        except Exception, e:
            errMsg = "ERROR Exception. Something seems to be wrong with the Zeronet Bundler."   
            raise Exception( errMsg + " " + str(e))        
            
        return info
390182
def download_url(url, back_off=True, **kwargs):
    
    if back_off:
        return _download_with_backoff(url, as_file=True, **kwargs)
    else:
        return _download_without_backoff(url, as_file=True, **kwargs)
951332
def _download(self, url, path=None, force=False):
        """"""
        if path is None:
            path = url.split('/')[-1]

        # Make dir if non existent
        folder = os.path.dirname(os.path.abspath(path))

        if not os.path.isdir(folder):
            os.makedirs(folder)

        # Start actual download
        try:
            r = requests.get(url, stream=True, proxies=self.proxy_servers)
        except Exception as error:
            print('ERROR', 'here', error)
            logger.error(str(error))
            # Break if error found!
#            self._sig_download_finished.emit(url, path)
#            return path

        total_size = int(r.headers.get('Content-Length', 0))

        # Check if file exists
        if os.path.isfile(path) and not force:
            file_size = os.path.getsize(path)

            # Check if existing file matches size of requested file
            if file_size == total_size:
                self._sig_download_finished.emit(url, path)
                return path

        # File not found or file size did not match. Download file.
        progress_size = 0
        with open(path, 'wb') as f:
            for chunk in r.iter_content(chunk_size=self._chunk_size):
                if chunk:
                    f.write(chunk)
                    progress_size += len(chunk)
                    self._sig_download_progress.emit(url, path,
                                                     progress_size,
                                                     total_size)
            self._sig_download_finished.emit(url, path)

        return path
1103529
def __stream_request(self, method, url, request_args, headers=None):
        
        headers = headers if headers else {}
        response = self.__request(method, url, request_args,
                                  headers=headers, stream=True)
        lines = response.iter_lines(ITER_LINES_CHUNKSIZE)
        for line in lines:
            if line:
                data = json.loads(line.decode("utf-8"))
                yield data
252255
def download_file(url, path=None, clobber=False):
    
    if path is None:
        local_filename = os.path.join(directory, url.split('/')[-1])
    else:
        local_filename = path

    if os.path.exists(local_filename) and not clobber:
        logging.info('{} exists; not downloading.'.format(local_filename))
        return local_filename
        
    # NOTE the stream=True parameter
    r = requests.get(url, stream=True)
    with open(local_filename, 'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)
                #f.flush() commented by recommendation from J.F.Sebastian
    return local_filename
364478
def download_file(url, path, session=None, params=None):
    

    if url[0:2] == '//':
        url = 'https://' + url[2:]

    # Use a temporary file so that we don't import incomplete files.
    tmp_path = path + '.tmp'

    if session and params:
        r = session.get( url, params=params, stream=True )
    elif session and not params:
        r = session.get( url, stream=True )
    else:
        r = requests.get(url, stream=True)
    with open(tmp_path, 'wb') as f:
        total_length = int(r.headers.get('content-length', 0))
        for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()

    os.rename(tmp_path, path)

    return path