Stream file from url and hash it using MD5. Must call setup method first.

        Args:
            url (str): URL to download

        Returns:
            str: MD5 hash of file
185396
def hash_stream(self, url):
        # type: (str) -> str
        
        md5hash = hashlib.md5()
        try:
            for chunk in self.response.iter_content(chunk_size=10240):
                if chunk:  # filter out keep-alive new chunks
                    md5hash.update(chunk)
            return md5hash.hexdigest()
        except Exception as e:
            raisefrom(DownloadError, 'Download of %s failed in retrieval of stream!' % url, e)
908386
def upload_from_url(self, url):
        
        self.check_token()
        
        params = {'fetchUrl': url}
        r = requests.get(FETCH_URL_ENDPOINT, params=params)

        if r.status_code != 200:
            raise GfycatClientError('Error fetching the URL', r.status_code)

        response = r.json()
        if 'error' in response:
            raise GfycatClientError(response['error'])

        return response
29445
def url_to_filename(url: str, etag: str = None) -> str:
    
    url_bytes = url.encode('utf-8')
    url_hash = sha256(url_bytes)
    filename = url_hash.hexdigest()

    if etag:
        etag_bytes = etag.encode('utf-8')
        etag_hash = sha256(etag_bytes)
        filename += '.' + etag_hash.hexdigest()

    return filename
846037
def download_to_bytesio(url):
    
    logger.info("Downloading url: {0}".format(url))
    r = cleaned_request('get', url, stream=True)
    stream = io.BytesIO()
    total_length = int(r.headers.get('content-length'))
    for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1):
        if chunk:
            stream.write(chunk)
    stream.seek(0)
    return stream
241358
def download(url, path)
      if url.match(FILE_URL)
        FileUtils.cp(url.sub(FILE_URL, ''), path)
      else
        File.open(path, 'w+') do |file|
          Net::HTTP.get_response(URI(url)) do |response|
            response.read_body do |chunk|
              file.write chunk
            end
          end
        end
      end
      path
    end
543767
def get_path_to_file_from_url(cls, url):
        
        file_name = urlparse(url).path.split('/')[-1]
        return os.path.join(PYUNIPROT_DATA_DIR, file_name)
843883
def download(cls, url, filename=None):
        
        return utility.download(url, cls.directory(), filename)
431414
def download(url):
    
    filepath = get_file(fname='tmp.zip', origin=url, extract=True)
    base_dir = os.path.dirname(filepath)
    weights_file = os.path.join(base_dir, 'weights.h5')
    params_file = os.path.join(base_dir, 'params.json')
    preprocessor_file = os.path.join(base_dir, 'preprocessor.pickle')

    return weights_file, params_file, preprocessor_file
18631
def _download(url):
    
    fh = StringIO()

    for line in get(url):
        fh.write(line)

    fh.seek(0)
    return fh
579203
def sha1sum_url(url):
    '''

    '''
    import hashlib
    import urllib
    from contextlib import closing

    sha1 = hashlib.sha1()
    chunk_size = sha1.block_size * 8192

    max_file_size = 10 * 1024 * 1024  # 10Mo limit in case of very large file

    total_read = 0
    with closing(urllib.urlopen(url)) as url_obj:
        for chunk in iter(lambda: url_obj.read(chunk_size), b''):
            sha1.update(chunk)
            total_read += chunk_size
            if total_read > max_file_size:
                break

    return sha1.hexdigest()
736312
def get_md5_file_hash(filename):
    
    import hashlib
    BLOCKSIZE = 65536
    hasher = hashlib.md5()
    with open(filename, 'rb') as afile:
        buf = afile.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(BLOCKSIZE)
    return hasher.hexdigest()
364478
def download_file(url, path, session=None, params=None):
    

    if url[0:2] == '//':
        url = 'https://' + url[2:]

    # Use a temporary file so that we don't import incomplete files.
    tmp_path = path + '.tmp'

    if session and params:
        r = session.get( url, params=params, stream=True )
    elif session and not params:
        r = session.get( url, stream=True )
    else:
        r = requests.get(url, stream=True)
    with open(tmp_path, 'wb') as f:
        total_length = int(r.headers.get('content-length', 0))
        for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
                f.flush()

    os.rename(tmp_path, path)

    return path
99049
def download(url, file_name):
    r = requests.get(url, stream=True)
    file_size = int(r.headers['Content-length'])
    '''
    
    '''
    file_exists = False
    if os.path.isfile(file_name):
        local_file_size = os.path.getsize(file_name)
        if local_file_size == file_size:
            sha1_file = file_name + '.sha1'
            if os.path.isfile(sha1_file):
                print('sha1 found')
                with open(sha1_file) as f:
                    expected_sha1 = f.read()
                BLOCKSIZE = 65536
                sha1 = hashlib.sha1()
                with open(file_name) as f:
                    buff = f.read(BLOCKSIZE)
                    while len(buff) > 0:
                        sha1.update(buff)
                        buff = f.read(BLOCKSIZE)
                if expected_sha1 == sha1:
                    file_exists = True
                else:
                    print("File corrupt. Downloading again.")
                    os.remove(file_name)
            else:
                file_exists = True
        else:
            print("File corrupt. Downloading again.")
            os.remove(file_name)
    if not file_exists:
        factor = int(math.floor(math.log(file_size) / math.log(1024)))
        display_file_size = str(file_size / 1024 ** factor) + \
            ['B', 'KB', 'MB', 'GB', 'TB', 'PB'][factor]
        print("Source: " + url)
        print("Destination " + file_name)
        print("Size: " + display_file_size)
        file_size_dl = 0
        block_sz = 8192
        f = open(file_name, 'wb')
        pbar = ProgressBar(file_size)
        for chunk in r.iter_content(chunk_size=block_sz):
            if not chunk:
                continue
            chunk_size = len(chunk)
            file_size_dl += chunk_size
            f.write(chunk)
            pbar.update(chunk_size)
            # status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
            # status = status + chr(8)*(len(status)+1)
            # print(status)
        f.close()
    else:
        print("File already exists - " + file_name)
        return True
17462
def ensure_file(url, path, md5sum=None):
    

    if not os.path.isfile(path) or (md5sum and md5sum != file_md5(path)):
        download_file(url, path)
271685
def file_download(user_session, request):
    
    logger.debug('Download file: %s', vars(request))
    response = user_session.session.get(
        request.href,
        params=request.params,
        headers=request.headers,
        stream=True)

    if response.status_code == 200:
        logger.debug('Streaming to file... Content length: %s', len(response.content))
        try:
            path = os.path.abspath(request.filename)
            logger.debug('Operation: %s, saving to file: %s', request.href, path)

            with open(path, "wb") as handle:
                for chunk in response.iter_content(chunk_size=1024):
                    if chunk:
                        handle.write(chunk)
                        handle.flush()
        except IOError as e:
            raise IOError('Error attempting to save to file: {}'.format(e))

        result = SMCResult(response, user_session=user_session)
        result.content = path
        return result
    else:
        raise SMCOperationFailure(response)
264305
def _md5(path: pathlib.PurePath):
        
        hash_ = hashlib.md5()
        with open(path, 'rb') as f:
            for chunk in iter(lambda: f.read(4096), b''):
                hash_.update(chunk)
        return hash_.hexdigest()
1193817
def from_file_url(url):
    
    if url.startswith('file://'):
        url = url[len('file://'):].replace('/', os.path.sep)

    return url