Function to fetch files using requests

    Works with https authentication
195370
def getfile2(url, auth=None, outdir=None):
    
    import requests
    print("Retrieving: %s" % url)
    fn = os.path.split(url)[-1]
    if outdir is not None:
        fn = os.path.join(outdir, fn)
    if auth is not None:
        r = requests.get(url, stream=True, auth=auth)
    else:
        r = requests.get(url, stream=True)
    chunk_size = 1000000
    with open(fn, 'wb') as fd:
        for chunk in r.iter_content(chunk_size):
            fd.write(chunk)
925977
function loadFromFileOrHttp (fileOrUrlOrData) {
  // If this is not a string,
  // it is probably already the raw data.
  if (typeof fileOrUrlOrData !== 'string') {
    return Q(fileOrUrlOrData)
  }
  // otherwise load data from url or file
  if (fileOrUrlOrData.match(/^https?:\/\//)) {
    // Use the "request" package to download data
    return httpGet(fileOrUrlOrData, {
      redirect: true,
      headers: {
        'User-Agent': 'Bootprint/' + require('./package').version
      }
    }).then(function (result) {
      if (result.status !== 200) {
        var error = new Error('HTTP request failed with code ' + result.status)
        error.result = result
        throw error
      }
      return yaml.safeLoad(result.data, {json: true})
    }, function (error) {
      if (error.status) {
        throw new Error('Got ' + error.status + ' ' + error.data + ' when requesting ' + error.url, 'E_HTTP')
      } else {
        throw error
      }
    })
  } else {
    return Q.nfcall(fs.readFile, fileOrUrlOrData, 'utf8').then(function (data) {
      return yaml.safeLoad(data, {json: true})
    })
  }
}
48787
def fetch(self, output_path, last_etag=None, lastmodified=None, progress_bar=False):
        
        req_headers = {}
        usr_agent = get_user_agent()
        if usr_agent is not None:
            req_headers['User-Agent'] = usr_agent

        if last_etag is not None:
            req_headers['If-None-Match'] = '"' + last_etag + '"'
        if lastmodified is not None:
            req_headers['If-Modified-Since'] = lastmodified

        response = requests.get(self.url, headers=req_headers, stream=progress_bar)

        if response.status_code == 304:
            # Conditions didn't match, no content returned
            return None
        if response.status_code != 200:
            raise requests.HTTPError

        with open(output_path, 'wb') as f:
            if progress_bar:
                total_size = long(response.headers.get('content-length'))
                for chunk in progress.bar(response.iter_content(chunk_size=self.chunk_size),
                                          expected_size=(total_size / self.chunk_size) + 1):
                    if chunk:
                        f.write(chunk)
                        f.flush()
            else:
                f.write(response.content)

        headers = normalize_headers(response.headers)

        if 'etag' not in headers:
            headers['etag'] = fs_util.calc_file_md5(output_path)

        return headers
312259
static function fetch( $http_name )
    {
        if ( !isset( $GLOBALS["eZHTTPFile-$http_name"] ) ||
             !( $GLOBALS["eZHTTPFile-$http_name"] instanceof eZHTTPFile ) )
        {
            $GLOBALS["eZHTTPFile-$http_name"] = null;

            if ( isset( $_FILES[$http_name] ) and
                 $_FILES[$http_name]["name"] != "" )
            {
                $mimeType = eZMimeType::findByURL( $_FILES[$http_name]['name'] );
                $_FILES[$http_name]['type'] = $mimeType['name'];
                $GLOBALS["eZHTTPFile-$http_name"] = new eZHTTPFile( $http_name, $_FILES[$http_name] );
            }
            else
            {
                eZDebug::writeError( "Unknown file for post variable: $http_name",
                                     "eZHTTPFile" );
            }
        }
        return $GLOBALS["eZHTTPFile-$http_name"];
    }
804966
function fetchHTTPFile( $httpFileIdentifier, &$errors, &$file, &$mimeData )
    {
        $returnCode = eZHTTPFile::canFetch( $httpFileIdentifier, 0 );
        if ( $returnCode !== eZHTTPFile::UPLOADEDFILE_OK && $returnCode !== true )
        {
            switch ( $returnCode )
            {
                case eZHTTPFile::UPLOADEDFILE_DOES_NOT_EXIST:
                    $errors[] = array( 'description' => ezpI18n::tr( 'kernel/content/upload', 'A file is required for upload, no file were found.' ) );
                    break;
                case eZHTTPFile::UPLOADEDFILE_EXCEEDS_PHP_LIMIT:
                case eZHTTPFile::UPLOADEDFILE_EXCEEDS_MAX_SIZE:
                    $errors[] = array( 'description' => ezpI18n::tr( 'kernel/content/upload', 'The uploaded file size is above the maximum limit.' ) );
                    break;
                case eZHTTPFile::UPLOADEDFILE_MISSING_TMP_DIR:
                case eZHTTPFile::UPLOADEDFILE_CANT_WRITE:
                case eZHTTPFile::UPLOADEDFILE_UNKNOWN_ERROR:
                    $errors[] = array( 'description' => ezpI18n::tr( 'kernel/content/upload', 'A system error occured while writing the uploaded file.' ) );
                    break;
            }
            return false;
        }

        $file = eZHTTPFile::fetch( $httpFileIdentifier );
        if ( !( $file instanceof eZHTTPFile ) )
        {
            $errors[] = array( 'description' => ezpI18n::tr( 'kernel/content/upload',
                                                        'Expected a eZHTTPFile object but got nothing.' ) );
            return false;
        }

        $mimeData = eZMimeType::findByFileContents( $file->attribute( "original_filename" ) );

        return true;
    }
856088
def wget(url):
    
    import urllib.parse
    request = urllib.request.urlopen(url)
    filestring = request.read()
    return filestring
945265
def download_file_insecure_to_io(url, target_file=None, headers={}):
    

    src = None
    try:
        req = Request(
            url,
            data=None,
            headers=headers
        )

        src = urlopen(req)

        # Read/write all in one block, so we don't create a corrupt file
        # if the download is interrupted.
        data = src.read()
        target_file.write(data)
    finally:
        if src:
            src.close()
106625
def get_stream(path, response_block, params = nil, headers = {}, auth = true)
      request(path: path, query: params, headers: headers, response_block: response_block, auth: auth, gzip: false)
    end
389521
def get(addr, args={},  headers={}, version='HTTP/1.1', auth=()):

    

    addr    = addr.strip().rstrip()
    url     = urlparse(addr)
    default = {
    'user-agent':'Websnake/1.0.0', 
    'accept-charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
    'connection':'close',
    'host': url.hostname}

    default.update(headers)
    args = '?%s' % urlencode(args) if args else ''

    if auth: default['authorization'] = build_auth(*auth)

    data = 'GET %s%s %s\r\n' % (url.path + ('?' + url.query if \
    url.query else ''), args, version)
    data = data + build_headers(default)
    port = url.port if url.port else getservbyname(url.scheme)
    data = data.encode('utf8')

    return create_con_ssl(url.hostname, port, data) \
    if url.scheme == 'https' else create_con(url.hostname, port, data)
998473
function fetch(opts, cb) {
  var options = {
    url: 'https://' + opts.host + ':' + opts.port + opts.path,
    timeout: opts.timeout,
    agentOptions: {
      cert: opts.clicert,
      key: opts.clikey,
      passphrase: opts.clipass,
      ca: opts.srvca,
      // TODO : remove this
      rejectUnauthorized: false } };

  fetchFromCache(options, opts, function(err, cached) {
    if (err) {
      cb(err);
    } else if (cached) {
      cb(null, cached);
    } else {
      fetchFromServer(options, opts, cb);
    }
  });
}
775118
def _fetch(self, request: Request, is_file: bool):
        '''
        '''
        _logger.info(_('Fetching ‘{url}’.'), url=request.url)

        self._item_session.request = request
        response = None

        try:
            with self._processor.ftp_client.session() as session:
                if is_file:
                    response = yield from session.start(request)
                else:
                    response = yield from session.start_listing(request)

                self._item_session.response = response

                action = self._result_rule.handle_pre_response(
                    self._item_session
                )

                if action in (Actions.RETRY, Actions.FINISH):
                    raise HookPreResponseBreak()

                self._file_writer_session.process_response(response)

                if not response.body:
                    response.body = Body(
                        directory=self._item_session.app_session.root_path,
                        hint='resp_cb')

                duration_timeout = self._fetch_rule.duration_timeout

                if is_file:
                    yield from session.download(
                        response.body, duration_timeout=duration_timeout)
                else:
                    yield from session.download_listing(
                        response.body, duration_timeout=duration_timeout)

        except HookPreResponseBreak:
            if response:
                response.body.close()

        except REMOTE_ERRORS as error:
            self._log_error(request, error)

            self._result_rule.handle_error(self._item_session, error)

            wait_time = self._result_rule.get_wait_time(
                self._item_session, error=error
            )

            if response:
                response.body.close()

            return wait_time
        else:
            self._log_response(request, response)
            self._handle_response(request, response)

            wait_time = self._result_rule.get_wait_time(
                self._item_session
            )

            if is_file and \
                    self._processor.fetch_params.preserve_permissions and \
                    hasattr(response.body, 'name'):
                yield from self._apply_unix_permissions(request, response)

            response.body.close()

            return wait_time
651228
def _api_get(self, url, **kwargs):
        
        kwargs['url'] = self.url + url
        kwargs['auth'] = self.auth

        headers = deepcopy(self.headers)
        headers.update(kwargs.get('headers', {}))
        kwargs['headers'] = headers
        return self._get(**kwargs)
1285865
function download(url) {
  if (!validator.isURL(url)) return Promise.reject(error(400, 'Invalid URL: ' + url));
  // download in progress
  if (progress[sha]) return progress[sha];
  var sha = hash(url);
  return progress[sha] = cache.access(sha).then(function (filename) {
    if (filename) return filename;

    return request(url)
    .redirects(3)
    .agent(false)
    .then(function (response) {
      assert(response.status < 400, response.status, 'Got status code ' + response.status + ' from ' + url);
      assert(response.status === 200, 'Got status code ' + response.status + ' from ' + url);

      debug('Downloading %s -> %s', url, sha);

      return cache.copy(sha, response.response);
    });
  }).then(function (filename) {
    delete progress[sha];
    return filename;
  }, /* istanbul ignore next */ function (err) {
    delete progress[sha];
    throw err;
  });
}
725753
def _request(self, url, method="GET", params=None, api_call=None):
        """"""
        method = method.lower()
        params = params or {}
        func = getattr(requests, method)
        requests_args = {}
        if method == "get" or method == "delete":
            requests_args["params"] = params
        else:
            if params.get("json"):
                requests_args["json"] = params.get("json")
            if params.get("files"):
                requests_args["files"] = params.get("files")
            if params.get("data"):
                requests_args["data"] = params.get("data")
        try:
            response = func(url, **requests_args)
        except requests.RequestException as e:
            raise SafecastPyError(str(e))
        # greater than 304 (not modified) is an error
        if response.status_code > 304:
            if response.status_code == 401:
                raise SafecastPyAuthError(response.json().get("error"))
            if response.status_code in [422]:
                raise SafecastPyError(response.json().get("errors"))
            raise SafecastPyError(response.content, error_code=response.status_code)
        try:
            if response.status_code == 204:
                content = response.content
            else:
                content = response.json()
        except ValueError:
            raise SafecastPyError(
                "Response was not valid JSON. \
                               Unable to decode."
            )
        return content
300807
def _fetch(url, ssl_verify = True):
    
    req = Request(url)
    if ssl_verify:
        page = urlopen(req)
    else:
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE

        page = urlopen(req, context=ctx)
    content = page.read().decode('utf-8')
    page.close()
    return content
29851
def fetch(url, timeout=5, userAgent=None, only_mime_types=None):
    
    headers = {}
    if userAgent:
        headers['User-agent'] = str(userAgent)
    else:
        headers['User-agent'] = ua.random

    #request = Request(url=url, headers=headers)
    #response = urlopen(request, timeout=timeout)
    response = requests.get(url, headers=headers, timeout=timeout)

    # Return nothing of the content isn't one of the target mime-types.
    if only_mime_types:
        assert isinstance(only_mime_types, (tuple, list))
        # Check for mimetype by looking at pattern in the URL.
        # Not super accurate, but very fast.
        ct, mt_encoding = mimetypes.guess_type(url)
        # Then check for mimetype by actually requesting the resource and
        # looking at the response.
        # More accurate, but slower since we actually have to send a request.
        if not ct:
            response_info = response.info()
            ct = (response_info.getheader('Content-Type') or '').split(';')[0]
        #TODO:if still undefined, use magic.Magic(mime=True).from_file(url)?
        if ct not in only_mime_types:
            return
    try:
        #return response.read()
        return response.text
    except httplib.IncompleteRead as e:
        # This should rarely happen, and is often the fault of the server
        # sending a malformed response.
        #TODO:just abandon all content and return '' instead?
        return e.partial
713053
def fetch(*args, **kwargs):
    
    data = kwargs.get('data', None)
    files = kwargs.get('files', {})

    if data and isinstance(data, (basestring, dict)) or files:
        return post(*args, **kwargs)
    return get(*args, **kwargs)