Gets the contents of the given url.
976610
def list_rocs_files(url=ROCS_URL):
    
    soup = BeautifulSoup(get(url))
    if not url.endswith('/'):
        url += '/'
    files = []
    for elem in soup.findAll('a'):
        if elem['href'].startswith('?'):
            continue
        if elem.string.lower() == 'parent directory':
            continue
        files.append(url + elem['href'])
    return files
740433
public static List<String> extractUriParams(String url) {
		List<String> outParams = new ArrayList<>();
		if (StringUtils.hasText(url)) {
			String[] split = StringUtils.split(url, "/");
			for (String part : split) {
				int indexOfStart = part.indexOf("{");
				int indexOfEnd = part.indexOf("}");
				if (indexOfStart != -1 && indexOfEnd != -1 && indexOfStart < indexOfEnd) {
					outParams.add(part.substring(indexOfStart + 1, indexOfEnd));
				}
			}

		}
		return outParams;
	}
525169
def _get_url(url):
        
        if isinstance(url, urlparse.ParseResult):
            return urlparse.urlunparse(url)
        else:
            return url
35782
def _get_data(url):
    """"""
    if urllib_parse.urlparse(url).scheme in ('http', 'https'):
        resp = urllib_request.urlopen(url)
        encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            data = data.decode('utf-8')
        elif encoding == 'gzip':
            data = BytesIO(data)
            data = gzip.GzipFile(fileobj=data).read().decode('utf-8')
        else:
            raise RuntimeError('unknown encoding')
    else:
        with codecs.open(url, mode='r', encoding='utf-8') as fid:
            data = fid.read()

    return data
347943
def get_url(url: str) -> Union[dict, int, float, str]:
        ''''''

        request = Request(url, headers={"User-Agent": "pypeerassets"})
        response = cast(HTTPResponse, urlopen(request))
        if response.status != 200:
            raise Exception(response.reason)
        return json.loads(response.read().decode())
1279513
def _get(url, params=None):
        """"""
        try:
            response = requests.get(url, params=params)
            response.raise_for_status()
            # If JSON fails, return raw data
            # (e.g. when downloading CSV job logs).
            try:
                return response.json()
            except ValueError:
                return response.text
        except NameError:
            url = '{0}?{1}'.format(url, urllib.urlencode(params))
            return json.loads(urllib2.urlopen(url).read().decode(ENCODING))
1346905
public static String getContentPost(String stringUrl, Map<String, String> parameters,
          String input, String charset) throws IOException {
      return getContentPost(stringUrl, parameters, input, charset, false);
  }
140437
def parse(cls, url, default_scheme='http', encoding='utf-8'):
        ''''''
        if url is None:
            return None

        url = url.strip()
        if frozenset(url) & C0_CONTROL_SET:
            raise ValueError('URL contains control codes: {}'.format(ascii(url)))

        scheme, sep, remaining = url.partition(':')

        if not scheme:
            raise ValueError('URL missing scheme: {}'.format(ascii(url)))

        scheme = scheme.lower()

        if not sep and default_scheme:
            # Likely something like example.com/mystuff
            remaining = url
            scheme = default_scheme
        elif not sep:
            raise ValueError('URI missing colon: {}'.format(ascii(url)))

        if default_scheme and '.' in scheme or scheme == 'localhost':
            # Maybe something like example.com:8080/mystuff or
            # maybe localhost:8080/mystuff
            remaining = '{}:{}'.format(scheme, remaining)
            scheme = default_scheme

        info = URLInfo()
        info.encoding = encoding

        if scheme not in RELATIVE_SCHEME_DEFAULT_PORTS:
            info.raw = url
            info.scheme = scheme
            info.path = remaining

            return info

        if remaining.startswith('//'):
            remaining = remaining[2:]

        path_index = remaining.find('/')
        query_index = remaining.find('?')
        fragment_index = remaining.find('#')

        try:
            index_tuple = (path_index, query_index, fragment_index)
            authority_index = min(num for num in index_tuple if num >= 0)
        except ValueError:
            authority_index = len(remaining)

        authority = remaining[:authority_index]
        resource = remaining[authority_index:]

        try:
            index_tuple = (query_index, fragment_index)
            path_index = min(num for num in index_tuple if num >= 0)
        except ValueError:
            path_index = len(remaining)

        path = remaining[authority_index + 1:path_index] or '/'

        if fragment_index >= 0:
            query_index = fragment_index
        else:
            query_index = len(remaining)

        query = remaining[path_index + 1:query_index]
        fragment = remaining[query_index + 1:]

        userinfo, host = cls.parse_authority(authority)
        hostname, port = cls.parse_host(host)
        username, password = cls.parse_userinfo(userinfo)

        if not hostname:
            raise ValueError('Hostname is empty: {}'.format(ascii(url)))

        info.raw = url
        info.scheme = scheme
        info.authority = authority
        info.path = normalize_path(path, encoding=encoding)
        info.query = normalize_query(query, encoding=encoding)
        info.fragment = normalize_fragment(fragment, encoding=encoding)

        info.userinfo = userinfo
        info.username = percent_decode(username, encoding=encoding)
        info.password = percent_decode(password, encoding=encoding)

        info.host = host
        info.hostname = hostname
        info.port = port or RELATIVE_SCHEME_DEFAULT_PORTS[scheme]

        info.resource = resource

        return info
488551
public RFuture<String> get(String url) {
    return req(url).execute().map(GET_PAYLOAD);
  }
697828
def _get_content(self, url):
        ""
        target_url = self._db_url + '/' + unquote(url)    # .encode('utf-8'))
        log.debug("Opening '{0}'".format(target_url))
        try:
            f = self.opener.open(target_url)
        except HTTPError as e:
            log.error("HTTP error, your session may be expired.")
            log.error(e)
            if input("Request new permanent session and retry? (y/n)") in 'yY':
                self.request_permanent_session()
                return self._get_content(url)
            else:
                return None
        log.debug("Accessing '{0}'".format(target_url))
        try:
            content = f.read()
        except IncompleteRead as icread:
            log.critical(
                "Incomplete data received from the DB, " +
                "the data could be corrupted."
            )
            content = icread.partial
        log.debug("Got {0} bytes of data.".format(len(content)))
        return content.decode('utf-8')
445971
def _get_html(self, url):
        
        self.log.info(u"/GET {}".format(url))
        r = requests.get(url)
        if hasattr(r, 'from_cache'):
            if r.from_cache:
                self.log.info("(from cache)")

        if r.status_code != 200:
            throw_request_err(r)

        return r.content
670605
def _fetch_url(url):
    
    try:
        resp = urllib2.urlopen(_Request(url))
    except urllib2.URLError:
        if 'wikileaks.org' in url:
            resp = urllib2.urlopen(_Request(url.replace('wikileaks.org', 'wikileaks.ch')))
        else:
            raise
    if resp.info().get('Content-Encoding') == 'gzip':
        return gzip.GzipFile(fileobj=StringIO(resp.read())).read().decode('utf-8')
    return resp.read().decode('utf-8')
813173
def _get_content(cls, url, headers=HTTP_HEADERS):
        
        session = requests.Session()
        return session.get(url, headers=headers)
873363
private String getContent(String url) throws RottenTomatoesException {
        LOG.trace("Requesting: {}", url);
        try {
            final HttpGet httpGet = new HttpGet(url);
            httpGet.addHeader("accept", "application/json");
            final DigestedResponse response = DigestedResponseReader.requestContent(httpClient, httpGet, charset);

            if (response.getStatusCode() >= HTTP_STATUS_500) {
                throw new RottenTomatoesException(ApiExceptionType.HTTP_503_ERROR, response.getContent(), response.getStatusCode(), url);
            } else if (response.getStatusCode() >= HTTP_STATUS_300) {
                throw new RottenTomatoesException(ApiExceptionType.HTTP_404_ERROR, response.getContent(), response.getStatusCode(), url);
            }

            return response.getContent();
        } catch (IOException ex) {
            throw new RottenTomatoesException(ApiExceptionType.CONNECTION_ERROR, "Error retrieving URL", url, ex);
        }
    }
1010857
def get(url):
    """"""
    response = urllib.request.urlopen(url)
    data = response.read()
    data = data.decode("utf-8")
    data = json.loads(data)
    return data
205456
def http_get(url, filename=None):
    """"""
    try:
        ret = requests.get(url)
    except requests.exceptions.SSLError as error:
        _LOGGER.error(error)
        return False

    if ret.status_code != 200:
        return False

    if filename is None:
        return ret.content

    with open(filename, 'wb') as data:
        data.write(ret.content)
    return True
465709
def get(url):
    
    text = urlopen(url).read()
    soup = BeautifulSoup(text)
    return Response(text=text, soup=soup)