Performs a multiprocess depth-first-search of the catalog references
        and yields a URL for each leaf dataset found
        :param str url: URL for the current catalog
        :param requests.auth.AuthBase auth: requets auth object to use
335521
def _run(self, url, auth):
        '''
        
        '''
        if url in self.visited:
            logger.debug("Skipping %s (already crawled)" % url)
            return
        self.visited.append(url)

        logger.info("Crawling: %s" % url)
        url = self._get_catalog_url(url)

        # Get an etree object
        xml_content = request_xml(url, auth)
        for ds in self._build_catalog(url, xml_content):
            yield ds
1226525
async def _async_request_soup(url):
    '''
    
    '''
    from bs4 import BeautifulSoup
    import aiohttp
    _LOGGER.debug('GET %s', url)
    async with aiohttp.ClientSession() as session:
        resp = await session.get(url)
        text = await resp.text()
        return BeautifulSoup(text, 'html.parser')
51762
def url(self, url):
        
        if url is None:
            raise ValueError("Invalid value for `url`, must not be `None`")
        if url is not None and len(url) > 4096:
            raise ValueError("Invalid value for `url`, length must be less than or equal to `4096`")
        if url is not None and len(url) < 1:
            raise ValueError("Invalid value for `url`, length must be greater than or equal to `1`")
        if url is not None and not re.search('^https?:.*', url):
            raise ValueError("Invalid value for `url`, must be a follow pattern or equal to `/^https?:.*/`")

        self._url = url
474146
def found_url(self, url, name, base):
        """"""
        assert isinstance(url, unicode) or url is None, repr(url)
        self.callback(url, line=self.parser.last_lineno(),
                      column=self.parser.last_column(), name=name, base=base)
1021444
def URL(base, path, segments=None, defaults=None):
    
    # Make a copy of the Segments class
    url_class = type(Segments.__name__, Segments.__bases__,
                     dict(Segments.__dict__))
    segments = [] if segments is None else segments
    defaults = [] if defaults is None else defaults
    # For each segment attach a property capable of getting and setting it
    for segment in segments:
        setattr(url_class, segment, url_class._segment(segment))
    # Instantiate the class with the actual parameters
    return url_class(base, path, segments, defaults)
204409
def __find_rows(self, find_url, **attributes):
        
        req = self.url + find_url

        # Add range and sort parameters
        params = {
            "range": attributes.get("range", "all"),
            "sort": attributes.get("sort", [])
        }

        # Add body
        data = {
            "query": attributes.get("query", {})
        }

        try:
            return requests.post(req, params=params, json=data, proxies=self.proxies, auth=self.auth, verify=self.cert)
        except requests.exceptions.RequestException as e:
            raise TheHiveException("Error: {}".format(e))
47032
def url_to_query(url):
        
        q = {}

        # pylint: disable=unpacking-non-sequence
        (ignore, ignore, path,
         ignore, query, ignore) = urlparse(url)

        base = os.path.basename(path)
        if base not in ('buglist.cgi', 'query.cgi'):
            return {}

        for (k, v) in parse_qsl(query):
            if k not in q:
                q[k] = v
            elif isinstance(q[k], list):
                q[k].append(v)
            else:
                oldv = q[k]
                q[k] = [oldv, v]

        # Handle saved searches
        if base == "buglist.cgi" and "namedcmd" in q and "sharer_id" in q:
            q = {
                "sharer_id": q["sharer_id"],
                "savedsearch": q["namedcmd"],
            }

        return q
1014749
def _get(self, url):
        
        return self.session.get(self.READ_URL_PRE + url).json()
1169863
def parse(self, url):
        
        parsed_url = urlparse.urlparse(url)
        try:
            default_config = self.CONFIG[parsed_url.scheme]
        except KeyError:
            raise ValueError(
                'unrecognised URL scheme for {}: {}'.format(
                    self.__class__.__name__, url))
        handler = self.get_handler_for_scheme(parsed_url.scheme)
        config = copy.deepcopy(default_config)
        return handler(parsed_url, config)
1245591
def get(self, url=None, params=None, retry=True):
        '''
        
        '''
        headers = self._gen_headers(self.access_token, url)
        attempts = 1
        while attempts <= HTTP_ATTEMPTS_MAX:
            try:
                res = requests.get(url,
                                   headers=headers,
                                   params=params,
                                   timeout=15,
                                   verify=self.certs)
                res.raise_for_status()
                return res.json()
            except requests.exceptions.RequestException as e:
                attempts += 1
                if res.status_code in [400]:
                    raise e
                elif retry and res.status_code in [403]:
                    self.relogin_oauth2()
1145210
def _request(self, url, values=None):
        
        if values:
            json_data = json.dumps(values)
            request = urllib2.Request(url, json_data.encode())
            request.add_header('content-type', 'application/json')
            request.add_header('authorization', self._auth)
            connection = urllib2.urlopen(request)
            response = connection.read()
            connection.close()
        else:
            request = urllib2.Request(url)
            request.add_header('authorization', self._auth)
            connection = urllib2.urlopen(request)
            response = connection.read()
            connection.close()

        try:
            result = json.loads(response.decode())
        except ValueError as exception:
            return {'errorCode': 1, 'message': str(exception)}

        return result
652653
def object_url(self, object_t, object_id=None, relation=None, **kwargs):
        
        if object_t not in self.objects_types:
            raise TypeError("{} is not a valid type".format(object_t))
        request_items = (
            str(item) for item in [object_t, object_id, relation] if item is not None
        )
        request = "/".join(request_items)
        base_url = self.url(request)
        if self.access_token is not None:
            kwargs["access_token"] = str(self.access_token)
        if kwargs:
            for key, value in kwargs.items():
                if not isinstance(value, str):
                    kwargs[key] = str(value)
            # kwargs are sorted (for consistent tests between Python < 3.7 and >= 3.7)
            sorted_kwargs = SortedDict.from_dict(kwargs)
            result = "{}?{}".format(base_url, urlencode(sorted_kwargs))
        else:
            result = base_url
        return result
109005
async def _collect_sample(self, url, url_pattern):
        
        samples = []

        urls = [self.path_generator.generate_url(url, url_pattern) for _ in range(self.confirmation_factor)]

        iterator = asyncio.as_completed([self._fetch_sample(url) for url in urls])
        for promise in iterator:
            try:
                sig = await promise
                if sig:
                    samples.append(sig)
            except RejectRequest as e:
                pass

        if not samples:
            raise StopRequest("Impossible to obtain sample")
        else:
            return samples
62512
def get(self, url):
        
        logger.debug('Making GET request to %s', url)
        return self.oauth_session.get(url)
503528
def get_base_url(html: str) -> str:
    
    forms = BeautifulSoup(html, 'html.parser').find_all('form')
    if not forms:
        raise VVKBaseUrlException('Form for login not found')
    elif len(forms) > 1:
        raise VVKBaseUrlException('More than one login form found')
    login_url = forms[0].get('action')
    if not login_url:
        raise VVKBaseUrlException('No action tag in form')
    return login_url
916724
def get_auth(self):
        
        if self.dynamic:
            return self.dynamic
        if self.settings_dict['USER'] == 'dynamic auth':
            return {'instance_url': self.settings_dict['HOST']}
        # If another thread is running inside this method, wait for it to
        # finish. Always release the lock no matter what happens in the block
        db_alias = self.db_alias
        with oauth_lock:
            if db_alias not in oauth_data:
                oauth_data[db_alias] = self.authenticate()
            return oauth_data[db_alias]
183861
def search_auth(self, list_or_file_path, source):
        
        _auth = source.args['auth']
        if isinstance(_auth, str):
            if ':' in _auth:
                _auth = _auth.split(':')
            elif _auth.endswith('}') and (
                    _auth.startswith('{') or ':' in _auth):  # {auth}, {user}:{password}, user:{password}
                _auth = self.get_auth(list_or_file_path, _auth)
                _auth = self.split_auth(_auth)

        if isinstance(_auth, list):
            for i in range(len(_auth)):
                if _auth[i].endswith('}') and (
                        _auth[i].startswith('{') or ':' in _auth[i]):  # {auth}, {user}:{password}, user:{password}
                    _auth[i] = self.get_auth(list_or_file_path, _auth[i])
                    if ':' in _auth[i]:
                        _auth = self.split_auth(_auth[i])

        source.args['auth'] = _auth