Extracting css links from BeautifulSoup object

    :param bs4: `BeautifulSoup`
    :return: `list` List of links
187257
def extract_css_links(bs4):
    

    links = extract_links(bs4)

    real_css = [anchor for anchor in links if anchor.endswith(('.css', '.CSS'))]

    css_link_tags = [anchor['href'] for anchor in bs4.select('link[type="text/css"]')
                     if anchor.has_attr('href')]

    return list(set(real_css+css_link_tags))
934542
def nb_to_html_cells(nb) -> list:
    
    html_exporter = HTMLExporter()
    html_exporter.template_file = 'basic'

    (body, resources) = html_exporter.from_notebook_node(nb)
    return BeautifulSoup(body, 'html.parser').findAll('div', class_='cell')
163214
def extract_table(html_string, row_count, column_count):
    
    try:
        from bs4 import BeautifulSoup
        from bs4.element import Tag
    except ImportError:
        print("ERROR: You must have BeautifulSoup to use html2data")
        return

    #html_string = convertRichText(html_string)

    data_table = []
    for row in range(0, row_count):
        data_table.append([])
        for column in range(0, column_count):
            data_table[-1].append(None)

    soup = BeautifulSoup(html_string, 'html.parser')

    table = soup.find('table')

    if not table:
        return ''

    trs = table.findAll('tr')
    if len(trs) == 0:
        return [['']]

    for tr in range(len(trs)):
        ths = trs[tr].findAll('th')
        if len(ths) == 0:
            tds = trs[tr].findAll('td')
        else:
            tds = ths

        if len(tds) == 0:
            tds = []
            for i in range(0, column_count):
                tds.append(Tag("", name=""))

        for i in range(len(tds)):
            td = tds[i]
            row, column = find_unassigned_table_cell(data_table)

            r_span_count = 1
            c_span_count = 1

            if td.has_attr('rowspan'):
                r_span_count = int(td['rowspan'])
            if td.has_attr('colspan'):
                c_span_count = int(td['colspan'])

            for row_prime in range(row, row + r_span_count):
                for column_prime in range(column, column + c_span_count):
                    if row_prime == row and column_prime == column:

                        items = []
                        for item in td.contents:
                            items.append(str(item))
                        string = ''.join(items).strip()

                        text = restructify(string).rstrip()

                        data_table[row_prime][column_prime] = text
                    else:
                        data_table[row_prime][column_prime] = ""

            if i + 1 < column_count and i == len(tds) - 1:
                for x in range(len(tds), column_count):
                    if data_table[row][x] is None:
                        data_table[row][x] = ""
    for row in range(len(data_table)):
        for column in range(len(data_table[row])):
            if not data_table[row][column]:
                data_table[row][column] = ""

    return data_table
379783
def findEndpoint(html):
    
    poss_rels = ['webmention', 'http://webmention.org', 'http://webmention.org/', 'https://webmention.org', 'https://webmention.org/']

    # find elements with correct rels and a href value
    all_links = BeautifulSoup(html, _html_parser).find_all(rel=poss_rels, href=True)
    for link in all_links:
        s = link.get('href', None)
        if s is not None:
            return s

    return None
731728
def absolute_urls(html):
    

    from bs4 import BeautifulSoup
    from yacms.core.request import current_request

    request = current_request()
    if request is not None:
        dom = BeautifulSoup(html, "html.parser")
        for tag, attr in ABSOLUTE_URL_TAGS.items():
            for node in dom.findAll(tag):
                url = node.get(attr, "")
                if url:
                    node[attr] = request.build_absolute_uri(url)
        html = str(dom)
    return html
718601
def extract_js_links(bs4):
    

    links = extract_links(bs4)

    real_js = [anchor for anchor in links if anchor.endswith(('.js', '.JS'))]

    js_tags = [anchor['src'] for anchor in bs4.select('script[type="text/javascript"]')
               if anchor.has_attr('src')]

    return list(set(real_js+js_tags))
301623
def get_links(self, recall, timeout):
        
        for _ in range(recall):
            try:  # setting timeout
                soup = BeautifulSoup(self.source)  # parse source
                out_links = []

                for tag in soup.findAll(["a", "link"], href=True):
                    tag["href"] = urljoin(self.url, tag["href"])
                    out_links.append(tag["href"])

                return sorted(out_links)  # sort array
            except:
                time.sleep(timeout)
1299698
def extract_html_urls(self, html):
        
        p = HTMLParser(tree=treebuilders.getTreeBuilder("dom"))
        dom = p.parse(html)
        urls = []

        for img in dom.getElementsByTagName('img'):
            src = img.getAttribute('src')
            if src:
                urls.append(unquote_utf8(src))

            srcset = img.getAttribute('srcset')
            if srcset:
                urls += self.extract_srcset(srcset)

        for source in dom.getElementsByTagName('source'):
            srcset = source.getAttribute('srcset')
            if srcset:
                urls += self.extract_srcset(srcset)

        for source in dom.getElementsByTagName('a'):
            href = source.getAttribute('href')
            if href:
                urls.append(unquote_utf8(href))

        return urls
154891
def links(self):
        
        links = Links()
        links["self"] = Link.for_(
            self._operation,
            self._ns,
            qs=self._page.to_items(),
            **self._context
        )
        return links
330138
def _get_links(self,):
        
        res = ''
        if self.links:
            for l in self.links:
                res += ' links = ' + str(l[0]) + '\n'
                if l[0].child_nodes:
                    for chld in l[0].child_nodes:
                        res += '   child = ' + str(chld) + '\n'
                if l[0].links:
                    for lnk in l[0].links:
                        res += '   sublink = ' + str(lnk[0]) + '\n'
        else:
            res += ' links = None\n'
            
        return res
190054
def backlinks(self):
        
        if self._backlinks is None:
            self._backlinks = list()
            params = {
                "action": "query",
                "list": "backlinks",
                "bltitle": self.title,
                "bllimit": "max",
                "blfilterredir": "nonredirects",
                "blnamespace": 0,
            }
            tmp = [link["title"] for link in self._continued_query(params, "backlinks")]
            self._backlinks = sorted(tmp)
        return self._backlinks
1250965
def get_link_pages(links):
        
        link_pages = []
        i = 0
        while i < len(links):
            link_page = []
            while i < len(links) and len(link_page) < 10:
                link_page.append(links[i])
                i += 1
            link_pages.append(link_page)
        return link_pages
1217229
def links(self):
        
        if self._links is None:
            self._links = list()
            self.__pull_combined_properties()
        return self._links
849592
def _bake_css(link):
    
    if "href" in link.attrs and (re.search("\.css$", link["href"])) or ("rel" in link.attrs and link["rel"] is "stylesheet") or ("type" in link.attrs and link["type"] is "text/css"):
        if re.match("https?://", link["href"]):
            css_data = _load_url(link["href"]).read()
        else:
            css_data = _load_file(link["href"]).read()
        link.clear()
        if USING_PYTHON2:
            link.string = css_data
        else:
            link.string = str(css_data)
        link.name = "style"
        del link["rel"]
        del link["href"]
1020143
def rel_links(cls, page):
    """"""
    for match in cls.REL_RE.finditer(page):
      href, rel = match.group(0), match.group(1)
      if rel not in cls.REL_TYPES:
        continue
      href_match = cls.HREF_RE.search(href)
      if href_match:
        href = cls.href_match_to_url(href_match)
        parsed_href = urlparse(href)
        if any(parsed_href.path.endswith(ext) for ext in cls.REL_SKIP_EXTENSIONS):
          continue
        yield href
72540
def _html_find_urls(bytes, mimetype, base_url=None):
    

    def _recursive_tag_values(tag, values=[]):
        """ This sub-function recursively loops through all of the tags to get all of the attribute values. """

        if hasattr(tag, 'children'):
            for child in tag.children:
                if hasattr(child, 'attrs'):
                    for key in child.attrs:
                        if isinstance(child.attrs[key], list):
                            for value in child.attrs[key]:
                                values.append(value)
                        elif isinstance(child.attrs[key], str):
                            values.append(child.attrs[key])

                    values = _recursive_tag_values(child, values)

        return values

    # Only convert the ASCII bytes to HTML.
    ascii_bytes = b''.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
    ascii_bytes = ascii_bytes.replace(b'\x00', b'')

    # Store all of the URLs we find.
    urls = []

    # Convert the bytes into soup. Also try url decoding the bytes to bypass some obfuscation.
    soups = []
    soups.append(BeautifulSoup(ascii_bytes, 'lxml'))
    try:
        soups.append(BeautifulSoup(urllib.parse.unquote(str(ascii_bytes)), 'lxml'))
    except:
        pass

    # Loop over both soups.
    for soup in soups:

        # Find any URLs embedded in script + document.write JavaScript.
        # \r\n\r\ndocument.write(unescape('<meta HTTP-EQUIV="REFRESH" content="0; url=http://somebadsite.com/catalog/index.php">'));\r\n\r\n
        script_urls = []
        script_tags = soup.find_all('script')
        for script_tag in script_tags:
            for tag_content in script_tag.contents:
                if 'document.write' in tag_content.lower():
                    # Find the last position of the ( character, which should denote where the
                    # code that they are writing to the page begins. Also find the position of
                    # the first ) character which should denote where the code ends.
                    code_begin = tag_content.rfind('(')
                    code_end = tag_content.find(')')
                    code = tag_content[code_begin+1:code_end]
                    # Strip off any ' or " quotes.
                    if code.startswith('\'') and code.endswith('\''):
                        code = code[1:-1]
                    if code.startswith('"') and code.endswith('"'):
                        code = code[1:-1]
                    # Turn the string into bytes and feed it back into the _html_find_urls function.
                    code = code.encode()
                    script_urls += _html_find_urls(code, mimetype)

        # Find any URLs embedded in <script> tags. Use the _ascii_find_urls function to try and catch everything.
        for script_tag in script_tags:
            script_urls += _ascii_find_urls(str(script_tag).encode('utf-8'), mimetype)

        # Find any meta-refresh URLs.
        # <meta HTTP-Equiv="refresh" content="0; URL=UntitledNotebook1.html">
        meta_urls = []
        meta_tags = soup.find_all('meta')
        for meta_tag in meta_tags:
            for key in meta_tag.attrs:
                if key.lower() == 'content':
                    value = meta_tag.attrs[key]
                    if 'url=' in value.lower():
                        split_value = value.split('=', 1)
                        url = split_value[1]
                        # Remove any quotes around the URL.
                        if url.startswith('"') and url.endswith('"'):
                            url = url[1:-1]
                        if url.startswith("'") and url.endswith("'"):
                            url = url[1:-1]
                        meta_urls.append(url)

        # Hacky way to find URLs in the CSS.
        css_urls = re.compile(r'url\((.*?)\)').findall(str(soup))
        
        # Make sure none of the CSS URLs are in quotes.
        for u in css_urls[:]:
            if u.startswith('\'') and u.endswith('\''):
                css_urls.remove(u)
                css_urls.append(u[1:-1])
            if u.startswith('\"') and u.endswith('\"'):
                css_urls.remove(u)
                css_urls.append(u[1:-1])

        # Look to see if there is a "base" HTML tag specified. This is the "baseStriker" method.
        for tag in soup.find_all('base'):
            try:
                if tag['href']:
                    base_url = tag['href'].replace('\\\\', '//')
            except:
                pass

        # If we were given a base URL, only extract specific tag values that are likely
        # to be URLs. Otherwise, we would end up with joined URLs for every single tag
        # attribute value that exists in the HTML, which is not the correct behavior.
        if base_url:

            # Join any of the CSS URLs we found.
            for css_url in css_urls:
                urls.append(urljoin(base_url, css_url))

            # Join any of the script URLs we found.
            for script_url in script_urls:
                urls.append(urljoin(base_url, script_url))

            # Join any of the meta-refresh URLs we found.
            for meta_url in meta_urls:
                urls.append(urljoin(base_url, meta_url))

            # Get all of the action URLs.
            for tag in soup.find_all(action=True):
                urls.append(urljoin(base_url, tag['action']))

            # Get all of the background URLs.
            for tag in soup.find_all(background=True):
                urls.append(urljoin(base_url, tag['background']))

            # Get all of the href URLs.
            for tag in soup.find_all(href=True):
                urls.append(urljoin(base_url, tag['href']))

            # Get all of the src URLs.
            for tag in soup.find_all(src=True):
                urls.append(urljoin(base_url, tag['src']))

            # Get all of the xmlns URLs.
            for tag in soup.find_all(xmlns=True):
                urls.append(urljoin(base_url, tag['xmlns']))

        # We weren't given a base URL, so just search for URLs by getting every single
        # tag attribute value. That way we will catch everything regardless of the attribute name.
        else:
            urls = _recursive_tag_values(soup)
            urls += css_urls
            urls += meta_urls
            urls += script_urls

        # As a last-ditch effort, find URLs in the visible text of the HTML. However,
        # we only want to add strings that are valid URLs as they are. What we do not
        # want is to add every string as a potential URL, since if a base_url was given,
        # we will likely end up with a joined URL for every string found in the HTML.
        for s in soup.stripped_strings:
            if is_valid(s):
                urls.append(s)

        # Remove any newlines from the potential URLs. Some HTML has newlines
        # after the href=" part before the actual URL begins. This renders
        # correctly in web browsers but is otherwise considered an invalid
        # URL by the is_valid function.
        urls = [u.strip() for u in urls]

        # Remove any leading //'s from the URLs.
        urls = [u[2:] if u.startswith('//') else u for u in urls]

    return urls
364072
def links(cls, page):
    """"""
    for match in cls.HREF_RE.finditer(page):
      yield cls.href_match_to_url(match)