%prog links url

    Extract all the links "<a href=''>" from web page.
977519
def links(args):
    
    p = OptionParser(links.__doc__)
    p.add_option("--img", default=False, action="store_true",
                 help="Extract <img> tags [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    url, = args
    img = opts.img

    htmlfile = download(url)
    page = open(htmlfile).read()
    soup = BeautifulSoup(page)

    tag = 'img' if img else 'a'
    src = 'src' if img else 'href'
    aa = soup.findAll(tag)
    for a in aa:
        link = a.get(src)
        link = urljoin(url, link)
        print(link)
249456
def _get_link_pages(page, per_page, count, page_url):
    # type: (int, int, int, str) -> Dict[str, str]
    
    current_page = _page_arg(page)
    links = {}
    end = page * per_page

    if page > 1:
        links['prev'] = page_url.replace(current_page, _page_arg(page - 1))

    if end < count:
        links['next'] = page_url.replace(current_page, _page_arg(page + 1))

    if per_page < count:
        links['first'] = page_url.replace(current_page, _page_arg(1))
        links['last'] = page_url.replace(current_page, _page_arg((count + per_page - 1) // per_page))

    return links
1308009
def cp(args):
    
    p = OptionParser(cp.__doc__)
    fp = sys.stdin

    for link_name in fp:
        link_name = link_name.strip()
        if not op.exists(link_name):
            continue

        source = get_abs_path(link_name)
        link_name = op.basename(link_name)
        if not op.exists(link_name):
            os.symlink(source, link_name)
        logging.debug(" => ".join((source, link_name)))
301623
def get_links(self, recall, timeout):
        
        for _ in range(recall):
            try:  # setting timeout
                soup = BeautifulSoup(self.source)  # parse source
                out_links = []

                for tag in soup.findAll(["a", "link"], href=True):
                    tag["href"] = urljoin(self.url, tag["href"])
                    out_links.append(tag["href"])

                return sorted(out_links)  # sort array
            except:
                time.sleep(timeout)
1250965
def get_link_pages(links):
        
        link_pages = []
        i = 0
        while i < len(links):
            link_page = []
            while i < len(links) and len(link_page) < 10:
                link_page.append(links[i])
                i += 1
            link_pages.append(link_page)
        return link_pages
635194
def urls_linking_to(urls)
      unless urls.is_a?(Array)
        urls = [urls] unless urls.is_a?(Array)
        single = true
      end

      links = pages_linking_to(urls)
      links.each { |url, pages| links[url] = pages.map{|p| p.url} }

      if single and !links.empty?
        return links[urls.first]
      else
        return links
      end
    end
1020143
def rel_links(cls, page):
    """"""
    for match in cls.REL_RE.finditer(page):
      href, rel = match.group(0), match.group(1)
      if rel not in cls.REL_TYPES:
        continue
      href_match = cls.HREF_RE.search(href)
      if href_match:
        href = cls.href_match_to_url(href_match)
        parsed_href = urlparse(href)
        if any(parsed_href.path.endswith(ext) for ext in cls.REL_SKIP_EXTENSIONS):
          continue
        yield href
359740
def open_listing_page(trailing_part_of_url):
    
    base_url = 'http://www.bbc.co.uk/programmes/'
    print("Opening web page: " + base_url + trailing_part_of_url)

    try:
        html = requests.get(base_url + trailing_part_of_url).text
    except (IOError, NameError):
        print("Error opening web page.")
        print("Check network connection and/or programme id.")
        sys.exit(1)

    try:
        return lxml.html.fromstring(html)
    except lxml.etree.ParserError:
        print("Error trying to parse web page.")
        print("Maybe there's no programme listing?")
        sys.exit(1)
295772
def extract_variables(href):
    """"""

    patterns = [re.sub(r'\*|:\d+', '', pattern)
                for pattern in re.findall(r'{[\+#\./;\?&]?([^}]+)*}', href)]
    variables = []
    for pattern in patterns:
        for part in pattern.split(","):
            if not part in variables:
                variables.append(part)

    return variables
72540
def _html_find_urls(bytes, mimetype, base_url=None):
    

    def _recursive_tag_values(tag, values=[]):
        """ This sub-function recursively loops through all of the tags to get all of the attribute values. """

        if hasattr(tag, 'children'):
            for child in tag.children:
                if hasattr(child, 'attrs'):
                    for key in child.attrs:
                        if isinstance(child.attrs[key], list):
                            for value in child.attrs[key]:
                                values.append(value)
                        elif isinstance(child.attrs[key], str):
                            values.append(child.attrs[key])

                    values = _recursive_tag_values(child, values)

        return values

    # Only convert the ASCII bytes to HTML.
    ascii_bytes = b''.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
    ascii_bytes = ascii_bytes.replace(b'\x00', b'')

    # Store all of the URLs we find.
    urls = []

    # Convert the bytes into soup. Also try url decoding the bytes to bypass some obfuscation.
    soups = []
    soups.append(BeautifulSoup(ascii_bytes, 'lxml'))
    try:
        soups.append(BeautifulSoup(urllib.parse.unquote(str(ascii_bytes)), 'lxml'))
    except:
        pass

    # Loop over both soups.
    for soup in soups:

        # Find any URLs embedded in script + document.write JavaScript.
        # \r\n\r\ndocument.write(unescape('<meta HTTP-EQUIV="REFRESH" content="0; url=http://somebadsite.com/catalog/index.php">'));\r\n\r\n
        script_urls = []
        script_tags = soup.find_all('script')
        for script_tag in script_tags:
            for tag_content in script_tag.contents:
                if 'document.write' in tag_content.lower():
                    # Find the last position of the ( character, which should denote where the
                    # code that they are writing to the page begins. Also find the position of
                    # the first ) character which should denote where the code ends.
                    code_begin = tag_content.rfind('(')
                    code_end = tag_content.find(')')
                    code = tag_content[code_begin+1:code_end]
                    # Strip off any ' or " quotes.
                    if code.startswith('\'') and code.endswith('\''):
                        code = code[1:-1]
                    if code.startswith('"') and code.endswith('"'):
                        code = code[1:-1]
                    # Turn the string into bytes and feed it back into the _html_find_urls function.
                    code = code.encode()
                    script_urls += _html_find_urls(code, mimetype)

        # Find any URLs embedded in <script> tags. Use the _ascii_find_urls function to try and catch everything.
        for script_tag in script_tags:
            script_urls += _ascii_find_urls(str(script_tag).encode('utf-8'), mimetype)

        # Find any meta-refresh URLs.
        # <meta HTTP-Equiv="refresh" content="0; URL=UntitledNotebook1.html">
        meta_urls = []
        meta_tags = soup.find_all('meta')
        for meta_tag in meta_tags:
            for key in meta_tag.attrs:
                if key.lower() == 'content':
                    value = meta_tag.attrs[key]
                    if 'url=' in value.lower():
                        split_value = value.split('=', 1)
                        url = split_value[1]
                        # Remove any quotes around the URL.
                        if url.startswith('"') and url.endswith('"'):
                            url = url[1:-1]
                        if url.startswith("'") and url.endswith("'"):
                            url = url[1:-1]
                        meta_urls.append(url)

        # Hacky way to find URLs in the CSS.
        css_urls = re.compile(r'url\((.*?)\)').findall(str(soup))
        
        # Make sure none of the CSS URLs are in quotes.
        for u in css_urls[:]:
            if u.startswith('\'') and u.endswith('\''):
                css_urls.remove(u)
                css_urls.append(u[1:-1])
            if u.startswith('\"') and u.endswith('\"'):
                css_urls.remove(u)
                css_urls.append(u[1:-1])

        # Look to see if there is a "base" HTML tag specified. This is the "baseStriker" method.
        for tag in soup.find_all('base'):
            try:
                if tag['href']:
                    base_url = tag['href'].replace('\\\\', '//')
            except:
                pass

        # If we were given a base URL, only extract specific tag values that are likely
        # to be URLs. Otherwise, we would end up with joined URLs for every single tag
        # attribute value that exists in the HTML, which is not the correct behavior.
        if base_url:

            # Join any of the CSS URLs we found.
            for css_url in css_urls:
                urls.append(urljoin(base_url, css_url))

            # Join any of the script URLs we found.
            for script_url in script_urls:
                urls.append(urljoin(base_url, script_url))

            # Join any of the meta-refresh URLs we found.
            for meta_url in meta_urls:
                urls.append(urljoin(base_url, meta_url))

            # Get all of the action URLs.
            for tag in soup.find_all(action=True):
                urls.append(urljoin(base_url, tag['action']))

            # Get all of the background URLs.
            for tag in soup.find_all(background=True):
                urls.append(urljoin(base_url, tag['background']))

            # Get all of the href URLs.
            for tag in soup.find_all(href=True):
                urls.append(urljoin(base_url, tag['href']))

            # Get all of the src URLs.
            for tag in soup.find_all(src=True):
                urls.append(urljoin(base_url, tag['src']))

            # Get all of the xmlns URLs.
            for tag in soup.find_all(xmlns=True):
                urls.append(urljoin(base_url, tag['xmlns']))

        # We weren't given a base URL, so just search for URLs by getting every single
        # tag attribute value. That way we will catch everything regardless of the attribute name.
        else:
            urls = _recursive_tag_values(soup)
            urls += css_urls
            urls += meta_urls
            urls += script_urls

        # As a last-ditch effort, find URLs in the visible text of the HTML. However,
        # we only want to add strings that are valid URLs as they are. What we do not
        # want is to add every string as a potential URL, since if a base_url was given,
        # we will likely end up with a joined URL for every string found in the HTML.
        for s in soup.stripped_strings:
            if is_valid(s):
                urls.append(s)

        # Remove any newlines from the potential URLs. Some HTML has newlines
        # after the href=" part before the actual URL begins. This renders
        # correctly in web browsers but is otherwise considered an invalid
        # URL by the is_valid function.
        urls = [u.strip() for u in urls]

        # Remove any leading //'s from the URLs.
        urls = [u[2:] if u.startswith('//') else u for u in urls]

    return urls
1014892
def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """"""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result
1015942
def _link_to(self, page, text=None, title=None, _class="", id="", alt="", **kwargs):
        
        anchor = ""
        if "#" in page:
            page, anchor = page.split("#")
            anchor = "#" + anchor
        meta = self._get_page_meta(page)
        return "<a href='{url}' class='{_class}' id='{id}'  title=\"{title}\">{text}</a>".format(
            url=meta.get("url", "/") + anchor,
            text=text or meta.get("title") or title,
            title=title or "",
            _class=_class,
            id=id
        )
824859
def get_new_links(self, url, resp):
        """"""
        links_on_page = resp.xpath('//a/@href')
        links = [utils.clean_url(u, url) for u in links_on_page]

        # Remove non-links through filtering by protocol
        links = [x for x in links if utils.check_protocol(x)]

        # Restrict new URLs by the domain of the input URL
        if not self.args['nonstrict']:
            domain = utils.get_domain(url)
            links = [x for x in links if utils.get_domain(x) == domain]

        # Filter URLs by regex keywords, if any
        if self.args['crawl']:
            links = utils.re_filter(links, self.args['crawl'])
        return links
731728
def absolute_urls(html):
    

    from bs4 import BeautifulSoup
    from yacms.core.request import current_request

    request = current_request()
    if request is not None:
        dom = BeautifulSoup(html, "html.parser")
        for tag, attr in ABSOLUTE_URL_TAGS.items():
            for node in dom.findAll(tag):
                url = node.get(attr, "")
                if url:
                    node[attr] = request.build_absolute_uri(url)
        html = str(dom)
    return html
1320890
def href(*args, **kw):
    
    result = [(request.script_root if request else "") + "/"]
    for idx, arg in enumerate(args):
        result.append(("/" if idx else "") + url_quote(arg))
    if kw:
        result.append("?" + url_encode(kw))
    return "".join(result)
642345
def parse_href(href):
    """"""
    url = urlparse(href)
    path = url.path.split('/')
    collection_name = path[1]
    id_ = path[2]
    return collection_name, id_
364072
def links(cls, page):
    """"""
    for match in cls.HREF_RE.finditer(page):
      yield cls.href_match_to_url(match)