Process document for links.
548336
def scrape_document(self, item_session: ItemSession):
        ''''''
        self.event_dispatcher.notify(
            PluginFunctions.get_urls, item_session
        )

        if not self._document_scraper:
            return

        demux_info = self._document_scraper.scrape_info(
            item_session.request, item_session.response,
            item_session.url_record.link_type
        )

        num_inline_urls = 0
        num_linked_urls = 0

        for scraper, scrape_result in demux_info.items():
            new_inline, new_linked = self._process_scrape_info(
                scraper, scrape_result, item_session
            )
            num_inline_urls += new_inline
            num_linked_urls += new_linked

        _logger.debug('Candidate URLs: inline={0} linked={1}',
                      num_inline_urls, num_linked_urls
        )
806105
def convert(self):
        """"""
        lines = []

        pages = self.flatten_pages(self.config['pages'])

        f_exclude = mkdocs_pandoc.filters.exclude.ExcludeFilter(
                exclude=self.exclude)

        f_include = mkdocs_pandoc.filters.include.IncludeFilter(
                base_path=self.config['docs_dir'],
                encoding=self.encoding)

        # First, do the processing that must be done on a per-file basis:
        # Adjust header levels, insert chapter headings and adjust image paths.

        f_headlevel = mkdocs_pandoc.filters.headlevels.HeadlevelFilter(pages)

        for page in pages:
            fname = os.path.join(self.config['docs_dir'], page['file'])
            try:
                p = codecs.open(fname, 'r', self.encoding)
            except IOError as e:
                raise FatalError("Couldn't open %s for reading: %s" % (fname,
                    e.strerror), 1)
            f_chapterhead = mkdocs_pandoc.filters.chapterhead.ChapterheadFilter(
                    headlevel=page['level'],
                    title=page['title']
                    )

            f_image = mkdocs_pandoc.filters.images.ImageFilter(
                    filename=page['file'],
                    image_path=self.config['site_dir'],
                    image_ext=self.image_ext)

            lines_tmp = []

            for line in p.readlines():
                lines_tmp.append(line.rstrip())

            if self.exclude:
                lines_tmp = f_exclude.run(lines_tmp)

            if self.filter_include:
                lines_tmp = f_include.run(lines_tmp)

            lines_tmp = f_headlevel.run(lines_tmp)
            lines_tmp = f_chapterhead.run(lines_tmp)
            lines_tmp = f_image.run(lines_tmp)
            lines.extend(lines_tmp)
            # Add an empty line between pages to prevent text from a previous
            # file from butting up against headers in a subsequent file.
            lines.append('')

        # Strip anchor tags
        if self.strip_anchors:
            lines = mkdocs_pandoc.filters.anchors.AnchorFilter().run(lines)

        # Fix cross references
        if self.filter_xrefs:
            lines = mkdocs_pandoc.filters.xref.XrefFilter().run(lines)

        if self.filter_toc:
            lines = mkdocs_pandoc.filters.toc.TocFilter().run(lines)

        if self.filter_tables:
            lines = mkdocs_pandoc.filters.tables.TableFilter().run(lines)

        return(lines)
732366
def process_documents(docs)
      return if @active_lang == @default_lang
      url = config.fetch('url', false)
      rel_regex = relative_url_regex
      abs_regex = absolute_url_regex(url)
      docs.each do |doc|
        relativize_urls(doc, rel_regex)
        if url
        then relativize_absolute_urls(doc, abs_regex, url)
        end
      end
    end
864174
def process_document(self, doc, std):
        """"""
        if doc.url not in self.blacklist:
            print("Indexing '%s' %s..." % (doc.url, std))
            name = self.extract_name(doc.text)
            self.results.add((name, doc.url, std))
        else:
            print("Skipping blacklisted page '%s' ..." % doc.url)
            return None
1144546
def links(self):
        """"""
        # <https://example.com/?page=2>; rel="next", <https://example.com/?page=34>; rel="last"'
        # becomes
        # {
        #     'last': {'rel': 'last', 'url': 'https://example.com/?page=34'},
        #     'next': {'rel': 'next', 'url': 'https://example.com/?page=2'},
        # },
        result = {}
        if 'Link' in self.headers:
            value = self.headers['Link']

            for part in re.split(r', *<', value):
                link = {}
                vs = part.split(';')

                # First section is always an url.
                link['url'] = vs.pop(0).strip('\'" <>')

                for v in vs:
                    if '=' in v:
                        key, v = v.split('=')
                        link[key.strip('\'" ')] = v.strip('\'" ')

                rkey = link.get('rel') or link['url']
                result[rkey] = link

        return result
824859
def get_new_links(self, url, resp):
        """"""
        links_on_page = resp.xpath('//a/@href')
        links = [utils.clean_url(u, url) for u in links_on_page]

        # Remove non-links through filtering by protocol
        links = [x for x in links if utils.check_protocol(x)]

        # Restrict new URLs by the domain of the input URL
        if not self.args['nonstrict']:
            domain = utils.get_domain(url)
            links = [x for x in links if utils.get_domain(x) == domain]

        # Filter URLs by regex keywords, if any
        if self.args['crawl']:
            links = utils.re_filter(links, self.args['crawl'])
        return links
1232927
function _getLinks (window, links) {
  var $ = window.$;

  $('a.lede__link').each(function(){
    var text = $(this).text().trim();
    if (!text) return;
    links.push(text);
  });
}
647306
async def getlinks(url):
    """"""
    page = Linkfetcher(url)
    await page.linkfetch()
    for i, url in enumerate(page):
        return (i, url)
38928
def process_children(self, node, elem, module, path, omit=[]):
        """"""
        for ch in node.i_children:
            if ch not in omit and (ch.i_config or self.doctype == "data"):
                self.node_handler.get(ch.keyword, self.ignore)(
                    ch, elem, module, path)
109982
def links(self):
        """"""
        for match in self._href_re.finditer(self.content):
            url = match.group(1) or match.group(2) or match.group(3)
            url = self.clean_link(urlparse.urljoin(self.base_url, url))
            yield Link(url, self)
285323
protected void processLinkTag(LinkTag tag) {

        processLink(tag, ATTRIBUTE_HREF, CmsRelationType.valueOf(tag.getTagName()));
    }
201335
private void processFiles() {
		StringBuffer buf = new StringBuffer();
		appendFileDetails(buf, allFiles);
		appendFileDetails(buf, textFiles);
		appendFileDetails(buf, pdfFiles);
		console.setText(buf.toString());
	}
220811
def parse_link_results(doc)
    uris = doc.xpath("/ds:links/ds:uri", @ds_namespaces)
    results = []
    uris.each do |uri_el|
      link = uri_el.content
      results << URI.parse(link)
    end
    results
  end
512068
def _ProcessFileSource(self, source):
    """"""

    if source.path_type != rdf_paths.PathSpec.PathType.OS:
      raise ValueError("Only supported path type is OS.")

    paths = artifact_utils.InterpolateListKbAttributes(
        source.base_source.attributes["paths"], self.knowledge_base,
        self.ignore_interpolation_errors)

    file_finder_action = rdf_file_finder.FileFinderAction.Stat()
    request = rdf_file_finder.FileFinderArgs(
        paths=paths, pathtype=source.path_type, action=file_finder_action)
    action = file_finder.FileFinderOSFromClient

    yield action, request
184987
def process_memberdocs(self, docs, codeEl, add=True):
        """"""        
        #Now we need to associate the members with their docstrings
        #Some of the members may be buried inside a group tag and
        #need to be handled separately.
        remainingdocs = []
        expandeddocs = []

        #Process any groups that are in the doc list.
        for doc in docs:
            if isinstance(doc, DocGroup):
                kids = self._process_docgroup(doc, codeEl, add)
                expandeddocs.extend(kids)
            else:
                expandeddocs.append(doc)

        for doc in expandeddocs:
            #Process the docstring, if it doesn't belong to a member
            #we will add it to the list of unassigned docstrings,
            #these most likely point to type declarations.           
            if not self._process_docstrings(doc, codeEl.members, add):
                remainingdocs.append(doc)
        return remainingdocs
551038
def process(source, target, rdfsonly, base=None, logger=logging):
    '''
    

    '''
    for link in source.match():
        s, p, o = link[:3]
        #SKip docheader statements
        if s == (base or '') + '@docheader': continue
        if p in RESOURCE_MAPPING: p = RESOURCE_MAPPING[p]
        if o in RESOURCE_MAPPING: o = RESOURCE_MAPPING[o]
        if p == VERSA_BASEIRI + 'refines':
            tlinks = list(source.match(s, TYPE_REL))
            if tlinks:
                if tlinks[0][TARGET] == VERSA_BASEIRI + 'Resource':
                    p = I(RDFS_NAMESPACE + 'subClassOf')
                elif tlinks[0][TARGET] == VERSA_BASEIRI + 'Property':
                    p = I(RDFS_NAMESPACE + 'subPropertyOf')
        if p == VERSA_BASEIRI + 'properties':
            suri = I(iri.absolutize(s, base)) if base else s
            target.add((URIRef(o), URIRef(RDFS_NAMESPACE + 'domain'), URIRef(suri)))
            continue
        if p == VERSA_BASEIRI + 'value':
            if o not in ['Literal', 'IRI']:
                ouri = I(iri.absolutize(o, base)) if base else o
                target.add((URIRef(s), URIRef(RDFS_NAMESPACE + 'range'), URIRef(ouri)))
                continue
        s = URIRef(s)
        #Translate v:type to rdf:type
        p = RDF.type if p == TYPE_REL else URIRef(p)
        o = URIRef(o) if isinstance(o, I) else Literal(o)
        if not rdfsonly or p.startswith(RDF_NAMESPACE) or p.startswith(RDFS_NAMESPACE):
            target.add((s, p, o))
    return
1252533
protected function process()
    {

        // prepare the product link entity
        if ($attr = $this->prepareAttributes()) {
            // initialize the product link entity
            $productLink = $this->initializeProductLink($attr);

            // persist the product link entity and store the link ID
            $this->setLastLinkId($this->persistProductLink($productLink));
        }
    }