Find new links given a seed URL and follow them breadth-first.

        Save page responses as PART.html files.
        Return the PART.html filenames created during crawling.
181015
def crawl_links(self, seed_url=None):
        
        if seed_url is not None:
            self.seed_url = seed_url

        if self.seed_url is None:
            sys.stderr.write('Crawling requires a seed URL.\n')
            return []

        prev_part_num = utils.get_num_part_files()
        crawled_links = set()
        uncrawled_links = OrderedSet()

        uncrawled_links.add(self.seed_url)
        try:
            while uncrawled_links:
                # Check limit on number of links and pages to crawl
                if self.limit_reached(len(crawled_links)):
                    break
                url = uncrawled_links.pop(last=False)

                # Remove protocol, fragments, etc. to get unique URLs
                unique_url = utils.remove_protocol(utils.clean_url(url))
                if unique_url not in crawled_links:
                    raw_resp = utils.get_raw_resp(url)
                    if raw_resp is None:
                        if not self.args['quiet']:
                            sys.stderr.write('Failed to parse {0}.\n'.format(url))
                        continue

                    resp = lh.fromstring(raw_resp)
                    if self.page_crawled(resp):
                        continue

                    crawled_links.add(unique_url)
                    new_links = self.get_new_links(url, resp)
                    uncrawled_links.update(new_links)
                    if not self.args['quiet']:
                        print('Crawled {0} (#{1}).'.format(url, len(crawled_links)))

                    # Write page response to PART.html file
                    utils.write_part_file(self.args, url, raw_resp, resp, len(crawled_links))
        except (KeyboardInterrupt, EOFError):
            pass

        curr_part_num = utils.get_num_part_files()
        return utils.get_part_filenames(curr_part_num, prev_part_num)
175192
def _find_num_pages(self, url):
    
    def _new_url(i):
      return '{}&page={}#page'.format(url, i)
    i = 0
    j = 2000
    k = (i + j) / 2
    crawler = SeriesCrawler(_new_url(k))
    while i < j:
      ids = crawler.get_ids()
      # If we only find one (we've gone too far)
      if len(ids) == 1:
        # Don't decrement by 1 b/c this could be the last page
        j = k
        k = (i + j) / 2
        crawler.set_url(_new_url(k))
      else:
        i = k + 1
        k = (i + j) / 2
        crawler.set_url(_new_url(k))
    return i
819777
def http_url(full_name, prefix)
      path = full_name.dup

      path.gsub!(/<<\s*(\w*)/, 'from-\1') if path['<<']

      ::File.join(prefix, path.split("::")) + ".html"
    end
274719
def enumerateURL(urlDict, outputFolder, startIndex= 0, maxErrors = 100):
    
    for i, url in enumerate(urlDict.keys()):
        # Grabbing domain name:
        domain = re.findall("://(.*)/", url)[0]

        # Defining the starting index
        index = startIndex

        # The app will stop when this value reaches maxErrors
        consecutiveErrors = 0

        i3Browser = browser.Browser()

        # Main loop that checks if the maximum number of errors has been reached
        while consecutiveErrors <= maxErrors:
            # creating the new URL to download
            newQuery = url.replace("<INDEX>", str(index))
            print(newQuery)
            # Downloading the file
            try:
                data = i3Browser.recoverURL(newQuery)

                filename = domain.replace("/", "|") + "_" + "-profile_" + str(index).rjust(10, "0") +".html"

                if urlDict[url] != None:
                    if urlDict[url] in data:
                        print(general.info("Storing resource as:\t" + filename + "..."))
                        # The profile was found  so we will store it:
                        with open( outputFolder + "/" + filename, "w") as oF:
                            oF.write(data)
                else:
                    # The profile was found  so we will store it:
                    print(general.info("Storing resource as:\t" + filename + "..."))
                    with open( outputFolder + "/" + filename, "w") as oF:
                        oF.write(data)
            except:
                pass
                #logger.error("The resource could not be downloaded.")

            index+=1
626169
def crawl(self):
        """"""
        page = Linkfetcher(self.root)
        page.linkfetch()
        queue = Queue()
        for url in page.urls:
            queue.put(url)
        followed = [self.root]

        n = 0

        while True:
            try:
                url = queue.get()
            except QueueEmpty:
                break

            n += 1

            if url not in followed:
                try:

                    host = urllib.parse.urlparse(url)[1]

                    if self.locked and re.match(".*%s" % self.host, host):
                        followed.append(url)
                        self.followed += 1
                        page = Linkfetcher(url)
                        page.linkfetch()
                        for i, url in enumerate(page):
                            if url not in self.urls:
                                self.links += 1
                                queue.put(url)
                                self.urls.append(url)

                        if n > self.depth and self.depth > 0:
                            break
                except Exception as e:
                    print("ERROR: The URL '%s' can't be crawled (%s)" % (url, e))
                    print(format_exc())
975976
function getLinkContents(err, queuedLink) {
  // no link found
  if (!queuedLink) {
    return;
  }
  // retrieve https directly
  var method = (queuedLink.uri.toString().indexOf('https') === 0) ? utils.retrieveHTTPS : utils.retrieve;

  method(queuedLink.uri, function(err, data) {
    contentLib.indexContentItem({ uri: ''+queuedLink.uri, content: data}, {isHTML: true, member: queuedLink.queued.member}, function(err, res, cItem) {
      if (err || !cItem) {
        GLOBAL.error('getLinkContents failed', err, res);
        return;
      }
      // if it recently gained content queue links
      if (cItem.state !== utils.states.content.queued && cItem.previousState === utils.states.content.queued) {
        GLOBAL.info('queueing links', cItem.uri, 'relevance', cItem.queued.relevance);
        queueLinks(cItem);
      }
      GLOBAL.debug('scraped', err, data ? data.length : 'no content');
    });
  });
}
657409
function crawlLinks(result, $, endCallback) {

    if (! result.links){
        return endCallback();
    }

    log.debug({"url" : result.url, "step" : "analyzeHTML", "message" : "CrawlLinks"});

    async.each($('link'), function(linkTag, callback) {
        crawLink($, result, linkTag, callback);
    }, endCallback);
}
3133
def _extract_next_page_link(self):
        

        # HEADS UP: we do not abort if next_page_link is already set:
        #           we try to find next (eg. find 3 if already at page 2).

        for pattern in self.config.next_page_link:
            items = self.parsed_tree.xpath(pattern)

            if not items:
                continue

            if len(items) == 1:
                item = items[0]

                if 'href' in item.keys():
                    self.next_page_link = item.get('href')

                else:
                    self.next_page_link = item.text.strip()

                LOGGER.info(u'Found next page link: %s.',
                            self.next_page_link)

                # First found link is the good one.
                break

            else:
                LOGGER.warning(u'%s items for next-page link %s',
                               items, pattern,
                               extra={'siteconfig': self.config.host})
499290
function crawl(error, result, callback) {

    // if HTTP error, delegate to the plugins
    if (error) {
        pm.error(error,result, callback);
        return;
    }

    var $ =  html.$(result);

    // Analyse the HTTP response in order to check the content (links, images, ...)
    // or apply a redirect
    async.parallel([
      // Call crawl function of the different plugins
      async.apply(pm.crawl.bind(pm), result, $),

      // Check if redirect
      async.apply(applyRedirect, result),

      // Check if HTML
      async.apply(analyzeHTML, result, $),
    ], function(error) {
        result = null;
        callback(error);
    });

}
1287385
private void crawl(File path) {
        File[] contents = path.listFiles(FileUtil.getFileFilter());
        if (contents != null) {
            Arrays.sort(contents);
            for (File sourceFile : contents) {
                if (sourceFile.isFile()) {
                    StringBuilder sb = new StringBuilder();
                    sb.append("Processing [").append(sourceFile.getPath()).append("]... ");
                    String sha1 = buildHash(sourceFile);
                    String uri = buildURI(sourceFile);
                    boolean process = true;
                    DocumentStatus status = DocumentStatus.NEW;
                    for (String docType : DocumentTypes.getDocumentTypes()) {
                        status = findDocumentStatus(docType, uri, sha1);
                        if (status == DocumentStatus.UPDATED) {
                            sb.append(" : modified ");
                            db.deleteContent(docType, uri);

                        } else if (status == DocumentStatus.IDENTICAL) {
                            sb.append(" : same ");
                            process = false;
                        }
                        if (!process) {
                            break;
                        }
                    }
                    if (DocumentStatus.NEW == status) {
                        sb.append(" : new ");
                    }
                    if (process) { // new or updated
                        crawlSourceFile(sourceFile, sha1, uri);
                    }
                    LOGGER.info("{}", sb);
                }
                if (sourceFile.isDirectory()) {
                    crawl(sourceFile);
                }
            }
        }
    }
977519
def links(args):
    
    p = OptionParser(links.__doc__)
    p.add_option("--img", default=False, action="store_true",
                 help="Extract <img> tags [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    url, = args
    img = opts.img

    htmlfile = download(url)
    page = open(htmlfile).read()
    soup = BeautifulSoup(page)

    tag = 'img' if img else 'a'
    src = 'src' if img else 'href'
    aa = soup.findAll(tag)
    for a in aa:
        link = a.get(src)
        link = urljoin(url, link)
        print(link)
249456
def _get_link_pages(page, per_page, count, page_url):
    # type: (int, int, int, str) -> Dict[str, str]
    
    current_page = _page_arg(page)
    links = {}
    end = page * per_page

    if page > 1:
        links['prev'] = page_url.replace(current_page, _page_arg(page - 1))

    if end < count:
        links['next'] = page_url.replace(current_page, _page_arg(page + 1))

    if per_page < count:
        links['first'] = page_url.replace(current_page, _page_arg(1))
        links['last'] = page_url.replace(current_page, _page_arg((count + per_page - 1) // per_page))

    return links
931756
def find(url, args = {})
      raise ArgumentError.new("url can't be nil!") if url.nil?
      
      harmonized_url = harmonize_url(url)

      raise ArgumentError.new("url's protocol must be 'http(s)' or 'feed' (#{url})") if harmonized_url.nil?

      feeds = []

      # Open the URL to check the content-type or crawl for feed links
      open(harmonized_url) do |file|

        if feed_content_type?(file)
          # Add the url to feeds if it shows a feed content type
          feeds << harmonized_url
        
        else
          # Else, parse the page to search for links
          doc = Nokogiri::HTML(file.read)

          # Check <link> elements
          doc.css('link').each do |link|
            discovery = complete_extracted_url(link[:href], harmonized_url) if link[:rel] =~ %r{(alternate|service.feed)}i && Feedisco.feed_content_types.include?(link[:type].downcase.strip)
            feeds << discovery unless discovery.nil? or feeds.include? discovery
          end

          # Check <a> elements
          doc.css('a').each do |a|
            if looks_like_feed? a[:href] and
               (a[:href] =~ %r{\A/} or a[:href] =~ %r{#{URI.parse(harmonized_url).host}/})

              discovery = complete_extracted_url(a[:href], harmonized_url)
              feeds << discovery unless discovery.nil? or feeds.include?(discovery)
            end
          end

          # Check <a> elements again, less restrictively, so we add all discovered feeds even the ones
          # on external domains, but the will come after in the feeds array.
          doc.css('a').each do |a|
            discovery = complete_extracted_url(a[:href], harmonized_url) if looks_like_feed? a[:href]
            feeds << discovery unless discovery.nil? or feeds.include?(discovery)
          end
        end
      end

      feeds
    end
718377
public Set<CrawlerURL> get(HTMLPageResponse theResponse) {

    final String url = theResponse.getUrl();

    Set<CrawlerURL> ahrefs = new HashSet<CrawlerURL>();

    // only populate if we have a valid response, else return empty set
    if (theResponse.getResponseCode() == HttpStatus.SC_OK) {
      ahrefs = fetch(AHREF, ABS_HREF, theResponse.getBody(), url);
    }

    return ahrefs;
  }
716901
def fetch(self, cache=None):
        
        self.reset()
        soup = get(self.url).soup
        details = soup.find(id="detailsframe")
        getdef = lambda s: [elem
                for elem in details.find("dt",
                text=re.compile(s)).next_siblings
                if elem.name == 'dd'][0]
        getdefstring = lambda s: getdef(s).string.strip()
        info = {
            "title": details.find(id="title").string.strip(),
            "type": getdefstring("Type:"),
            "files": getdefstring("Files:"),
            "size": getdefstring("Size:"),
            "uploaded": getdefstring("Uploaded:"),
            "submitter": getdef("By:").parent.find("a", href=re.compile("user")).string.strip(),
            "seeders": getdefstring("Seeders:"),
            "leechers": getdefstring("Leechers:"),
            "comments": details.find(id="NumComments").string.strip(),
            "link": details.find("a", href=re.compile("^magnet\:"))['href'].strip(),
        }
        if self._use_cache(cache):
            self._attrs = info
        self._fetched = True
        return info
362517
def urljoin(*parts):
    
    # first strip extra forward slashes (except http:// and the likes) and
    # create list
    part_list = []
    for part in parts:
        p = str(part)
        if p.endswith('//'):
            p = p[0:-1]
        else:
            p = p.strip('/')
        part_list.append(p)
    # join everything together
    url = '/'.join(part_list)
    return url
1166223
def _get_pages(self, locations, project_name):
        # type: (Iterable[Link], str) -> Iterable[HTMLPage]
        
        seen = set()  # type: Set[Link]
        for location in locations:
            if location in seen:
                continue
            seen.add(location)

            page = _get_html_page(location, session=self.session)
            if page is None:
                continue

            yield page