main function to search for links and return valid ones
436313
def search(query, engine='g', site="", file_type = 'pdf', limit = 10):
	
	if site == "":
		search_query = "filetype:{0} {1}".format(file_type, query)
	else:
		search_query = "site:{0} filetype:{1} {2}".format(site,file_type, query)

	headers = {
		'User Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:53.0) \
		Gecko/20100101 Firefox/53.0'
	}
	if engine == "g":
		params = {
			'q': search_query,
			'start': 0,
		}
		links = get_google_links(limit, params, headers)

	elif engine == "d":
		params = {
			'q': search_query,
		}
		links = get_duckduckgo_links(limit,params,headers)
	else:
		print("Wrong search engine selected!")
		sys.exit()
	
	valid_links = validate_links(links)
	return valid_links
1228647
def do_search(self, string):
        """"""
        results = self.n.doSearch(string, full_path = self.current_path)

        if results:
            for r in results:
                self.stdout.write("%s\n" % r['path'])
766376
function checkBrokenLinks(docs, apis, options) {
  var byFullId = Object.create(null);

  docs.forEach(function(doc) {
    byFullId[doc.section + '/' + doc.id] = doc;
    if (apis[doc.section]) {
      doc.anchors.push('directive', 'service', 'filter', 'function');
    }
  });

  docs.forEach(function(doc) {
    doc.links.forEach(function(link) {
      if (options && !options.html5Mode) {
        link = link.substring(2);
      }
      // convert #id to path#id
      if (link[0] == '#') {
        link = doc.section + '/' + doc.id.split('#').shift() + link;
      }

      var parts = link.split('#');
      var pageLink = parts[0];
      var anchorLink = parts[1];
      var linkedPage = byFullId[pageLink];

      if (!linkedPage) {
        console.log('WARNING: ' + doc.section + '/' + doc.id + ' (defined in ' + doc.file + ') points to a non existing page "' + link + '"!');
      } else if (anchorLink && linkedPage.anchors.indexOf(anchorLink) === -1) {
        console.log('WARNING: ' + doc.section + '/' + doc.id + ' (defined in ' + doc.file + ') points to a non existing anchor "' + link + '"!');
      }
    });
  });
}
919839
function checkLink()
    {
        if (!isset($_POST['link'])) {
            $this->sendError();
        }

        $link = strtolower(str_replace([" ",'"',"'",';','.',','], ["-",""], preg_replace("/&([a-z])[a-z]+;/i", "$1", htmlentities(trim($_POST['link'])))));

        $base = new Model\Base;
        $status = $base->checkLink($link, $_POST['aID']) === false ? 'ok' : 'error';

        $this->send(['status'=>$status,'link'=>$link]);
    }
517754
def check(self, url_data):
        """"""
        content = url_data.get_content()
        self._check_by_re(url_data, content)
        self._check_inline_links(url_data, content)
153667
def search(query, num_results=10):
	
	data = download(query, num_results)
	results = re.findall(r'\<h3.*?\>.*?\<\/h3\>', data, re.IGNORECASE)
	if results is None or len(results) == 0:
		print('No results where found. Did the rate limit exceed?')
		return []
	# search has results
	links = []
	for r in results:
		mtch = re.match(r'.*?a\s*?href=\"(.*?)\".*?\>(.*?)\<\/a\>.*$', r, flags=re.IGNORECASE)
		if mtch is None:
			continue
		# parse url
		url = mtch.group(1)
		# clean url https://github.com/aviaryan/pythons/blob/master/Others/GoogleSearchLinks.py
		url = re.sub(r'^.*?=', '', url, count=1) # prefixed over urls \url=q?
		url = re.sub(r'\&amp.*$', '', url, count=1) # suffixed google things
		url = unquote(url)
		# url = re.sub(r'\%.*$', '', url) # NOT SAFE, causes issues with Youtube watch url
		# parse name
		name = prune_html(mtch.group(2))
		name = convert_unicode(name)
		# append to links
		if is_url(url): # can be google images result
			links.append((name, url))
	return links
1020143
def rel_links(cls, page):
    """"""
    for match in cls.REL_RE.finditer(page):
      href, rel = match.group(0), match.group(1)
      if rel not in cls.REL_TYPES:
        continue
      href_match = cls.HREF_RE.search(href)
      if href_match:
        href = cls.href_match_to_url(href_match)
        parsed_href = urlparse(href)
        if any(parsed_href.path.endswith(ext) for ext in cls.REL_SKIP_EXTENSIONS):
          continue
        yield href
1373441
public function detect_links()
	{
		$this->loop_text_nodes(function($child) {
			preg_match_all("/(?:(?:https?|ftp):\/\/|(?:www|ftp)\.)(?:[a-zA-Z0-9\-\.]{1,255}\.[a-zA-Z]{1,20})(?::[0-9]{1,5})?(?:\/[^\s'\"]*)?(?:(?<![,\)\.])|[\S])/",
				$child->get_text(),
				$matches,
				PREG_PATTERN_ORDER | PREG_OFFSET_CAPTURE);

			if(count($matches[0]) == 0)
				return;

			$replacment = array();
			$last_pos   = 0;

			foreach($matches[0] as $match)
			{
				if(substr($match[0], 0, 3) === 'ftp' && $match[0][3] !== ':')
					$url = 'ftp://' . $match[0];
				else if($match[0][0] === 'w')
					$url = 'http://' . $match[0];
				else
					$url = $match[0];

				$url      = new Node_Container_Tag('url', array('default' => htmlentities($url, ENT_QUOTES | ENT_IGNORE, "UTF-8")));
				$url_text = new Node_Text($match[0]);
				$url->add_child($url_text);

				$replacment[] = new Node_Text(substr($child->get_text(), $last_pos, $match[1] - $last_pos));
				$replacment[] = $url;
				$last_pos = $match[1] + strlen($match[0]);
			}

			$replacment[] = new Node_Text(substr($child->get_text(), $last_pos));
			$child->parent()->replace_child($child, $replacment);
		}, $this->get_excluded_tags(BBCode::AUTO_DETECT_EXCLUDE_URL));

		return $this;
	}
217602
function isFoundIn (str) {
  str = stripUrls(str)
  return function find (searchfor) {
    if (searchfor instanceof RegExp) return searchfor.test(str)
    else if (typeof searchfor === 'function') return searchfor(str)
    else return str && str.indexOf(searchfor) >= 0
  }
}
50045
def process_links(self, links):
        
        links_list = []
        for link in links:
            if os.path.splitext(link)[1][1:].strip().lower() in self.format_list:
                links_list.append(link)
        return links_list
1295928
def fetchUrls(cls, url, data, urlSearch):
        """"""
        searchUrls = []
        if cls.css:
            searchFun = data.cssselect
        else:
            searchFun = data.xpath
        searches = makeSequence(urlSearch)
        for search in searches:
            for match in searchFun(search):
                try:
                    for attrib in html_link_attrs:
                        if attrib in match.attrib:
                            searchUrls.append(match.get(attrib))
                except AttributeError:
                    searchUrls.append(str(match))
            if not cls.multipleImagesPerStrip and searchUrls:
                # do not search other links if one pattern matched
                break
        if not searchUrls:
            raise ValueError("XPath %s not found at URL %s." % (searches, url))
        return searchUrls
221308
function _main() {
  const srcIndexPath = join(DIST_DIR, 'index.html');
  const src404BodyPath = join(SRC_DIR, '404-body.html');
  const dst404PagePath = join(DIST_DIR, '404.html');

  const srcIndexContent = readFileSync(srcIndexPath, 'utf8');
  const src404BodyContent = readFileSync(src404BodyPath, 'utf8');
  const dst404PageContent = srcIndexContent.replace(/<body>[\s\S]+<\/body>/, src404BodyContent);

  if (dst404PageContent === srcIndexContent) {
    throw new Error(
        'Failed to generate \'404.html\'. ' +
        'The content of \'index.html\' does not match the expected pattern.');
  }

  writeFileSync(dst404PagePath, dst404PageContent);
}
112962
function QueryString(search) {
        "use strict";
    
        if (search.substr(0, 1) === '?') {
            search = search.substr(1);
        }
    
        search = search.split('&')
    
        if (search == "") return {};
    
        for (var i = 0; i < search.length; ++i) {
    
            var param = search[i].split('=', 2);
            if (param.length != 2) {
                continue;
            }
    
            this[param[0]] = decodeURIComponent(param[1].replace(/\+/g, " "));
    
        }
    
    }
184567
function (str) {
        // Try fast path regexp
        // See: https://github.com/joyent/node/pull/7878
        var simplePath = typeof str === 'string' && simplePathRegExp.exec(str);

        // Construct simple URL
        if (simplePath) {
            var pathname = simplePath[1];
            var search = simplePath[2] || null;
            var url = urlModule.Url !== undefined ? new urlModule.Url() : {};
            url.path = str;
            url.href = str;
            url.pathname = pathname;
            url.search = search;
            url.query = search && search.substr(1);

            return url;
        }

        return urlModule.parse(str);
    }
101542
def list_links(args = {})
      if args.empty?
        links
      else
        links.select { |link| link.match? args }
      end
    end
185788
def getLinks(page):
    

    CheckParent(page)
    ln = page.firstLink
    links = []
    while ln:
        nl = getLinkDict(ln)
        if nl["kind"] == LINK_GOTO:
            if type(nl["to"]) is Point and nl["page"] >= 0:
                doc = page.parent
                target_page = doc[nl["page"]]
                ctm = target_page._getTransformation()
                point = nl["to"] * ctm
                nl["to"] = point
        links.append(nl)
        ln = ln.next
    if len(links) > 0:
        linkxrefs = page._getLinkXrefs()
        if len(linkxrefs) == len(links):
            for i in range(len(linkxrefs)):
                links[i]["xref"] = linkxrefs[i]
    return links
998341
def is_valid_url(url):
    """"""
    pieces = urlparse(url)
    return all([pieces.scheme, pieces.netloc])