// Checks if the given URL can be fetched based on robots.txt policies.
496002
func (w *worker) isAllowedPerRobotsPolicies(u *url.URL) bool {
	if w.robotsGroup != nil {
		// Is this URL allowed per robots.txt policy?
		ok := w.robotsGroup.Test(u.Path)
		if !ok {
			w.logFunc(LogIgnored, "ignored on robots.txt policy: %s", u.String())
		}
		return ok
	}

	// No robots.txt = everything is allowed
	return true
}
378076
func (goPlatform *Platform) ValidatePath(rawPath string) error {
	path, err := url.Parse(rawPath)
	if err != nil || path == nil {
		return fmt.Errorf("invalid path: %s", err)
	}

	//we have no real good way of checking existence of remote urls except by downloading and testing
	//which we do later anyway. But we *can* - and *should* - test for existence of local paths.
	//Treat empty scheme as a local filesystem path
	if path.Scheme == "" {
		gopath, err := getGopath()
		if err != nil {
			return err
		}
		pathToCheck := filepath.Join(gopath, "src", rawPath)
		exists, err := pathExists(pathToCheck)
		if err != nil {
			return fmt.Errorf("error validating chaincode path: %s", err)
		}
		if !exists {
			return fmt.Errorf("path to chaincode does not exist: %s", pathToCheck)
		}
	}
	return nil
}
1138619
func (p *Policy) RequireParseableURLs(require bool) *Policy {

	p.requireParseableURLs = require

	return p
}
1089839
func (de *DefaultExtender) RequestRobots(ctx *URLContext, robotAgent string) (data []byte, doRequest bool) {
	return nil, true
}
1107321
func (connector *Connector) IsSupportURL(url string) bool {
	return strings.HasPrefix(url, "git+") || strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git")
}
962472
func IsRetriableError(err error) bool {
	if e, ok := err.(interface {
		RetriableError() bool
	}); ok {
		return e.RetriableError()
	}
	if cause, ok := Cause(err).(*url.Error); ok {
		return cause.Temporary() || cause.Timeout()
	}
	if parent := parentOf(err); parent != nil {
		return IsRetriableError(parent)
	}
	return false
}
457194
func Fetch(target interface{}, url string) error {
	doc, err := goquery.NewDocument(url)
	if err != nil {
		return err
	}
	return Populate(target, doc)
}
947356
func (rd *Redirector) ClickableURL() string {
	if rd.server == nil {
		return "<undefined - server is not started>"
	}
	return utils.ClickableURL(rd.server.URL + rd.shortPath)
}
1243640
func Fetch(url string) (ret []byte, err error) {
	resp, err := http.Get(url)
	defer func() {
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}()
	if err != nil {
		Log(WARN, err.Error())
		return
	}
	ret, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	return
}
1030287
func Fetch(url string) (doc *goquery.Document, err error) {
	log.WithFields(log.Fields{"url": url}).Info("Fetch sends request")

	// set request
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.WithFields(log.Fields{"url": url, "err": err}).Error("Failed to generate new request")
		return nil, err
	}
	req.Header.Set("User-Agent", USER_AGENT)

	// send request and get response
	client := http.DefaultClient
	res, err := client.Do(req)
	if err != nil {
		log.WithFields(log.Fields{"url": url, "err": err}).Error("Failed to get response")
		return nil, err
	}

	// generate new Document
	doc, err = goquery.NewDocumentFromResponse(res)
	log.WithFields(log.Fields{"doc.url": doc.Url}).Info("goquery.Document is generated")
	if err != nil {
		log.WithFields(log.Fields{"url": url, "err": err}).Error("Generating goquery.Documentation failed")
		return nil, err
	}

	// 1. check the "Please show you're not a robot" page. See #61
	// 2. check the "We're sorry..."
	if s := doc.Find("h1").First().Text(); strings.Contains(s, "robot") || strings.Contains(s, "sorry") {
		log.WithFields(log.Fields{"h1": s, "doc.Url": doc.Url}).Error("Robot check occurs")
		return nil, errors.New("Failed to fetch Document")
	}

	// check the "To continue, please type the characters below:". See #55
	if strings.Contains(doc.Url.String(), "sorry") {
		log.WithFields(log.Fields{"doc.Url": doc.Url}).Error("Request is rejected from Google")
		return nil, errors.New("Failed to fetch Document")
	}

	return doc, nil
}
593787
func checkURL(url string) error {
	resp, err := http.Get(url)
	glog.Infof("%s response: %v %+v", url, err, resp)
	if err != nil {
		return errors.Wrap(err, "checkURL")
	}
	if resp.StatusCode != http.StatusOK {
		return &util.RetriableError{
			Err: fmt.Errorf("unexpected response code: %d", resp.StatusCode),
		}
	}
	return nil
}
1268877
func checkCandidateBase(l *absurllexer) {
	l.consumeQuote()

	if !bytes.HasPrefix(l.content[l.pos:], relURLPrefix) {
		return
	}

	// check for schemaless URLs
	posAfter := l.pos + relURLPrefixLen
	if posAfter >= len(l.content) {
		return
	}
	r, _ := utf8.DecodeRune(l.content[posAfter:])
	if r == '/' {
		// schemaless: skip
		return
	}
	if l.pos > l.start {
		l.emit()
	}
	l.pos += relURLPrefixLen
	l.w.Write(l.path)
	l.start = l.pos
}
141505
func IsGitURL(str string) bool {
	if IsURL(str) && urlPathWithFragmentSuffix.MatchString(str) {
		return true
	}
	return checkURL(str, "git")
}
483432
func (h *urlDataSource) Fetch(path string) (io.ReadCloser, string, error) {
	dataURL := urlJoin(h.baseURL, path)
	client := utils.GetHTTPClient(h.hostnameVerification)
	// dataURL can be http:// or file://
	// MakeFileURL will only modify the URL if it's a file URL
	dataURL = utils.MakeFileURL(dataURL)
	resp, err := client.Get(dataURL)
	if err != nil {
		logger.Tracef("Got error requesting %q: %v", dataURL, err)
		return nil, dataURL, errors.NotFoundf("invalid URL %q", dataURL)
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		switch resp.StatusCode {
		case http.StatusNotFound:
			return nil, dataURL, errors.NotFoundf("cannot find URL %q", dataURL)
		case http.StatusUnauthorized:
			return nil, dataURL, errors.Unauthorizedf("unauthorised access to URL %q", dataURL)
		}
		return nil, dataURL, fmt.Errorf("cannot access URL %q, %q", dataURL, resp.Status)
	}
	return resp.Body, dataURL, nil
}
200006
func IsUri(url string) bool {
	if !govalidator.IsURL(url) || !strings.HasPrefix(url, "http") {
		return false
	}
	return true
}
589763
func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool {
	asString, ok := input.(string)
	if !ok {
		return false
	}

	_, err := url.Parse(asString)
	return err == nil && !strings.Contains(asString, `\`)
}
1020351
func (p *UserAgent) googleBot() bool {
	// This is a hackish way to detect Google's mobile bot (Googlebot, AdsBot-Google-Mobile, etc.).
	// See https://support.google.com/webmasters/answer/1061943
	if strings.Index(p.ua, "Google") != -1 {
		p.platform = ""
		p.undecided = true
	}
	return p.undecided
}