// Check if the specified URL is from the same host as its source URL, or if
// nil, from the same host as one of the seed URLs.
970985
func (c *Crawler) isSameHost(ctx *URLContext) bool {
	// If there is a source URL, then just check if the new URL is from the same host
	if ctx.normalizedSourceURL != nil {
		return ctx.normalizedURL.Host == ctx.normalizedSourceURL.Host
	}

	// Otherwise, check if the URL is from one of the seed hosts
	_, ok := c.hosts[ctx.normalizedURL.Host]
	return ok
}
603449
func sameOrigin(a, b *url.URL) bool {
	return (a.Scheme == b.Scheme && a.Host == b.Host)
}
901290
func (q *Query) FindUrl() (url string) {
	return fmt.Sprintf(find_url, q.ClusterId)
}
421206
func (a *Addr) isURL(value string) bool {
	prefixes := urlPrefixes
	if a.DefaultScheme != "" {
		prefixes = append(prefixes, fmt.Sprintf("%s://", a.DefaultScheme))
	}
	for _, p := range prefixes {
		if strings.HasPrefix(value, p) {
			return true
		}
	}
	return false
}
257312
func IsLocalHost(hostport string) bool {
	host, _, err := net.SplitHostPort(hostport)
	if err != nil {
		return false
	}
	switch {
	case host == "localhost", host == "":
	case net.ParseIP(host).IsLoopback():

	default:
		return false
	}
	return true
}
475176
func (uc *URLContext) cloneForRedirect(dst *url.URL, normFlags purell.NormalizationFlags) *URLContext {
	var src, normalizedSrc *url.URL
	if uc.sourceURL != nil {
		src = &url.URL{}
		*src = *uc.sourceURL
	}
	if src == nil && uc.url != nil {
		// if the current context doesn't have a source URL, use its URL as
		// source (e.g. for a seed URL that triggers a redirect)
		src = &url.URL{}
		*src = *uc.url
	}

	if uc.normalizedSourceURL != nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedSourceURL
	}
	if normalizedSrc == nil {
		normalizedSrc = &url.URL{}
		*normalizedSrc = *uc.normalizedURL
	}

	rawDst := &url.URL{}
	*rawDst = *dst
	purell.NormalizeURL(dst, normFlags)
	return &URLContext{
		HeadBeforeGet:       uc.HeadBeforeGet,
		State:               uc.State,
		url:                 rawDst,
		normalizedURL:       dst,
		sourceURL:           src,
		normalizedSourceURL: normalizedSrc,
	}
}
113445
func (mbs maybeSources) getURL() string {
	strslice := make([]string, 0, len(mbs))
	for _, mb := range mbs {
		strslice = append(strslice, mb.getURL())
	}
	return strings.Join(strslice, "\n")
}
883093
func Url() string {
	n := intRange(0, 3)

	base := `http://www.` + Host()

	switch n {
	case 0:
		break
	case 1:
		base += "/" + Word(2, 8)
	case 2:
		base += "/" + Word(2, 8) + "/" + Word(2, 8) + ".html"
	}
	return base
}
771466
func urlsAreEqual(u1, u2 *url.URL) bool {
	return reflect.DeepEqual(u1, u2)
}
1219654
func isRemoteHTTPArchive(source string) bool {
	if strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") {
		for suffix := range Extractors {
			if strings.HasSuffix(source, suffix) {
				return true
			}
		}
	}
	return false
}
894689
func isHTTPSURL(s string) bool {
	u, err := url.Parse(s)
	return err == nil && u.Scheme == "https"
}
521664
func CopyURL(u *url.URL) (ret *url.URL) {
	ret = new(url.URL)
	*ret = *u
	if u.User != nil {
		ret.User = new(url.Userinfo)
		*ret.User = *u.User
	}
	return
}
44704
func URL(c CloneURL, path string) *Resource {
	r := &Resource{u: c.URL()}
	r.u.Path = Path + path
	return r
}
251399
func doShareDownloadURL(targetURL string, isRecursive bool, expiry time.Duration) *probe.Error {
	targetAlias, targetURLFull, _, err := expandAlias(targetURL)
	if err != nil {
		return err.Trace(targetURL)
	}
	clnt, err := newClientFromAlias(targetAlias, targetURLFull)
	if err != nil {
		return err.Trace(targetURL)
	}

	// Load previously saved upload-shares. Add new entries and write it back.
	shareDB := newShareDBV1()
	shareDownloadsFile := getShareDownloadsFile()
	err = shareDB.Load(shareDownloadsFile)
	if err != nil {
		return err.Trace(shareDownloadsFile)
	}

	// Generate share URL for each target.
	isIncomplete := false
	isFetchMeta := false
	// Channel which will receive objects whose URLs need to be shared
	objectsCh := make(chan *clientContent)

	content, err := clnt.Stat(isIncomplete, isFetchMeta, nil)
	if err != nil {
		return err.Trace(clnt.GetURL().String())
	}

	if !content.Type.IsDir() {
		go func() {
			defer close(objectsCh)
			objectsCh <- content
		}()
	} else {
		if !strings.HasSuffix(targetURLFull, string(clnt.GetURL().Separator)) {
			targetURLFull = targetURLFull + string(clnt.GetURL().Separator)
		}
		clnt, err = newClientFromAlias(targetAlias, targetURLFull)
		if err != nil {
			return err.Trace(targetURLFull)
		}
		// Recursive mode: Share list of objects
		go func() {
			defer close(objectsCh)
			for content := range clnt.List(isRecursive, isIncomplete, DirNone) {
				objectsCh <- content
			}
		}()
	}

	// Iterate over all objects to generate share URL
	for content := range objectsCh {
		if content.Err != nil {
			return content.Err.Trace(clnt.GetURL().String())
		}
		// if any incoming directories, we don't need to calculate.
		if content.Type.IsDir() {
			continue
		}
		objectURL := content.URL.String()
		newClnt, err := newClientFromAlias(targetAlias, objectURL)
		if err != nil {
			return err.Trace(objectURL)
		}

		// Generate share URL.
		shareURL, err := newClnt.ShareDownload(expiry)
		if err != nil {
			// add objectURL and expiry as part of the trace arguments.
			return err.Trace(objectURL, "expiry="+expiry.String())
		}

		// Make new entries to shareDB.
		contentType := "" // Not useful for download shares.
		shareDB.Set(objectURL, shareURL, expiry, contentType)
		printMsg(shareMesssage{
			ObjectURL:   objectURL,
			ShareURL:    shareURL,
			TimeLeft:    expiry,
			ContentType: contentType,
		})
	}

	// Save downloads and return.
	return shareDB.Save(shareDownloadsFile)
}
633007
func (t *vhostTrie) matchHost(host string) *vhostTrie {
	// try exact match
	if subtree, ok := t.edges[host]; ok {
		return subtree
	}

	// then try replacing labels in the host
	// with wildcards until we get a match
	labels := strings.Split(host, ".")
	for i := range labels {
		labels[i] = "*"
		candidate := strings.Join(labels, ".")
		if subtree, ok := t.edges[candidate]; ok {
			return subtree
		}
	}

	return nil
}
372138
func isURLContains(srcURL, tgtURL, sep string) bool {
	// Add a separator to source url if not found
	if !strings.HasSuffix(srcURL, sep) {
		srcURL += sep
	}
	if !strings.HasSuffix(tgtURL, sep) {
		tgtURL += sep
	}
	// Check if we are going to copy a directory into itself
	if strings.HasPrefix(tgtURL, srcURL) {
		return true
	}
	return false
}
665964
func (u *URI) CloneURL() *url.URL {
	return &url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
		Path:   u.Path,
	}
}