// Download downloads remote package without version control.
429307
func (n *Node) Download(ctx *cli.Context) ([]string, error) {
	for _, s := range services {
		if !strings.HasPrefix(n.DownloadURL, s.prefix) {
			continue
		}

		m := s.pattern.FindStringSubmatch(n.DownloadURL)
		if m == nil {
			if s.prefix != "" {
				return nil, errors.New("Cannot match package service prefix by given path")
			}
			continue
		}

		match := map[string]string{"downloadURL": n.DownloadURL}
		for i, n := range s.pattern.SubexpNames() {
			if n != "" {
				match[n] = m[i]
			}
		}
		return s.get(HttpClient, match, n, ctx)

	}

	if n.ImportPath != n.DownloadURL {
		return nil, errors.New("Didn't find any match service")
	}

	log.Info("Cannot match any service, getting dynamic...")
	return n.getDynamic(HttpClient, ctx)
}
1358077
func (s *URLScriptHandler) Install(r *api.InstallResult) error {
	downloadURL, err := url.Parse(r.URL)
	if err != nil {
		return err
	}
	dst := filepath.Join(s.DestinationDir, constants.UploadScripts, r.Script)
	if _, err := s.Download.Download(downloadURL, dst); err != nil {
		if e, ok := err.(s2ierr.Error); ok {
			if e.ErrorCode == s2ierr.ScriptsInsideImageError {
				r.Installed = true
				return nil
			}
		}
		return err
	}
	if err := s.FS.Chmod(dst, 0755); err != nil {
		return err
	}
	r.Installed = true
	r.Downloaded = true
	return nil
}
264580
func (batcher *DownloadObjectsIterator) DownloadObject() BatchDownloadObject {
	object := batcher.Objects[batcher.index]
	return object
}
1033967
func (m *Metadata) DownloadURL() (*url.URL, error) {
	if m.BaseURL == "" {
		m.BaseURL = imagemetadata.UbuntuCloudImagesURL
	}
	u, err := url.Parse(m.BaseURL + "/" + m.Path)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create url")
	}
	return u, nil
}
35443
func (c *Client) Fetch(ctx context.Context, ref string, opts ...RemoteOpt) (images.Image, error) {
	fetchCtx := defaultRemoteContext()
	for _, o := range opts {
		if err := o(c, fetchCtx); err != nil {
			return images.Image{}, err
		}
	}

	if fetchCtx.Unpack {
		return images.Image{}, errors.New("unpack on fetch not supported, try pull")
	}

	if fetchCtx.PlatformMatcher == nil {
		if len(fetchCtx.Platforms) == 0 {
			fetchCtx.PlatformMatcher = platforms.All
		} else {
			var ps []ocispec.Platform
			for _, s := range fetchCtx.Platforms {
				p, err := platforms.Parse(s)
				if err != nil {
					return images.Image{}, errors.Wrapf(err, "invalid platform %s", s)
				}
				ps = append(ps, p)
			}

			fetchCtx.PlatformMatcher = platforms.Any(ps...)
		}
	}

	ctx, done, err := c.WithLease(ctx)
	if err != nil {
		return images.Image{}, err
	}
	defer done(ctx)

	return c.fetch(ctx, fetchCtx, ref, 0)
}
567606
func (m *MWApi) Download(filename string) (io.ReadCloser, error) {
	// First get the direct url of the file
	query := map[string]string{
		"action": "query",
		"prop":   "imageinfo",
		"iiprop": "url",
		"titles": filename,
	}

	body, err := m.API(query)
	if err != nil {
		return nil, err
	}

	var response Response
	err = json.Unmarshal(body, &response)
	if err != nil {
		return nil, err
	}
	pl := response.PageSlice()

	if len(pl) < 1 {
		return nil, errors.New("no file found")
	}
	page := pl[0]
	if len(page.Imageinfo) < 1 {
		return nil, errors.New("no file found")
	}
	fileurl := page.Imageinfo[0].Url

	// Then return the body of the response
	request, err := http.NewRequest("GET", fileurl, nil)
	if err != nil {
		return nil, err
	}
	request.Header.Set("user-agent", m.userAgent)
	if m.UseBasicAuth {
		request.SetBasicAuth(m.BasicAuthUser, m.BasicAuthPass)
	}

	resp, err := m.client.Do(request)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}
869928
func (self *Phantom) Download(req Request) (resp *http.Response, err error) {
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.GetHeader().Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.GetHeader().Del("Content-Type")

	param, err := NewParam(req)
	if err != nil {
		return nil, err
	}

	cookie := ""
	if req.GetEnableCookie() {
		httpCookies := self.CookieJar.Cookies(param.url)
		if len(httpCookies) > 0 {
			surferCookies := make([]*Cookie, len(httpCookies))

			for n, c := range httpCookies {
				surferCookie := &Cookie{Name: c.Name, Value: c.Value, Domain: param.url.Host, Path: "/"}
				surferCookies[n] = surferCookie
			}

			c, err := json.Marshal(surferCookies)
			if err != nil {
				log.Printf("cookie marshal error:%v", err)
			}
			cookie = string(c)
		}
	}

	resp = param.writeback(resp)
	resp.Request.URL = param.url

	var args = []string{
		self.jsFileMap["js"],
		req.GetUrl(),
		cookie,
		encoding,
		param.header.Get("User-Agent"),
		req.GetPostData(),
		strings.ToLower(param.method),
		fmt.Sprint(int(req.GetDialTimeout() / time.Millisecond)),
	}
	if req.GetProxy() != "" {
		args = append([]string{"--proxy=" + req.GetProxy()}, args...)
	}

	for i := 0; i < param.tryTimes; i++ {
		if i != 0 {
			time.Sleep(param.retryPause)
		}

		cmd := exec.Command(self.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			continue
		}

		if retResp.Error != "" {
			log.Printf("phantomjs response error:%s", retResp.Error)
			continue
		}

		//设置header
		for _, h := range retResp.Header {
			resp.Header.Add(h.Name, h.Value)
		}

		//设置cookie
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		if req.GetEnableCookie() {
			if rc := resp.Cookies(); len(rc) > 0 {
				self.CookieJar.SetCookies(param.url, rc)
			}
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}
	return
}
1165054
func (c *Context) Download(name string) (string, error) {
	deps := &contextDeps{
		APIClient: c.apiClient,
		name:      name,
		dataDir:   c.dataDir,
	}
	path, err := internal.ContextDownload(deps)
	if err != nil {
		return "", errors.Trace(err)
	}
	return path, nil
}
1062360
func (r *Repository) GetDownloadsURL() string {
	if r == nil || r.DownloadsURL == nil {
		return ""
	}
	return *r.DownloadsURL
}
961544
func (d *BundlesDir) download(info BundleInfo, target string, abort <-chan struct{}) (err error) {
	// First download...
	curl, err := url.Parse(info.URL().String())
	if err != nil {
		return errors.Annotate(err, "could not parse charm URL")
	}
	expectedSha256, err := info.ArchiveSha256()
	req := downloader.Request{
		URL:       curl,
		TargetDir: downloadsPath(d.path),
		Verify:    downloader.NewSha256Verifier(expectedSha256),
		Abort:     abort,
	}
	logger.Infof("downloading %s from API server", info.URL())
	filename, err := d.downloader.Download(req)
	if err != nil {
		return errors.Annotatef(err, "failed to download charm %q from API server", info.URL())
	}
	defer errors.DeferredAnnotatef(&err, "downloaded but failed to copy charm to %q from %q", target, filename)

	// ...then move the right location.
	if err := os.MkdirAll(d.path, 0755); err != nil {
		return errors.Trace(err)
	}
	if err := os.Rename(filename, target); err != nil {
		return errors.Trace(err)
	}
	return nil
}
601200
func (s *Downloader) Download(handler http.Handler, etag string) (returnSpec *spec.Swagger, newEtag string, httpStatus int, err error) {
	handler = s.handlerWithUser(handler, &user.DefaultInfo{Name: aggregatorUser})
	handler = http.TimeoutHandler(handler, specDownloadTimeout, "request timed out")

	req, err := http.NewRequest("GET", "/openapi/v2", nil)
	if err != nil {
		return nil, "", 0, err
	}
	req.Header.Add("Accept", "application/json")

	// Only pass eTag if it is not generated locally
	if len(etag) > 0 && !strings.HasPrefix(etag, locallyGeneratedEtagPrefix) {
		req.Header.Add("If-None-Match", etag)
	}

	writer := newInMemoryResponseWriter()
	handler.ServeHTTP(writer, req)

	switch writer.respCode {
	case http.StatusNotModified:
		if len(etag) == 0 {
			return nil, etag, http.StatusNotModified, fmt.Errorf("http.StatusNotModified is not allowed in absence of etag")
		}
		return nil, etag, http.StatusNotModified, nil
	case http.StatusNotFound:
		// Gracefully skip 404, assuming the server won't provide any spec
		return nil, "", http.StatusNotFound, nil
	case http.StatusOK:
		openAPISpec := &spec.Swagger{}
		if err := json.Unmarshal(writer.data, openAPISpec); err != nil {
			return nil, "", 0, err
		}
		newEtag = writer.Header().Get("Etag")
		if len(newEtag) == 0 {
			newEtag = etagFor(writer.data)
			if len(etag) > 0 && strings.HasPrefix(etag, locallyGeneratedEtagPrefix) {
				// The function call with an etag and server does not report an etag.
				// That means this server does not support etag and the etag that passed
				// to the function generated previously by us. Just compare etags and
				// return StatusNotModified if they are the same.
				if etag == newEtag {
					return nil, etag, http.StatusNotModified, nil
				}
			}
		}
		return openAPISpec, newEtag, http.StatusOK, nil
	default:
		return nil, "", 0, fmt.Errorf("failed to retrieve openAPI spec, http error: %s", writer.String())
	}
}
1178086
func (d *downloader) Download(url *url.URL, targetFile string) (*git.SourceInfo, error) {
	r := d.schemeReaders[url.Scheme]
	info := &git.SourceInfo{}
	if r == nil {
		glog.Errorf("No URL handler found for %s", url.String())
		return nil, s2ierr.NewURLHandlerError(url.String())
	}

	reader, err := r.Read(url)
	if err != nil {
		return nil, err
	}
	defer reader.Close()

	out, err := os.Create(targetFile)
	defer out.Close()

	if err != nil {
		glog.Errorf("Unable to create target file %s (%v)", targetFile, err)
		return nil, err
	}

	if _, err = io.Copy(out, reader); err != nil {
		os.Remove(targetFile)
		glog.Warningf("Skipping file %s due to error copying from source: %v", targetFile, err)
		return nil, err
	}

	glog.V(2).Infof("Downloaded '%s'", url.String())
	info.Location = url.String()
	return info, nil
}
429567
func Fetch(remotes ...string) error {
	if len(remotes) == 0 {
		return nil
	}

	var args []string
	if len(remotes) > 1 {
		args = []string{"--multiple", "--"}
	}
	args = append(args, remotes...)

	_, err := gitNoLFSSimple(append([]string{"fetch"}, args...)...)
	return err
}
1199583
func (n *Noop) Download(config *api.Config) (*git.SourceInfo, error) {
	glog.V(1).Info("No source location defined (the assemble script is responsible for obtaining the source)")

	return &git.SourceInfo{}, nil
}
798875
func (r *Repo) fetch(ctx context.Context) error {
	if err := fetch(ctx, r.dir, "origin"); err != nil {
		return err
	}
	return nil
}
481538
func (d *Downloader) Download(parentCtx context.Context, paths []string, fn DownloaderFunc) (err error) {
	// Init
	ctx, cancel := context.WithCancel(parentCtx)
	m := &sync.Mutex{} // Locks err
	gwg := &sync.WaitGroup{}
	gwg.Add(len(paths))
	lwg := &sync.WaitGroup{}

	// Loop through src paths
	var idx int
	for idx < len(paths) {
		// Check context
		if parentCtx.Err() != nil {
			m.Lock()
			err = errors.Wrap(err, "astihttp: context error")
			m.Unlock()
			return
		}

		// Lock cond here in case a worker finishes between checking the number of busy workers and the if statement
		d.cond.L.Lock()

		// Check if a worker is available
		var ok bool
		d.mw.Lock()
		if ok = d.numberOfWorkers > d.busyWorkers; ok {
			d.busyWorkers++
		}
		d.mw.Unlock()

		// No worker is available
		if !ok {
			d.cond.Wait()
			d.cond.L.Unlock()
			continue
		}
		d.cond.L.Unlock()

		// Check error
		m.Lock()
		if err != nil {
			m.Unlock()
			lwg.Wait()
			return
		}
		m.Unlock()

		// Download
		go func(idx int) {
			lwg.Add(1)
			if errR := d.download(ctx, idx, paths[idx], fn, gwg); errR != nil {
				m.Lock()
				if err == nil {
					err = errR
				}
				m.Unlock()
				cancel()
			}
			lwg.Done()
		}(idx)
		idx++
	}
	gwg.Wait()
	return
}
1207339
func Download(pkgSrc PkgSrc, dstFilePath string, w io.Writer) (rErr error) {
	wantChecksum := pkgSrc.Checksum()
	if info, err := os.Stat(dstFilePath); err == nil {
		if info.IsDir() {
			return errors.Errorf("destination path %s already exists and is a directory", dstFilePath)
		}
		if wantChecksum != "" {
			// if file already exists at destination and checksum is known, verify checksum of existing file.
			// If it matches, use existing file.
			checksum, err := computeSHA256Checksum(dstFilePath)
			if err != nil {
				return errors.Wrapf(err, "failed to compute checksum of %s", dstFilePath)
			}
			if checksum == wantChecksum {
				return nil
			}
		}
	}

	// open reader from source
	r, size, err := pkgSrc.Reader()
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil && rErr == nil {
			rErr = errors.Wrapf(err, "failed to close reader for %s in defer", pkgSrc.Path())
		}
	}()

	// create new file for package (overwrite any existing file)
	dstFile, err := os.Create(dstFilePath)
	if err != nil {
		return errors.Wrapf(err, "failed to create file %s", dstFilePath)
	}
	defer func() {
		if err := dstFile.Close(); err != nil && rErr == nil {
			rErr = errors.Wrapf(err, "failed to close file %s in defer", dstFilePath)
		}
	}()

	h := sha256.New()
	mw := io.MultiWriter(h, dstFile)

	_, _ = fmt.Fprintf(w, "Getting package from %v...\n", pkgSrc.Path())
	if err := copyWithProgress(mw, r, size, w); err != nil {
		return errors.Wrapf(err, "failed to copy package %s to %s", pkgSrc.Path(), dstFilePath)
	}

	// verify checksum if provided
	if wantChecksum != "" {
		actualChecksum := hex.EncodeToString(h.Sum(nil))
		if wantChecksum != actualChecksum {
			return errors.Errorf("SHA-256 checksum of downloaded package did not match expected checksum: expected %s, was %s", wantChecksum, actualChecksum)
		}
	}
	return nil
}