// Download downloads in parallel a set of src paths and executes a custom callback on each downloaded buffers
481538
func (d *Downloader) Download(parentCtx context.Context, paths []string, fn DownloaderFunc) (err error) {
	// Init
	ctx, cancel := context.WithCancel(parentCtx)
	m := &sync.Mutex{} // Locks err
	gwg := &sync.WaitGroup{}
	gwg.Add(len(paths))
	lwg := &sync.WaitGroup{}

	// Loop through src paths
	var idx int
	for idx < len(paths) {
		// Check context
		if parentCtx.Err() != nil {
			m.Lock()
			err = errors.Wrap(err, "astihttp: context error")
			m.Unlock()
			return
		}

		// Lock cond here in case a worker finishes between checking the number of busy workers and the if statement
		d.cond.L.Lock()

		// Check if a worker is available
		var ok bool
		d.mw.Lock()
		if ok = d.numberOfWorkers > d.busyWorkers; ok {
			d.busyWorkers++
		}
		d.mw.Unlock()

		// No worker is available
		if !ok {
			d.cond.Wait()
			d.cond.L.Unlock()
			continue
		}
		d.cond.L.Unlock()

		// Check error
		m.Lock()
		if err != nil {
			m.Unlock()
			lwg.Wait()
			return
		}
		m.Unlock()

		// Download
		go func(idx int) {
			lwg.Add(1)
			if errR := d.download(ctx, idx, paths[idx], fn, gwg); errR != nil {
				m.Lock()
				if err == nil {
					err = errR
				}
				m.Unlock()
				cancel()
			}
			lwg.Done()
		}(idx)
		idx++
	}
	gwg.Wait()
	return
}
961544
func (d *BundlesDir) download(info BundleInfo, target string, abort <-chan struct{}) (err error) {
	// First download...
	curl, err := url.Parse(info.URL().String())
	if err != nil {
		return errors.Annotate(err, "could not parse charm URL")
	}
	expectedSha256, err := info.ArchiveSha256()
	req := downloader.Request{
		URL:       curl,
		TargetDir: downloadsPath(d.path),
		Verify:    downloader.NewSha256Verifier(expectedSha256),
		Abort:     abort,
	}
	logger.Infof("downloading %s from API server", info.URL())
	filename, err := d.downloader.Download(req)
	if err != nil {
		return errors.Annotatef(err, "failed to download charm %q from API server", info.URL())
	}
	defer errors.DeferredAnnotatef(&err, "downloaded but failed to copy charm to %q from %q", target, filename)

	// ...then move the right location.
	if err := os.MkdirAll(d.path, 0755); err != nil {
		return errors.Trace(err)
	}
	if err := os.Rename(filename, target); err != nil {
		return errors.Trace(err)
	}
	return nil
}
732724
func (r *ChartRepository) DownloadIndexFile(cachePath string) error {
	var indexURL string
	parsedURL, err := url.Parse(r.Config.URL)
	if err != nil {
		return err
	}
	parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") + "/index.yaml"

	indexURL = parsedURL.String()

	r.setCredentials()
	resp, err := r.Client.Get(indexURL)
	if err != nil {
		return err
	}

	index, err := ioutil.ReadAll(resp)
	if err != nil {
		return err
	}

	if _, err := loadIndex(index); err != nil {
		return err
	}

	// In Helm 2.2.0 the config.cache was accidentally switched to an absolute
	// path, which broke backward compatibility. This fixes it by prepending a
	// global cache path to relative paths.
	//
	// It is changed on DownloadIndexFile because that was the method that
	// originally carried the cache path.
	cp := r.Config.Cache
	if !filepath.IsAbs(cp) {
		cp = filepath.Join(cachePath, cp)
	}

	return ioutil.WriteFile(cp, index, 0644)
}
895874
func (c *Client) Download(name string) (io.ReadCloser, error) {
	out, err := c.Files.Download(&dropbox.DownloadInput{name})
	if err != nil {
		return nil, err
	}

	return out.Body, nil
}
1199583
func (n *Noop) Download(config *api.Config) (*git.SourceInfo, error) {
	glog.V(1).Info("No source location defined (the assemble script is responsible for obtaining the source)")

	return &git.SourceInfo{}, nil
}
869928
func (self *Phantom) Download(req Request) (resp *http.Response, err error) {
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.GetHeader().Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.GetHeader().Del("Content-Type")

	param, err := NewParam(req)
	if err != nil {
		return nil, err
	}

	cookie := ""
	if req.GetEnableCookie() {
		httpCookies := self.CookieJar.Cookies(param.url)
		if len(httpCookies) > 0 {
			surferCookies := make([]*Cookie, len(httpCookies))

			for n, c := range httpCookies {
				surferCookie := &Cookie{Name: c.Name, Value: c.Value, Domain: param.url.Host, Path: "/"}
				surferCookies[n] = surferCookie
			}

			c, err := json.Marshal(surferCookies)
			if err != nil {
				log.Printf("cookie marshal error:%v", err)
			}
			cookie = string(c)
		}
	}

	resp = param.writeback(resp)
	resp.Request.URL = param.url

	var args = []string{
		self.jsFileMap["js"],
		req.GetUrl(),
		cookie,
		encoding,
		param.header.Get("User-Agent"),
		req.GetPostData(),
		strings.ToLower(param.method),
		fmt.Sprint(int(req.GetDialTimeout() / time.Millisecond)),
	}
	if req.GetProxy() != "" {
		args = append([]string{"--proxy=" + req.GetProxy()}, args...)
	}

	for i := 0; i < param.tryTimes; i++ {
		if i != 0 {
			time.Sleep(param.retryPause)
		}

		cmd := exec.Command(self.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			continue
		}

		if retResp.Error != "" {
			log.Printf("phantomjs response error:%s", retResp.Error)
			continue
		}

		//设置header
		for _, h := range retResp.Header {
			resp.Header.Add(h.Name, h.Value)
		}

		//设置cookie
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		if req.GetEnableCookie() {
			if rc := resp.Cookies(); len(rc) > 0 {
				self.CookieJar.SetCookies(param.url, rc)
			}
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}
	return
}
1033967
func (m *Metadata) DownloadURL() (*url.URL, error) {
	if m.BaseURL == "" {
		m.BaseURL = imagemetadata.UbuntuCloudImagesURL
	}
	u, err := url.Parse(m.BaseURL + "/" + m.Path)
	if err != nil {
		return nil, errors.Annotate(err, "failed to create url")
	}
	return u, nil
}
536984
func (c *WindowsContainerCommunicator) Download(src string, dst io.Writer) error {
	log.Printf("Downloading file from container: %s:%s", c.ContainerID, src)
	// Copy file onto temp file on mounted volume inside container
	var stdout, stderr bytes.Buffer
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("Copy-Item -Path %s -Destination %s/%s", src, c.ContainerDir,
			filepath.Base(src)),
		Stdout: &stdout,
		Stderr: &stderr,
	}
	ctx := context.TODO()
	if err := c.Start(ctx, cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()

	if cmd.ExitStatus() != 0 {
		return fmt.Errorf("Failed to copy file to shared drive: %s, %s, %d", stderr.String(), stdout.String(), cmd.ExitStatus())
	}

	// Read that copied file into a new file opened on host machine
	fsrc, err := os.Open(filepath.Join(c.HostDir, filepath.Base(src)))
	if err != nil {
		return err
	}
	defer fsrc.Close()
	defer os.Remove(fsrc.Name())

	_, err = io.Copy(dst, fsrc)
	if err != nil {
		return err
	}

	return nil
}
1314492
func (s *apidSnapshotManager) downloadSnapshot(isBoot bool, scopes []string, snapshot *common.Snapshot) {

	log.Debug("downloadSnapshot")

	snapshotUri, err := url.Parse(config.GetString(configSnapServerBaseURI))
	if err != nil {
		log.Panicf("bad url value for config %s: %s", snapshotUri, err)
	}

	snapshotUri.Path = path.Join(snapshotUri.Path, "snapshots")

	v := url.Values{}
	for _, scope := range scopes {
		v.Add("scope", scope)
	}
	snapshotUri.RawQuery = v.Encode()
	uri := snapshotUri.String()
	log.Infof("Snapshot Download: %s", uri)

	//pollWithBackoff only accepts function that accept a single quit channel
	//to accommodate functions which need more parameters, wrap them in closures
	attemptDownload := s.getAttemptDownloadClosure(isBoot, snapshot, uri)
	pollWithBackoff(s.quitChan, attemptDownload, handleSnapshotServerError)
}
597690
func (b *Bot) Download(file *File, localFilename string) error {
	reader, err := b.GetFile(file)
	if err != nil {
		return wrapSystem(err)
	}
	defer reader.Close()

	out, err := os.Create(localFilename)
	if err != nil {
		return wrapSystem(err)
	}
	defer out.Close()

	_, err = io.Copy(out, reader)
	if err != nil {
		return wrapSystem(err)
	}
	file.FileLocal = localFilename

	return nil
}
794197
func DownloadAssetAsync(asset Downloadable, out io.Writer, c AsyncDownloadChannel) {
	go func() {
		results := &AsyncDownloadResult{Asset: asset, Writer: out}
		size, err := DownloadAsset(asset, out)
		if err != nil {
			results.Error = err
		} else {
			results.Size = size
		}
		c <- results
	}()
}
651752
func (d Datastore) DownloadFile(ctx context.Context, path string, file string, param *soap.Download) error {
	u, p, err := d.downloadTicket(ctx, path, param)
	if err != nil {
		return err
	}
	return d.Client().DownloadFile(ctx, file, u, p)
}
627080
func (c *Client) Download(ctx context.Context, src string) (io.ReadCloser, int64, error) {
	vc := c.ProcessManager.Client()

	info, err := c.FileManager.InitiateFileTransferFromGuest(ctx, c.Authentication, src)
	if err != nil {
		return nil, 0, err
	}

	u, err := c.FileManager.TransferURL(ctx, info.Url)
	if err != nil {
		return nil, 0, err
	}

	p := soap.DefaultDownload

	f, n, err := vc.Download(ctx, u, &p)
	if err != nil {
		return nil, n, err
	}

	if strings.HasPrefix(src, "/archive:/") || isDir(src) {
		f = &archiveReader{ReadCloser: f} // look for the gzip trailer
	}

	return f, n, nil
}
1264567
func (d *downloader) DownloadFile(url string) (int64, string, error) {
	c := http.Client{
		CheckRedirect: func(r *http.Request, via []*http.Request) error {
			r.URL.Opaque = r.URL.Path

			//some redirect return '/' as url
			if strings.Trim(r.URL.Opaque, "/") != "" {
				url = r.URL.Opaque
			}

			return nil
		},
	}

	r, err := c.Get(url)

	if err != nil {
		return 0, "", err
	}
	defer r.Body.Close()

	if r.StatusCode == 200 {
		d.filename = getFilenameFromHeader(r.Header.Get("Content-Disposition"))

		if d.filename == "" {
			d.filename = getFilenameFromURL(url)
		}

		f, err := os.Create(filepath.Join(d.saveDir, d.filename))
		if err != nil {
			return 0, "", err
		}
		defer f.Close()

		size, err := io.Copy(f, r.Body)
		if err != nil {
			return 0, "", err
		}

		d.downloaded = true
		return size, d.filename, nil

	}
	return 0, "", fmt.Errorf("Error downloading file from %s", url)
}
840005
func (d *Downloader) DownloadInDirectory(ctx context.Context, dst string, paths ...string) error {
	return d.Download(ctx, paths, func(ctx context.Context, idx int, path string, r io.ReadCloser) (err error) {
		// Make sure to close the reader
		defer r.Close()

		// Make sure destination directory exists
		if err = os.MkdirAll(dst, 0700); err != nil {
			err = errors.Wrapf(err, "astihttp: mkdirall %s failed", dst)
			return
		}

		// Create destination file
		var f *os.File
		dst := filepath.Join(dst, filepath.Base(path))
		if f, err = os.Create(dst); err != nil {
			err = errors.Wrapf(err, "astihttp: creating %s failed", dst)
			return
		}
		defer f.Close()

		// Copy
		if _, err = astiio.Copy(ctx, r, f); err != nil {
			err = errors.Wrapf(err, "astihttp: copying content to %s failed", dst)
			return
		}
		return
	})
}
1165054
func (c *Context) Download(name string) (string, error) {
	deps := &contextDeps{
		APIClient: c.apiClient,
		name:      name,
		dataDir:   c.dataDir,
	}
	path, err := internal.ContextDownload(deps)
	if err != nil {
		return "", errors.Trace(err)
	}
	return path, nil
}
765814
func downloadPackage(ctx *cli.Context, n *doc.Node) (*doc.Node, []string, error) {

	// fmt.Println(n.VerString())
	log.Info("Downloading package: %s", n.VerString())
	downloadCache.Set(n.VerString())

	vendor := base.GetTempDir()
	defer os.RemoveAll(vendor)

	var (
		err     error
		imports []string
		srcPath string
	)

	// Check if only need to use VCS tools.
	vcs := doc.GetVcsName(n.InstallGopath)
	// If update, gopath and VCS tools set then use VCS tools to update the package.
	if ctx.Bool("update") && (ctx.Bool("gopath") || ctx.Bool("local")) && len(vcs) > 0 {
		if err = n.UpdateByVcs(vcs); err != nil {
			return nil, nil, fmt.Errorf("fail to update by VCS(%s): %v", n.ImportPath, err)
		}
		srcPath = n.InstallGopath
	} else {
		if !n.IsGetDepsOnly || !n.IsExist() {
			// Get revision value from local records.
			n.Revision = setting.LocalNodes.MustValue(n.RootPath, "value")
			if err = n.DownloadGopm(ctx); err != nil {
				errors.AppendError(errors.NewErrDownload(n.ImportPath + ": " + err.Error()))
				failCount++
				os.RemoveAll(n.InstallPath)
				return nil, nil, nil
			}
		}
		srcPath = n.InstallPath
	}

	if n.IsGetDeps {
		imports, err = getDepList(ctx, n.ImportPath, srcPath, vendor)
		if err != nil {
			return nil, nil, fmt.Errorf("fail to list imports(%s): %v", n.ImportPath, err)
		}
		if setting.Debug {
			log.Debug("New imports: %v", imports)
		}
	}
	return n, imports, err
}