// smartFetch the things that blobs point to, not just blobs.
324343
func smartFetch(ctx context.Context, src blob.Fetcher, targ string, br blob.Ref) error {
	rc, err := fetch(ctx, src, br)
	if err != nil {
		return err
	}
	rcc := types.NewOnceCloser(rc)
	defer rcc.Close()

	sniffer := index.NewBlobSniffer(br)
	_, err = io.CopyN(sniffer, rc, sniffSize)
	if err != nil && err != io.EOF {
		return err
	}

	sniffer.Parse()
	b, ok := sniffer.SchemaBlob()

	if !ok {
		if *flagVerbose {
			log.Printf("Fetching opaque data %v into %q", br, targ)
		}

		// opaque data - put it in a file
		f, err := os.Create(targ)
		if err != nil {
			return fmt.Errorf("opaque: %v", err)
		}
		defer f.Close()
		body, _ := sniffer.Body()
		r := io.MultiReader(bytes.NewReader(body), rc)
		_, err = io.Copy(f, r)
		return err
	}
	rcc.Close()

	switch b.Type() {
	case "directory":
		dir := filepath.Join(targ, b.FileName())
		if *flagVerbose {
			log.Printf("Fetching directory %v into %s", br, dir)
		}
		if err := os.MkdirAll(dir, b.FileMode()); err != nil {
			return err
		}
		if err := setFileMeta(dir, b); err != nil {
			log.Print(err)
		}
		entries, ok := b.DirectoryEntries()
		if !ok {
			return fmt.Errorf("bad entries blobref in dir %v", b.BlobRef())
		}
		return smartFetch(ctx, src, dir, entries)
	case "static-set":
		if *flagVerbose {
			log.Printf("Fetching directory entries %v into %s", br, targ)
		}

		// directory entries
		const numWorkers = 10
		type work struct {
			br   blob.Ref
			errc chan<- error
		}
		members := b.StaticSetMembers()
		workc := make(chan work, len(members))
		defer close(workc)
		for i := 0; i < numWorkers; i++ {
			go func() {
				for wi := range workc {
					wi.errc <- smartFetch(ctx, src, targ, wi.br)
				}
			}()
		}
		var errcs []<-chan error
		for _, mref := range members {
			errc := make(chan error, 1)
			errcs = append(errcs, errc)
			workc <- work{mref, errc}
		}
		for _, errc := range errcs {
			if err := <-errc; err != nil {
				return err
			}
		}
		return nil
	case "file":
		fr, err := schema.NewFileReader(ctx, src, br)
		if err != nil {
			return fmt.Errorf("NewFileReader: %v", err)
		}
		fr.LoadAllChunks()
		defer fr.Close()

		name := filepath.Join(targ, b.FileName())

		if fi, err := os.Stat(name); err == nil && fi.Size() == fr.Size() {
			if *flagVerbose {
				log.Printf("Skipping %s; already exists.", name)
			}
			return nil
		}

		if *flagVerbose {
			log.Printf("Writing %s to %s ...", br, name)
		}

		f, err := os.Create(name)
		if err != nil {
			return fmt.Errorf("file type: %v", err)
		}
		defer f.Close()
		if _, err := io.Copy(f, fr); err != nil {
			return fmt.Errorf("Copying %s to %s: %v", br, name, err)
		}
		if err := setFileMeta(name, b); err != nil {
			log.Print(err)
		}
		return nil
	case "symlink":
		if *flagSkipIrregular {
			return nil
		}
		sf, ok := b.AsStaticFile()
		if !ok {
			return errors.New("blob is not a static file")
		}
		sl, ok := sf.AsStaticSymlink()
		if !ok {
			return errors.New("blob is not a symlink")
		}
		name := filepath.Join(targ, sl.FileName())
		if _, err := os.Lstat(name); err == nil {
			if *flagVerbose {
				log.Printf("Skipping creating symbolic link %s: A file with that name exists", name)
			}
			return nil
		}
		target := sl.SymlinkTargetString()
		if target == "" {
			return errors.New("symlink without target")
		}

		// On Windows, os.Symlink isn't yet implemented as of Go 1.3.
		// See https://code.google.com/p/go/issues/detail?id=5750
		err := os.Symlink(target, name)
		// We won't call setFileMeta for a symlink because:
		// the permissions of a symlink do not matter and Go's
		// os.Chtimes always dereferences (does not act on the
		// symlink but its target).
		return err
	case "fifo":
		if *flagSkipIrregular {
			return nil
		}
		name := filepath.Join(targ, b.FileName())

		sf, ok := b.AsStaticFile()
		if !ok {
			return errors.New("blob is not a static file")
		}
		_, ok = sf.AsStaticFIFO()
		if !ok {
			return errors.New("blob is not a static FIFO")
		}

		if _, err := os.Lstat(name); err == nil {
			log.Printf("Skipping FIFO %s: A file with that name already exists", name)
			return nil
		}

		err = osutil.Mkfifo(name, 0600)
		if err == osutil.ErrNotSupported {
			log.Printf("Skipping FIFO %s: Unsupported filetype", name)
			return nil
		}
		if err != nil {
			return fmt.Errorf("%s: osutil.Mkfifo(): %v", name, err)
		}

		if err := setFileMeta(name, b); err != nil {
			log.Print(err)
		}

		return nil

	case "socket":
		if *flagSkipIrregular {
			return nil
		}
		name := filepath.Join(targ, b.FileName())

		sf, ok := b.AsStaticFile()
		if !ok {
			return errors.New("blob is not a static file")
		}
		_, ok = sf.AsStaticSocket()
		if !ok {
			return errors.New("blob is not a static socket")
		}

		if _, err := os.Lstat(name); err == nil {
			log.Printf("Skipping socket %s: A file with that name already exists", name)
			return nil
		}

		err = osutil.Mksocket(name)
		if err == osutil.ErrNotSupported {
			log.Printf("Skipping socket %s: Unsupported filetype", name)
			return nil
		}
		if err != nil {
			return fmt.Errorf("%s: %v", name, err)
		}

		if err := setFileMeta(name, b); err != nil {
			log.Print(err)
		}

		return nil

	default:
		return errors.New("unknown blob type: " + b.Type())
	}
	panic("unreachable")
}
500163
func (s *storage) fetch(ctx context.Context, br blob.Ref, offset, length int64) (rc io.ReadCloser, size uint32, err error) {
	meta, err := s.meta(br)
	if err != nil {
		return nil, 0, err
	}

	if meta.file >= len(s.fds) {
		return nil, 0, fmt.Errorf("diskpacked: attempt to fetch blob from out of range pack file %d > %d", meta.file, len(s.fds))
	}
	rac := s.fds[meta.file]
	var rs io.ReadSeeker
	if length == -1 {
		// normal Fetch mode
		rs = io.NewSectionReader(rac, meta.offset, int64(meta.size))
	} else {
		if offset > int64(meta.size) {
			return nil, 0, blob.ErrOutOfRangeOffsetSubFetch
		} else if offset+length > int64(meta.size) {
			length = int64(meta.size) - offset
		}
		rs = io.NewSectionReader(rac, meta.offset+offset, length)
	}
	fn := rac.Name()
	// Ensure entry is in map.
	readVar.Add(fn, 0)
	if v, ok := readVar.Get(fn).(*expvar.Int); ok {
		rs = readerutil.NewStatsReadSeeker(v, rs)
	}
	readTotVar.Add(s.root, 0)
	if v, ok := readTotVar.Get(s.root).(*expvar.Int); ok {
		rs = readerutil.NewStatsReadSeeker(v, rs)
	}
	rsc := struct {
		io.ReadSeeker
		io.Closer
	}{
		rs,
		types.NopCloser,
	}
	return rsc, meta.size, nil
}
506215
func (c container) Blobs() ([]Blob, error) {
	//TODO(axw) handle pagination.
	resp, err := c.Container.ListBlobs(storage.ListBlobsParameters{})
	if err != nil {
		return nil, errors.Trace(err)
	}
	blobs := make([]Blob, len(resp.Blobs))
	for i := range blobs {
		blobs[i] = blob{&resp.Blobs[i]}
	}
	return blobs, nil
}
293699
func (p *BlockPrefetchClient) retarget(hash string) {
	if hash == "" {
		return
	}

	if p.next.headerResult != nil && hash == p.next.headerResult.Hash {
		return
	}
	// Do not go backwards
	if p.current.headerResult != nil && hash == p.current.headerResult.Hash {
		return
	}

	h, err := chainhash.NewHashFromStr(hash)
	if err != nil {
		log.Errorf("Invalid hash %s: %v", hash, err)
		return
	}

	p.retargetChan <- h
}
1148578
func (sto *unionStorage) Fetch(ctx context.Context, b blob.Ref) (file io.ReadCloser, size uint32, err error) {
	type result struct {
		file io.ReadCloser
		size uint32
		err  error
	}
	results := make(chan result, len(sto.subsets))
	var wg sync.WaitGroup
	for _, bs := range sto.subsets {
		bs := bs
		wg.Add(1)
		go func() {
			defer wg.Done()
			var res result
			res.file, res.size, res.err = bs.Fetch(ctx, b)
			results <- res
		}()
	}
	go func() {
		wg.Wait()
		close(results)
	}()
	var firstErr error
	var firstRes result
	for r := range results {
		if r.err != nil {
			if firstErr == nil {
				firstErr = r.err
			}
			continue
		}
		if firstRes.file != nil {
			if r.file != nil {
				r.file.Close() // don't need, we already have a successful Fetch
			}
			continue
		}

		firstRes = r
	}
	if firstRes.file != nil {
		return firstRes.file, firstRes.size, nil
	}
	return nil, 0, firstErr
}
152159
func (qre *QueryExecutor) txFetch(conn *TxConnection, parsedQuery *sqlparser.ParsedQuery, bindVars map[string]*querypb.BindVariable, extras map[string]sqlparser.Encodable, buildStreamComment string, wantfields, record bool) (*sqltypes.Result, error) {
	sql, _, err := qre.generateFinalSQL(parsedQuery, bindVars, extras, buildStreamComment)
	if err != nil {
		return nil, err
	}
	qr, err := qre.execSQL(conn, sql, wantfields)
	if err != nil {
		return nil, err
	}
	// Only record successful queries.
	if record {
		conn.RecordQuery(sql)
	}
	return qr, nil
}
96638
func (s bulkFetchDataSorter) Less(i, j int) bool {
	if s[i].blockFileNum < s[j].blockFileNum {
		return true
	}
	if s[i].blockFileNum > s[j].blockFileNum {
		return false
	}

	return s[i].fileOffset < s[j].fileOffset
}
960893
func (o *httpOps) AscRemoteFetcher() *remoteAscFetcher {
	ensureLogger(o.Debug)
	f := func(u *url.URL, file *os.File) error {
		switch u.Scheme {
		case "http", "https":
		default:
			return fmt.Errorf("invalid signature location: expected %q scheme, got %q", "http(s)", u.Scheme)
		}
		session := o.getSession(u, file, "signature", "")
		dl := o.getDownloader(session)
		err := dl.Download(u, file)
		if err != nil {
			return err
		}
		if session.Cd.UseCached {
			return fmt.Errorf("unexpected cache reuse request for signature %q", u.String())
		}
		return nil
	}
	return &remoteAscFetcher{
		F: f,
		S: o.S,
	}
}
162009
func (p *BlockPrefetchClient) Misses() uint64 {
	p.Lock()
	defer p.Unlock()
	return p.misses
}
39339
func (bot *ExchangeBot) fetchFromCache(chartID string) (data []byte, bestVersion int, hit bool) {
	bot.mtx.RLock()
	defer bot.mtx.RUnlock()
	bestVersion = bot.cachedChartVersion(chartID)
	cache, found := bot.versionedCharts[chartID]
	if found && cache.dataID == bestVersion {
		data = cache.chart
		hit = true
	}
	return
}
303919
func (bra BlockRequestAction) Prefetch(block data.Block) bool {
	// When syncing, always prefetch child blocks of an indirect
	// block, since it makes no sense to sync just part of a
	// multi-block object.
	if block.IsIndirect() && bra.Sync() {
		return true
	}
	return bra.prefetch()
}
1105223
func (s Secret) Blobs() []NamedBlob {
	out := make([]NamedBlob, 0, 1+len(s.Previous))
	out = append(out, s.Current)
	out = append(out, s.Previous...)
	return out
}
471997
func (ds *Storage) fetch(ctx context.Context, br blob.Ref, offset, length int64) (rc io.ReadCloser, size uint32, err error) {
	// TODO: use ctx, if the os package ever supports that.
	fileName := ds.blobPath(br)
	stat, err := ds.fs.Stat(fileName)
	if os.IsNotExist(err) {
		return nil, 0, os.ErrNotExist
	}
	size = u32(stat.Size())
	file, err := ds.fs.Open(fileName)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.ErrNotExist
		}
		return nil, 0, err
	}
	// normal Fetch
	if length < 0 && offset == 0 {
		return file, size, nil
	}
	// SubFetch:
	if offset < 0 || offset > stat.Size() {
		if offset < 0 {
			return nil, 0, blob.ErrNegativeSubFetch
		}
		return nil, 0, blob.ErrOutOfRangeOffsetSubFetch
	}
	if offset != 0 {
		if at, err := file.Seek(offset, io.SeekStart); err != nil || at != offset {
			file.Close()
			return nil, 0, fmt.Errorf("localdisk: error seeking to %d: got %v, %v", offset, at, err)
		}
	}
	return struct {
		io.Reader
		io.Closer
	}{
		Reader: io.LimitReader(file, length),
		Closer: file,
	}, 0 /* unused */, nil
}
1225630
func MergePatch(docData, patchData []byte) ([]byte, error) {
	return doMergePatch(docData, patchData, false)
}
981644
func (qre *QueryExecutor) streamFetch(conn *connpool.DBConn, parsedQuery *sqlparser.ParsedQuery, bindVars map[string]*querypb.BindVariable, buildStreamComment string, callback func(*sqltypes.Result) error) error {
	sql, _, err := qre.generateFinalSQL(parsedQuery, bindVars, nil, buildStreamComment)
	if err != nil {
		return err
	}
	return qre.execStreamSQL(conn, sql, callback)
}
1127382
func (s *storage) fetchMeta(ctx context.Context, b blob.Ref) (plainSize uint32, encBR blob.Ref, err error) {
	v, err := s.index.Get(b.String())
	if err == sorted.ErrNotFound {
		err = os.ErrNotExist
	}
	if err != nil {
		return 0, blob.Ref{}, err
	}
	return unpackIndexEntry(v)
}
1281386
func (q *Queryx) Prefetch(p float64) *Queryx {
	q.Query.Prefetch(p)
	return q
}