// readTombstoneV4 reads the fourth version of tombstone files that are capable
// of storing multiple v3 files appended together.
968780
func (t *Tombstoner) readTombstoneV4(f *os.File, fn func(t Tombstone) error) error {
	// Skip header, already checked earlier
	if t.lastAppliedOffset != 0 {
		if _, err := f.Seek(t.lastAppliedOffset, io.SeekStart); err != nil {
			return err
		}
	} else {
		if _, err := f.Seek(headerSize, io.SeekStart); err != nil {
			return err
		}
	}

	var (
		prefix   bool
		min, max int64
		key      []byte
		kmask    = 0xff000000 // Mask for non key-length bits
	)

	br := bufio.NewReaderSize(f, 64*1024)
	gr, err := gzip.NewReader(br)
	if err == io.EOF {
		return nil
	} else if err != nil {
		return err
	}
	defer gr.Close()

	b := make([]byte, 4096)
	for {
		gr.Multistream(false)
		if err := func() error {
			for {
				if _, err = io.ReadFull(gr, b[:4]); err == io.EOF || err == io.ErrUnexpectedEOF {
					return nil
				} else if err != nil {
					return err
				}

				keyLen := int(binary.BigEndian.Uint32(b[:4]))
				prefix = keyLen>>31 == 1 // Prefix is set according to whether the highest bit is set.

				// Remove 8 MSB to get correct length.
				keyLen &^= kmask

				if keyLen+16 > len(b) {
					b = make([]byte, keyLen+16)
				}

				if _, err := io.ReadFull(gr, b[:keyLen]); err != nil {
					return err
				}

				// Copy the key since b is re-used
				key = b[:keyLen]

				minBuf := b[keyLen : keyLen+8]
				maxBuf := b[keyLen+8 : keyLen+16]
				if _, err := io.ReadFull(gr, minBuf); err != nil {
					return err
				}

				min = int64(binary.BigEndian.Uint64(minBuf))
				if _, err := io.ReadFull(gr, maxBuf); err != nil {
					return err
				}

				max = int64(binary.BigEndian.Uint64(maxBuf))
				if err := fn(Tombstone{
					Key:    key,
					Min:    min,
					Max:    max,
					Prefix: prefix,
				}); err != nil {
					return err
				}
			}
		}(); err != nil {
			return err
		}

		for _, t := range t.tombstones {
			if err := fn(t); err != nil {
				return err
			}
		}

		err = gr.Reset(br)
		if err == io.EOF {
			break
		}
	}

	// Save the position of tombstone file so we don't re-apply the same set again if there are
	// more deletes.
	pos, err := f.Seek(0, io.SeekCurrent)
	if err != nil {
		return err
	}
	t.lastAppliedOffset = pos
	return nil
}
78767
func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error) {
LOOP:
	// No matching blocks to decode
	if len(c.current) == 0 {
		return nil, nil
	}

	// First block is the oldest block containing the points we're searching for.
	first := c.current[0]
	*buf = (*buf)[:0]
	var values BooleanValues
	values, err := first.r.ReadBooleanBlockAt(&first.entry, buf)
	if err != nil {
		return nil, err
	}
	if c.col != nil {
		c.col.GetCounter(booleanBlocksDecodedCounter).Add(1)
		c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(first.entry.Size))
	}

	// Remove values we already read
	values = values.Exclude(first.readMin, first.readMax)

	// Remove any tombstones
	c.trbuf = first.r.TombstoneRange(c.key, c.trbuf[:0])
	values = excludeTombstonesBooleanValues(c.trbuf, values)
	// If there are no values in this first block (all tombstoned or previously read) and
	// we have more potential blocks too search.  Try again.
	if values.Len() == 0 && len(c.current) > 0 {
		c.current = c.current[1:]
		goto LOOP
	}

	// Only one block with this key and time range so return it
	if len(c.current) == 1 {
		if values.Len() > 0 {
			first.markRead(values.MinTime(), values.MaxTime())
		}
		return values, nil
	}

	// Use the current block time range as our overlapping window
	minT, maxT := first.readMin, first.readMax
	if values.Len() > 0 {
		minT, maxT = values.MinTime(), values.MaxTime()
	}
	if c.ascending {
		// Blocks are ordered by generation, we may have values in the past in later blocks, if so,
		// expand the window to include the min time range to ensure values are returned in ascending
		// order
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			if cur.entry.MinTime < minT && !cur.read() {
				minT = cur.entry.MinTime
			}
		}

		// Find first block that overlaps our window
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			if cur.entry.OverlapsTimeRange(minT, maxT) && !cur.read() {
				// Shrink our window so it's the intersection of the first overlapping block and the
				// first block.  We do this to minimize the region that overlaps and needs to
				// be merged.
				if cur.entry.MaxTime > maxT {
					maxT = cur.entry.MaxTime
				}
				values = values.Include(minT, maxT)
				break
			}
		}

		// Search the remaining blocks that overlap our window and append their values so we can
		// merge them.
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			// Skip this block if it doesn't contain points we looking for or they have already been read
			if !cur.entry.OverlapsTimeRange(minT, maxT) || cur.read() {
				cur.markRead(minT, maxT)
				continue
			}

			var a []BooleanValue
			var v BooleanValues
			v, err := cur.r.ReadBooleanBlockAt(&cur.entry, &a)
			if err != nil {
				return nil, err
			}
			if c.col != nil {
				c.col.GetCounter(booleanBlocksDecodedCounter).Add(1)
				c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(cur.entry.Size))
			}

			c.trbuf = cur.r.TombstoneRange(c.key, c.trbuf[:0])
			// Remove any tombstoned values
			v = excludeTombstonesBooleanValues(c.trbuf, v)

			// Remove values we already read
			v = v.Exclude(cur.readMin, cur.readMax)

			if v.Len() > 0 {
				// Only use values in the overlapping window
				v = v.Include(minT, maxT)
				// Merge the remaining values with the existing
				values = values.Merge(v)
			}
			cur.markRead(minT, maxT)
		}

	} else {
		// Blocks are ordered by generation, we may have values in the past in later blocks, if so,
		// expand the window to include the max time range to ensure values are returned in descending
		// order
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			if cur.entry.MaxTime > maxT && !cur.read() {
				maxT = cur.entry.MaxTime
			}
		}

		// Find first block that overlaps our window
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			if cur.entry.OverlapsTimeRange(minT, maxT) && !cur.read() {
				// Shrink our window so it's the intersection of the first overlapping block and the
				// first block.  We do this to minimize the region that overlaps and needs to
				// be merged.
				if cur.entry.MinTime < minT {
					minT = cur.entry.MinTime
				}
				values = values.Include(minT, maxT)
				break
			}
		}

		// Search the remaining blocks that overlap our window and append their values so we can
		// merge them.
		for i := 1; i < len(c.current); i++ {
			cur := c.current[i]
			// Skip this block if it doesn't contain points we looking for or they have already been read
			if !cur.entry.OverlapsTimeRange(minT, maxT) || cur.read() {
				cur.markRead(minT, maxT)
				continue
			}

			var a []BooleanValue
			var v BooleanValues
			v, err := cur.r.ReadBooleanBlockAt(&cur.entry, &a)
			if err != nil {
				return nil, err
			}
			if c.col != nil {
				c.col.GetCounter(booleanBlocksDecodedCounter).Add(1)
				c.col.GetCounter(booleanBlocksSizeCounter).Add(int64(cur.entry.Size))
			}
			c.trbuf = cur.r.TombstoneRange(c.key, c.trbuf[:0])
			// Remove any tombstoned values
			v = excludeTombstonesBooleanValues(c.trbuf, v)

			// Remove values we already read
			v = v.Exclude(cur.readMin, cur.readMax)

			// If the block we decoded should have all of it's values included, mark it as read so we
			// don't use it again.
			if v.Len() > 0 {
				v = v.Include(minT, maxT)
				// Merge the remaining values with the existing
				values = v.Merge(values)
			}
			cur.markRead(minT, maxT)
		}
	}

	first.markRead(minT, maxT)

	return values, err
}
894101
func (p *TBinaryProtocol) ReadMessageBegin() (name string, typeId TMessageType, seqId int32, err error) {
	size, e := p.ReadI32()
	if e != nil {
		return "", typeId, 0, NewTProtocolException(e)
	}
	if size < 0 {
		typeId = TMessageType(size & 0x0ff)
		version := int64(int64(size) & VERSION_MASK)
		if version != VERSION_1 {
			return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Bad version in ReadMessageBegin"))
		}
		name, e = p.ReadString()
		if e != nil {
			return name, typeId, seqId, NewTProtocolException(e)
		}
		seqId, e = p.ReadI32()
		if e != nil {
			return name, typeId, seqId, NewTProtocolException(e)
		}
		return name, typeId, seqId, nil
	}
	if p.strictRead {
		return name, typeId, seqId, NewTProtocolExceptionWithType(BAD_VERSION, fmt.Errorf("Missing version in ReadMessageBegin"))
	}
	name, e2 := p.readStringBody(int(size))
	if e2 != nil {
		return name, typeId, seqId, e2
	}
	b, e3 := p.ReadByte()
	if e3 != nil {
		return name, typeId, seqId, e3
	}
	typeId = TMessageType(b)
	seqId, e4 := p.ReadI32()
	if e4 != nil {
		return name, typeId, seqId, e4
	}
	return name, typeId, seqId, nil
}
418570
func readBufChain(arena *slab.Arena, maxBufSize int, r io.ReaderAt, offset int64,
	valLength uint32) ([]byte, error) {
	n := int(valLength)
	if n > maxBufSize {
		n = maxBufSize
	}
	b := arena.Alloc(n)
	_, err := r.ReadAt(b, offset)
	if err != nil {
		arena.DecRef(b)
		return nil, err
	}
	remaining := valLength - uint32(n)
	if remaining > 0 {
		next, err := readBufChain(arena, maxBufSize, r, offset + int64(n), remaining)
		if err != nil {
			arena.DecRef(b)
			return nil, err
		}
		arena.SetNext(b, next)
	}
	return b, nil
}
738868
func ReadRelease(f string) (map[string]string, error) {
	stream, err := ioutil.ReadFile(osFile)
	if err != nil {
		return nil, err
	}
	values := make(map[string]string)
	releaseDetail := strings.Split(string(stream), " ")
	values["ID"] = releaseDetail[0]
	values["VERSION_ID"] = releaseDetail[2]
	id, ok := values["ID"]
	if !ok {
		return nil, errors.New("OS release file is missing ID")
	}
	if _, ok := values["VERSION_ID"]; !ok {
		values["VERSION_ID"], ok = defaultVersionIDs[id]
		if !ok {
			return nil, errors.New("OS release file is missing VERSION_ID")
		}
	}
	return values, nil
}
440094
func (o *Generator) ReadV4(ids []UUID) {
	for i := range ids {
		id := UUID{}
		o.v4(&id)
		ids[i] = id
		continue
	}
	return
}
812110
func (c *ClientsetCAStore) readCAKeypairs(id string) (*keyset, error) {
	c.mutex.Lock()
	defer c.mutex.Unlock()

	cached := c.cachedCaKeysets[id]
	if cached != nil {
		return cached, nil
	}

	keyset, err := c.loadKeyset(id)
	if err != nil {
		return nil, err
	}

	if keyset == nil {
		return nil, nil
	}
	c.cachedCaKeysets[id] = keyset
	return keyset, nil
}
1130436
func (db *DB) CleanTombstones() (err error) {
	db.cmtx.Lock()
	defer db.cmtx.Unlock()

	start := time.Now()
	defer db.metrics.tombCleanTimer.Observe(time.Since(start).Seconds())

	newUIDs := []ulid.ULID{}
	defer func() {
		// If any error is caused, we need to delete all the new directory created.
		if err != nil {
			for _, uid := range newUIDs {
				dir := filepath.Join(db.Dir(), uid.String())
				if err := os.RemoveAll(dir); err != nil {
					level.Error(db.logger).Log("msg", "failed to delete block after failed `CleanTombstones`", "dir", dir, "err", err)
				}
			}
		}
	}()

	db.mtx.RLock()
	blocks := db.blocks[:]
	db.mtx.RUnlock()

	for _, b := range blocks {
		if uid, er := b.CleanTombstones(db.Dir(), db.compactor); er != nil {
			err = errors.Wrapf(er, "clean tombstones: %s", b.Dir())
			return err
		} else if uid != nil { // New block was created.
			newUIDs = append(newUIDs, *uid)
		}
	}
	return errors.Wrap(db.reload(), "reload blocks")
}
175442
func (t *TSMReader) ReadAll(key []byte) ([]Value, error) {
	t.mu.RLock()
	v, err := t.accessor.readAll(key)
	t.mu.RUnlock()
	return v, err
}
747010
func (d *ConsulStateDriver) ReadAll(baseKey string) ([][]byte, error) {
	baseKey = processKey(baseKey)

	var err error
	var kvs api.KVPairs

	for i := 0; i < maxConsulRetries; i++ {
		kvs, _, err = d.Client.KV().List(baseKey, nil)
		if err != nil {
			if api.IsServerError(err) || strings.Contains(err.Error(), "EOF") || strings.Contains(err.Error(), "connection refused") {
				time.Sleep(time.Second)
				continue
			}

			return [][]byte{}, err
		}

		// err == nil
		if kvs == nil {
			// Consul returns success and a nil kv when a key is not found,
			// translate it to 'key not found' error
			return nil, core.Errorf("key not found")
		}

		values := [][]byte{}
		for _, kv := range kvs {
			values = append(values, kv.Value)
		}
		return values, nil

	}

	return [][]byte{}, err
}
1285272
func (t *Tombstoner) HasTombstones() bool {
	files := t.TombstoneFiles()
	t.mu.RLock()
	n := len(t.tombstones)
	t.mu.RUnlock()

	return len(files) > 0 && files[0].Size > 0 || n > 0
}
532155
func (m *mmapAccessor) readAll(key []byte) ([]Value, error) {
	m.incAccess()

	blocks, err := m.index.ReadEntries(key, nil)
	if len(blocks) == 0 || err != nil {
		return nil, err
	}

	tombstones := m.index.TombstoneRange(key, nil)

	m.mu.RLock()
	defer m.mu.RUnlock()

	var temp []Value
	var values []Value
	for _, block := range blocks {
		var skip bool
		for _, t := range tombstones {
			// Should we skip this block because it contains points that have been deleted
			if t.Min <= block.MinTime && t.Max >= block.MaxTime {
				skip = true
				break
			}
		}

		if skip {
			continue
		}
		//TODO: Validate checksum
		temp = temp[:0]
		// The +4 is the 4 byte checksum length
		temp, err = DecodeBlock(m.b[block.Offset+4:block.Offset+int64(block.Size)], temp)
		if err != nil {
			return nil, err
		}

		// Filter out any values that were deleted
		for _, t := range tombstones {
			temp = Values(temp).Exclude(t.Min, t.Max)
		}

		values = append(values, temp...)
	}

	return values, nil
}
1110186
func GetStdlibFuncs() map[string]function.Function {
	return map[string]function.Function{
		"abs":        stdlib.AbsoluteFunc,
		"coalesce":   stdlib.CoalesceFunc,
		"concat":     stdlib.ConcatFunc,
		"hasindex":   stdlib.HasIndexFunc,
		"int":        stdlib.IntFunc,
		"jsondecode": stdlib.JSONDecodeFunc,
		"jsonencode": stdlib.JSONEncodeFunc,
		"length":     stdlib.LengthFunc,
		"lower":      stdlib.LowerFunc,
		"max":        stdlib.MaxFunc,
		"min":        stdlib.MinFunc,
		"reverse":    stdlib.ReverseFunc,
		"strlen":     stdlib.StrlenFunc,
		"substr":     stdlib.SubstrFunc,
		"upper":      stdlib.UpperFunc,
	}
}
408450
func (a *Aggregator) ReadAll() [][]*rfc5424.Message {
	// TODO(benburkert): restructure Aggregator & ring.Buffer to avoid nested locks
	a.bmu.Lock()
	defer a.bmu.Unlock()

	buffers := make([][]*rfc5424.Message, 0, len(a.buffers))
	for _, buf := range a.buffers {
		buffers = append(buffers, buf.Read())
	}

	return buffers
}
119924
func ReadAsByteOffs(fp sophie.FsPath) (buffer bytesp.Slice, keyOffs, keyEnds, valOffs, valEnds villa.IntSlice, err error) {
	fi, err := fp.Stat()
	if err != nil {
		return nil, nil, nil, nil, nil, errorsp.WithStacks(err)
	}

	reader, err := fp.Open()
	if err != nil {
		return nil, nil, nil, nil, nil, errorsp.WithStacks(err)
	}
	defer reader.Close()

	buffer = make(bytesp.Slice, fi.Size())
	if n, err := reader.Read(buffer); n != len(buffer) || err != nil {
		if err != nil {
			return nil, nil, nil, nil, nil, errorsp.WithStacksAndMessage(err, "n = %d", n)
		}
		return nil, nil, nil, nil, nil, errorsp.NewWithStacks("Expected %d bytes, but only read %d bytes", len(buffer), n)
	}
	buf := countReadCloser(bytesp.NewPSlice(buffer))
	for buf.Pos < int64(len(buffer)) {
		var l sophie.VInt
		if err := (&l).ReadFrom(buf, -1); err != nil {
			return nil, nil, nil, nil, nil, errorsp.WithStacksAndMessage(sophie.ErrBadFormat, "failed to read key-lenth: %v", err)
		}
		keyOffs = append(keyOffs, int(buf.Pos))
		if _, err := buf.Skip(int64(l)); err != nil {
			return nil, nil, nil, nil, nil, errorsp.WithStacksAndMessage(sophie.ErrBadFormat, "failed to skip key: %v", err)
		}
		keyEnds = append(keyEnds, int(buf.Pos))
		if err := (&l).ReadFrom(buf, -1); err != nil {
			return nil, nil, nil, nil, nil, errorsp.WithStacksAndMessage(sophie.ErrBadFormat, "failed to read value-lenth: %v", err)
		}
		valOffs = append(valOffs, int(buf.Pos))
		if _, err := buf.Skip(int64(l)); err != nil {
			return nil, nil, nil, nil, nil, errorsp.WithStacksAndMessage(sophie.ErrBadFormat, "failed to read value: %v", err)
		}
		valEnds = append(valEnds, int(buf.Pos))
	}
	return
}
1003453
func (c *Conn) ReadBytes(delim byte) ([]byte, error) {
	var line []byte
	for {
		b, err := c.ReadByte()
		if err != nil {
			return nil, err
		}
		line = append(line, b)
		if b == delim {
			break
		}
	}
	return line, nil
}
412419
func (p *PopOp) makeTombstonesEntity() *tombstonesEntity {
	p.entity.Tombstones = p.entity.Tombstones[:0]
	for id, ts := range p.tombs {
		p.entity.Tombstones = append(p.entity.Tombstones, struct {
			ID         string
			Tombstoned time.Time
		}{id, ts})
	}
	return p.entity
}