// processShard is the tumble backend endpoint. This accepts a shard number
// which is expected to be < GlobalConfig.NumShards.
965730
func processShard(c context.Context, cfg *Config, timestamp time.Time, shard uint64, loop bool) error {

	logging.Fields{
		"shard": shard,
	}.Infof(c, "Processing tumble shard.")

	q := processShardQuery(c, cfg, shard)
	if q == nil {
		logging.Warningf(c, "dead shard, quitting")
		return nil
	}

	// Calculate our end itme. If we're not looping or we have a <= 0 duration,
	// we will perform a single loop.
	var endTime time.Time
	if cfg.ProcessLoopDuration > 0 {
		endTime = clock.Now(c).Add(time.Duration(cfg.ProcessLoopDuration))
		logging.Debugf(c, "Process loop is configured to exit after [%s] at %s",
			cfg.ProcessLoopDuration.String(), endTime)
	}

	// Lock around the shard that we are trying to modify.
	//
	// Since memcache is namespaced, we don't need to include the namespace in our
	// lock name.
	task := makeProcessTask(timestamp, endTime, shard, loop)
	lockKey := fmt.Sprintf("%s.%d.lock", baseName, shard)
	clientID := fmt.Sprintf("%d_%d_%s", timestamp.Unix(), shard, info.RequestID(c))

	err := memlock.TryWithLock(c, lockKey, clientID, func(c context.Context) error {
		return task.process(c, cfg, q)
	})
	if err == memlock.ErrFailedToLock {
		logging.Infof(c, "Couldn't obtain lock (giving up): %s", err)
		return nil
	}
	return err
}
1069660
func (b Backend) ProcessEvents(events ...*auditinternal.Event) bool {
	for _, event := range events {
		if event == nil {
			continue
		}
		attr, err := ev.NewAttributes(event)
		if err != nil {
			audit.HandlePluginError(PluginName, err, event)
			continue
		}
		level, stages := b.policyChecker.LevelAndStages(attr)
		if level == auditinternal.LevelNone {
			continue
		}
		// make shallow copy before modifying to satisfy interface definition
		ev := *event
		e, err := policy.EnforcePolicy(&ev, level, stages)
		if err != nil {
			audit.HandlePluginError(PluginName, err, event)
			continue
		}
		if e == nil {
			continue
		}
		b.delegateBackend.ProcessEvents(e)
	}
	// Returning true regardless of results, since dynamic audit backends
	// can never cause apiserver request to fail.
	return true
}
193711
func (b *backgroundFlusher) Process() {
	b.logger.outputMutex.RLock()
	defer b.logger.outputMutex.RUnlock()

	for _, o := range b.logger.outputs {
		o.OutputWrapper.Output.Flush()
	}
	if b.updateChan != nil {
		close(b.updateChan)
	}
}
550179
func (c *Client) ProcessKubeCSR(req KubeCSR) (*KubeCSRResponse, error) {
	if err := req.CheckAndSetDefaults(); err != nil {
		return nil, trace.Wrap(err)
	}
	out, err := c.PostJSON(c.Endpoint("kube", "csr"), req)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	var re KubeCSRResponse
	if err := json.Unmarshal(out.Bytes(), &re); err != nil {
		return nil, trace.Wrap(err)
	}
	return &re, nil
}
691932
func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) {
	if !validHeaderAcceptJSON(r.Header) {
		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
		return
	}
	if err := json.NewEncoder(w).Encode(getShardsMaxResponse{
		Standard: h.api.MaxShards(r.Context()),
	}); err != nil {
		h.logger.Printf("write shards-max response error: %s", err)
	}
}
671644
func ParseShard(blob []byte) (Shard, error) {
	var sorted []string
	dec := gob.NewDecoder(bytes.NewReader(blob))
	if err := dec.Decode(&sorted); err != nil {
		return nil, err
	}
	shard := make(Shard, len(sorted))
	for _, blob := range sorted {
		shard[blob] = struct{}{}
	}
	return shard, nil
}
960989
func (api *API) ShardNodes(ctx context.Context, indexName string, shard uint64) ([]*Node, error) {
	span, _ := tracing.StartSpanFromContext(ctx, "API.ShardNodes")
	defer span.Finish()

	if err := api.validate(apiShardNodes); err != nil {
		return nil, errors.Wrap(err, "validating api method")
	}

	return api.cluster.shardNodes(indexName, shard), nil
}
530661
func Consume(shardID string, streamName string, region string, shardIteratorType string, eventChannel chan []byte) {
	svc := kinesis.New(session.New(), &aws.Config{Region: aws.String(region)})
	getShardIteratorParams := &kinesis.GetShardIteratorInput{
		ShardId:           aws.String(shardID),           // Required
		ShardIteratorType: aws.String(shardIteratorType), // Required
		StreamName:        aws.String(streamName),        // Required
	}
	shardIterator, err := svc.GetShardIterator(getShardIteratorParams)
	if err != nil {
		log.Panic(err)
	}

	nextShardIterator := shardIterator.ShardIterator
	for {
		getRecordsParams := &kinesis.GetRecordsInput{
			ShardIterator: nextShardIterator, // Required
			Limit:         aws.Int64(1000),
		}
		getRecordsResp, err := svc.GetRecords(getRecordsParams)
		if err != nil {
			log.Panic(err)
		}
		for _, record := range getRecordsResp.Records {
			eventChannel <- record.Data
		}
		nextShardIterator = getRecordsResp.NextShardIterator
		time.Sleep(time.Second)
	}
}
492614
func Restart(c *client.Client, appID string, procType string, num int) ([]api.Process, error) {
	u := fmt.Sprintf("/v1/apps/%s/containers/", appID)

	if procType == "" {
		u += "restart/"
	} else {
		if num == -1 {
			u += procType + "/restart/"
		} else {
			u += procType + "/" + strconv.Itoa(num) + "/restart/"
		}
	}

	body, err := c.BasicRequest("POST", u, nil)

	if err != nil {
		return []api.Process{}, err
	}

	procs := []api.Process{}
	if err = json.Unmarshal([]byte(body), &procs); err != nil {
		return []api.Process{}, err
	}

	return procs, nil
}
344097
func (s Shard) Serialize() []byte {
	sorted := make([]string, 0, len(s))
	for blob := range s {
		sorted = append(sorted, blob)
	}
	sort.Strings(sorted)
	out := bytes.Buffer{}
	enc := gob.NewEncoder(&out)
	err := enc.Encode(sorted)
	if err != nil {
		panic("impossible error when encoding []string")
	}
	return out.Bytes()
}
161044
func (h *altsHandshaker) processUntilDone(resp *altspb.HandshakerResp, extra []byte) (*altspb.HandshakerResult, []byte, error) {
	for {
		if len(resp.OutFrames) > 0 {
			if _, err := h.conn.Write(resp.OutFrames); err != nil {
				return nil, nil, err
			}
		}
		if resp.Result != nil {
			return resp.Result, extra, nil
		}
		buf := make([]byte, frameLimit)
		n, err := h.conn.Read(buf)
		if err != nil && err != io.EOF {
			return nil, nil, err
		}
		// If there is nothing to send to the handshaker service, and
		// nothing is received from the peer, then we are stuck.
		// This covers the case when the peer is not responding. Note
		// that handshaker service connection issues are caught in
		// accessHandshakerService before we even get here.
		if len(resp.OutFrames) == 0 && n == 0 {
			return nil, nil, core.PeerNotRespondingError
		}
		// Append extra bytes from the previous interaction with the
		// handshaker service with the current buffer read from conn.
		p := append(extra, buf[:n]...)
		resp, err = h.accessHandshakerService(&altspb.HandshakerReq{
			ReqOneof: &altspb.HandshakerReq_Next{
				Next: &altspb.NextHandshakeMessageReq{
					InBytes: p,
				},
			},
		})
		if err != nil {
			return nil, nil, err
		}
		// Set extra based on handshaker service response.
		if n == 0 {
			extra = nil
		} else {
			extra = buf[resp.GetBytesConsumed():n]
		}
	}
}
4153
func (e *executor) executeSumCountShard(ctx context.Context, index string, c *pql.Call, shard uint64) (ValCount, error) {
	span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeSumCountShard")
	defer span.Finish()

	var filter *Row
	if len(c.Children) == 1 {
		row, err := e.executeBitmapCallShard(ctx, index, c.Children[0], shard)
		if err != nil {
			return ValCount{}, errors.Wrap(err, "executing bitmap call")
		}
		filter = row
	}

	fieldName, _ := c.Args["field"].(string)

	field := e.Holder.Field(index, fieldName)
	if field == nil {
		return ValCount{}, nil
	}

	bsig := field.bsiGroup(fieldName)
	if bsig == nil {
		return ValCount{}, nil
	}

	fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
	if fragment == nil {
		return ValCount{}, nil
	}

	vsum, vcount, err := fragment.sum(filter, bsig.BitDepth())
	if err != nil {
		return ValCount{}, errors.Wrap(err, "computing sum")
	}
	return ValCount{
		Val:   int64(vsum) + (int64(vcount) * bsig.Min),
		Count: int64(vcount),
	}, nil
}
480583
func (p *AuroraSchedulerManagerClient) RestartShards(ctx context.Context, job *JobKey, shardIds []int32) (r *Response, err error) {
  var _args139 AuroraSchedulerManagerRestartShardsArgs
  _args139.Job = job
  _args139.ShardIds = shardIds
  var _result140 AuroraSchedulerManagerRestartShardsResult
  if err = p.Client_().Call(ctx, "restartShards", &_args139, &_result140); err != nil {
    return
  }
  return _result140.GetSuccess(), nil
}
1289950
func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) error {
	// Keep a count of uncertain headers to roll back
	rollback := []*types.Header{}
	defer func() {
		if len(rollback) > 0 {
			// Flatten the headers and roll them back
			hashes := make([]common.Hash, len(rollback))
			for i, header := range rollback {
				hashes[i] = header.Hash()
			}
			lastHeader, lastFastBlock, lastBlock := d.lightchain.CurrentHeader().Number, common.Big0, common.Big0
			if d.mode != LightSync {
				lastFastBlock = d.blockchain.CurrentFastBlock().Number()
				lastBlock = d.blockchain.CurrentBlock().Number()
			}
			d.lightchain.Rollback(hashes)
			curFastBlock, curBlock := common.Big0, common.Big0
			if d.mode != LightSync {
				curFastBlock = d.blockchain.CurrentFastBlock().Number()
				curBlock = d.blockchain.CurrentBlock().Number()
			}
			log.Warn("Rolled back headers", "count", len(hashes),
				"header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number),
				"fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock),
				"block", fmt.Sprintf("%d->%d", lastBlock, curBlock))
		}
	}()

	// Wait for batches of headers to process
	gotHeaders := false

	for {
		select {
		case <-d.cancelCh:
			return errCancelHeaderProcessing

		case headers := <-d.headerProcCh:
			// Terminate header processing if we synced up
			if len(headers) == 0 {
				// Notify everyone that headers are fully processed
				for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} {
					select {
					case ch <- false:
					case <-d.cancelCh:
					}
				}
				// If no headers were retrieved at all, the peer violated its TD promise that it had a
				// better chain compared to ours. The only exception is if its promised blocks were
				// already imported by other means (e.g. fetcher):
				//
				// R <remote peer>, L <local node>: Both at block 10
				// R: Mine block 11, and propagate it to L
				// L: Queue block 11 for import
				// L: Notice that R's head and TD increased compared to ours, start sync
				// L: Import of block 11 finishes
				// L: Sync begins, and finds common ancestor at 11
				// L: Request new headers up from 11 (R's TD was higher, it must have something)
				// R: Nothing to give
				if d.mode != LightSync {
					head := d.blockchain.CurrentBlock()
					if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 {
						return errStallingPeer
					}
				}
				// If fast or light syncing, ensure promised headers are indeed delivered. This is
				// needed to detect scenarios where an attacker feeds a bad pivot and then bails out
				// of delivering the post-pivot blocks that would flag the invalid content.
				//
				// This check cannot be executed "as is" for full imports, since blocks may still be
				// queued for processing when the header download completes. However, as long as the
				// peer gave us something useful, we're already happy/progressed (above check).
				if d.mode == FastSync || d.mode == LightSync {
					head := d.lightchain.CurrentHeader()
					if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 {
						return errStallingPeer
					}
				}
				// Disable any rollback and return
				rollback = nil
				return nil
			}
			// Otherwise split the chunk of headers into batches and process them
			gotHeaders = true
			for len(headers) > 0 {
				// Terminate if something failed in between processing chunks
				select {
				case <-d.cancelCh:
					return errCancelHeaderProcessing
				default:
				}
				// Select the next chunk of headers to import
				limit := maxHeadersProcess
				if limit > len(headers) {
					limit = len(headers)
				}
				chunk := headers[:limit]

				// In case of header only syncing, validate the chunk immediately
				if d.mode == FastSync || d.mode == LightSync {
					// Collect the yet unknown headers to mark them as uncertain
					unknown := make([]*types.Header, 0, len(headers))
					for _, header := range chunk {
						if !d.lightchain.HasHeader(header.Hash(), header.Number.Uint64()) {
							unknown = append(unknown, header)
						}
					}
					// If we're importing pure headers, verify based on their recentness
					frequency := fsHeaderCheckFrequency
					if chunk[len(chunk)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot {
						frequency = 1
					}
					if n, err := d.lightchain.InsertHeaderChain(chunk, frequency); err != nil {
						// If some headers were inserted, add them too to the rollback list
						if n > 0 {
							rollback = append(rollback, chunk[:n]...)
						}
						log.Debug("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "err", err)
						return errInvalidChain
					}
					// All verifications passed, store newly found uncertain headers
					rollback = append(rollback, unknown...)
					if len(rollback) > fsHeaderSafetyNet {
						rollback = append(rollback[:0], rollback[len(rollback)-fsHeaderSafetyNet:]...)
					}
				}
				// Unless we're doing light chains, schedule the headers for associated content retrieval
				if d.mode == FullSync || d.mode == FastSync {
					// If we've reached the allowed number of pending headers, stall a bit
					for d.queue.PendingBlocks() >= maxQueuedHeaders || d.queue.PendingReceipts() >= maxQueuedHeaders {
						select {
						case <-d.cancelCh:
							return errCancelHeaderProcessing
						case <-time.After(time.Second):
						}
					}
					// Otherwise insert the headers for content retrieval
					inserts := d.queue.Schedule(chunk, origin)
					if len(inserts) != len(chunk) {
						log.Debug("Stale headers")
						return errBadPeer
					}
				}
				headers = headers[limit:]
				origin += uint64(limit)
			}
			// Update the highest block number we know if a higher one is found.
			d.syncStatsLock.Lock()
			if d.syncStatsChainHeight < origin {
				d.syncStatsChainHeight = origin - 1
			}
			d.syncStatsLock.Unlock()

			// Signal the content downloaders of the availablility of new tasks
			for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} {
				select {
				case ch <- true:
				default:
				}
			}
		}
	}
}
28506
func (m *Method) postProcess(r *Resource, name string, traitsMap map[string]Trait, apiDef *APIDefinition) {
	m.Name = name
	m.inheritFromTraits(r, append(r.Is, m.Is...), traitsMap, apiDef)
	r.Methods = append(r.Methods, m)

	// post process the responses
	resps := make(map[HTTPCode]Response)
	for code, resp := range m.Responses {
		resp.postProcess()
		resps[code] = resp
	}
	m.Responses = resps

	// post process request body
	m.Bodies.postProcess()
}
832903
func (l *Synchronizer) PostProcess(processing map[string]struct{}) {
	// Get all locally stored data
	nodes, err := l.GetAll()
	if err != nil {
		glog.Warningf("Could not access locally stored data: %s", err)
		return
	}

	// Delete any local nodes that do not exist remotely and are not in process
	for _, node := range nodes {
		if _, ok := processing[node.GetID()]; ok {
			// pass
		} else if err := l.Delete(node.GetID()); err != nil {
			glog.Warningf("Could not delete %s from locally stored data: %s", node.GetID(), err)
		}
	}
}
242366
func (c *Connection) processChunk(chunk chunk) {
	m := c.msgStore.Get(chunk.MessageID)
	if m == nil {
		// Unexpected chunk, ignore it
		return
	}

	// Add chunk to message
	m.addChunk(chunk)

	// Try to assembly
	if m.assemble() {
		// Message is complete
		// Remove message from store
		c.msgStore.Remove(m.ID)

		//fmt.Println("Chunk: " + hex.EncodeToString(chunk.Data) + "\nMessage: " + hex.EncodeToString(m.Data))

		// Notify listener
		m.notifyListener()
	}
}