// splitAndLaunchHandler splits the job into shards and enqueues tasks that
// process shards.
970638
func (ctl *Controller) splitAndLaunchHandler(c context.Context, payload proto.Message) error {
	msg := payload.(*tasks.SplitAndLaunch)
	now := clock.Now(c).UTC()

	// Fetch job details. Make sure it isn't canceled and isn't running already.
	job, err := getJobInState(c, JobID(msg.JobId), State_STARTING)
	if err != nil || job == nil {
		return errors.Annotate(err, "in SplitAndLaunch").Err()
	}

	// Figure out key ranges for shards. There may be fewer shards than requested
	// if there are too few entities.
	dq := job.Config.Query.ToDatastoreQuery()
	ranges, err := splitter.SplitIntoRanges(c, dq, splitter.Params{
		Shards:  job.Config.ShardCount,
		Samples: 512, // should be enough for everyone...
	})
	if err != nil {
		return errors.Annotate(err, "failed to split the query into shards").Tag(transient.Tag).Err()
	}

	// Create entities that hold shards state. Each one is in its own entity
	// group, since the combined write rate to them is O(ShardCount), which can
	// overcome limits of a single entity group.
	shards := make([]*shard, len(ranges))
	for idx, rng := range ranges {
		shards[idx] = &shard{
			JobID:         job.ID,
			Index:         idx,
			State:         State_STARTING,
			Range:         rng,
			ExpectedCount: -1,
			Created:       now,
			Updated:       now,
		}
	}

	// Calculate number of entities in each shard to track shard processing
	// progress. Note that this can be very slow if there are many entities.
	if job.Config.TrackProgress {
		logging.Infof(c, "Estimating the size of each shard...")
		if err := fetchShardSizes(c, dq, shards); err != nil {
			return errors.Annotate(err, "when estimating shard sizes").Err()
		}
	}

	// We use auto-generated keys for shards to make sure crashed SplitAndLaunch
	// task retries cleanly, even if the underlying key space we are mapping over
	// changes between the retries (making a naive put using "<job-id>:<index>"
	// key non-idempotent!).
	logging.Infof(c, "Instantiating shards...")
	if err := datastore.Put(c, shards); err != nil {
		return errors.Annotate(err, "failed to store shards").Tag(transient.Tag).Err()
	}

	// Prepare shardList which is basically a manual fully consistent index for
	// Job -> [Shard] relation. We can't use a regular index, since shards are all
	// in different entity groups (see O(ShardCount) argument above).
	//
	// Log the resulting shards along the way.
	shardsEnt := shardList{
		Parent: datastore.KeyForObj(c, job),
		Shards: make([]int64, len(shards)),
	}
	for idx, s := range shards {
		shardsEnt.Shards[idx] = s.ID

		l, r := "-inf", "+inf"
		if s.Range.Start != nil {
			l = s.Range.Start.String()
		}
		if s.Range.End != nil {
			r = s.Range.End.String()
		}
		count := ""
		if s.ExpectedCount != 0 {
			count = fmt.Sprintf(" (%d entities)", s.ExpectedCount)
		}
		logging.Infof(c, "Shard #%d is %d: %s - %s%s", idx, s.ID, l, r, count)
	}

	// Transactionally associate shards with the job and launch the TQ task that
	// kicks off the processing of each individual shard. We use an intermediary
	// task for this since transactionally launching O(ShardCount) tasks hits TQ
	// transaction limits.
	//
	// If SplitAndLaunch crashes before this transaction lands, there'll be some
	// orphaned Shard entities, no big deal.
	logging.Infof(c, "Updating the job and launching the fan out task...")
	return runTxn(c, func(c context.Context) error {
		job, err := getJobInState(c, JobID(msg.JobId), State_STARTING)
		if err != nil || job == nil {
			return errors.Annotate(err, "in SplitAndLaunch txn").Err()
		}

		job.State = State_RUNNING
		job.Updated = now
		if err := datastore.Put(c, job, &shardsEnt); err != nil {
			return errors.Annotate(err,
				"when storing Job %d and ShardList with %d shards", job.ID, len(shards),
			).Tag(transient.Tag).Err()
		}

		return ctl.tq().AddTask(c, &tq.Task{
			Title: fmt.Sprintf("fanout:job-%d", job.ID),
			Payload: &tasks.FanOutShards{
				JobId: int64(job.ID),
			},
		})
	})
}
20473
func LoadLaunchConfig(s3c *s3.Client, bucket, prefix, hash string) (*LaunchConfig, error) {
	lc := new(LaunchConfig)
	err := lc.load(s3c, bucket, prefix, fmt.Sprintf("container.%s.json", hash))
	if err != nil {
		return nil, err
	}

	if lc.hash != hash {
		return nil, fmt.Errorf("given hash %q doesn't match actual hash %q", hash, lc.hash)
	}

	return lc, nil
}
519149
func (s *EnvoyApi) clustersHandler(response http.ResponseWriter, req *http.Request, params map[string]string) {
	defer req.Body.Close()

	response.Header().Set("Content-Type", "application/json")

	clusters := s.EnvoyClustersFromState()

	log.Debugf("Reporting Envoy cluster information for cluster '%s' and node '%s'",
		params["service_cluster"], params["service_node"])

	result := CDSResult{clusters}

	jsonBytes, err := result.MarshalJSON()
	defer ffjson.Pool(jsonBytes)
	if err != nil {
		log.Errorf("Error marshaling state in servicesHandler: %s", err.Error())
		sendJsonError(response, 500, "Internal server error")
		return
	}

	response.Write(jsonBytes)
}
1243665
func (os *OpenStack) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
	glog.V(4).Info("openstack.LoadBalancer() called")

	// TODO: Search for and support Rackspace loadbalancer API, and others.
	network, err := os.NewNetworkV2()
	if err != nil {
		return nil, false
	}

	compute, err := os.NewComputeV2()
	if err != nil {
		return nil, false
	}

	lbVersion := os.lbOpts.LBVersion
	if lbVersion == "" {
		// No version specified, try newest supported by server
		netExts, err := networkExtensions(network)
		if err != nil {
			glog.Warningf("Failed to list neutron extensions: %v", err)
			return nil, false
		}

		if netExts["lbaasv2"] {
			lbVersion = "v2"
		} else if netExts["lbaas"] {
			lbVersion = "v1"
		} else {
			glog.Warningf("Failed to find neutron LBaaS extension (v1 or v2)")
			return nil, false
		}
		glog.V(3).Infof("Using LBaaS extension %v", lbVersion)
	}

	glog.V(1).Info("Claiming to support LoadBalancer")

	if lbVersion == "v2" {
		return &LbaasV2{LoadBalancer{network, compute, os.lbOpts}}, true
	} else if lbVersion == "v1" {
		return &LbaasV1{LoadBalancer{network, compute, os.lbOpts}}, true
	} else {
		glog.Warningf("Config error: unrecognised lb-version \"%v\"", lbVersion)
		return nil, false
	}
}
457610
func joinHandlers(h1 context.Handlers, h2 context.Handlers) context.Handlers {
	nowLen := len(h1)
	totalLen := nowLen + len(h2)
	// create a new slice of Handlers in order to merge the "h1" and "h2"
	newHandlers := make(context.Handlers, totalLen)
	// copy the already Handlers to the just created
	copy(newHandlers, h1)
	// start from there we finish, and store the new Handlers too
	copy(newHandlers[nowLen:], h2)
	return newHandlers
}
1101775
func (c *Client) workHandler() {
	for {
		select {
		case w := <-c.work:
			c.process(w)
		case <-time.After(c.keepAlive):
			c.process(keepAliveData)
		case <-c.disconnect:
			return
		}
	}
}
319454
func (r *registry) loadHandlers(c Config) {
	for _, config := range c.AuthFilters {
		r.evaluateModeAndLoad(config, Auth)
	}
	for _, config := range c.Decorators {
		r.evaluateModeAndLoad(config, Decoration)
	}

	for chaincodeID, config := range c.Endorsers {
		r.evaluateModeAndLoad(config, Endorsement, chaincodeID)
	}

	for chaincodeID, config := range c.Validators {
		r.evaluateModeAndLoad(config, Validation, chaincodeID)
	}
}
1010813
func (sc scanCloser) Split(split bufio.SplitFunc) {
	sc.scanner.Split(split)
}
255643
func MultiHandler(handlers ...Handler) Handler {
	multi := make([]Handler, 0, len(handlers))

	for _, h := range handlers {
		if h != nil {
			if m, ok := h.(*multiHandler); ok {
				multi = append(multi, m.handlers...) // flatten multi handlers
			} else {
				multi = append(multi, h)
			}
		}
	}

	if len(multi) == 1 {
		return multi[0]
	}

	return &multiHandler{handlers: multi}
}
986143
func serviceLoader(h http.Handler, svcs ...service) http.Handler {
	for _, svc := range svcs {
		h = svc(h)
	}
	return h
}
830816
func (c *Controller) syncHandler(key string) error {
	// Convert the namespace/name string into a distinct namespace and name
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key))
		return nil
	}

	// Get the Foo resource with this namespace/name
	foo, err := c.foosLister.Foos(namespace).Get(name)
	if err != nil {
		// The Foo resource may no longer exist, in which case we stop
		// processing.
		if errors.IsNotFound(err) {
			utilruntime.HandleError(fmt.Errorf("foo '%s' in work queue no longer exists", key))
			return nil
		}

		return err
	}

	deploymentName := foo.Spec.DeploymentName
	if deploymentName == "" {
		// We choose to absorb the error here as the worker would requeue the
		// resource otherwise. Instead, the next time the resource is updated
		// the resource will be queued again.
		utilruntime.HandleError(fmt.Errorf("%s: deployment name must be specified", key))
		return nil
	}

	// Get the deployment with the name specified in Foo.spec
	deployment, err := c.deploymentsLister.Deployments(foo.Namespace).Get(deploymentName)
	// If the resource doesn't exist, we'll create it
	if errors.IsNotFound(err) {
		deployment, err = c.kubeclientset.AppsV1().Deployments(foo.Namespace).Create(newDeployment(foo))
	}

	// If an error occurs during Get/Create, we'll requeue the item so we can
	// attempt processing again later. This could have been caused by a
	// temporary network failure, or any other transient reason.
	if err != nil {
		return err
	}

	// If the Deployment is not controlled by this Foo resource, we should log
	// a warning to the event recorder and ret
	if !metav1.IsControlledBy(deployment, foo) {
		msg := fmt.Sprintf(MessageResourceExists, deployment.Name)
		c.recorder.Event(foo, corev1.EventTypeWarning, ErrResourceExists, msg)
		return fmt.Errorf(msg)
	}

	// If this number of the replicas on the Foo resource is specified, and the
	// number does not equal the current desired replicas on the Deployment, we
	// should update the Deployment resource.
	if foo.Spec.Replicas != nil && *foo.Spec.Replicas != *deployment.Spec.Replicas {
		klog.V(4).Infof("Foo %s replicas: %d, deployment replicas: %d", name, *foo.Spec.Replicas, *deployment.Spec.Replicas)
		deployment, err = c.kubeclientset.AppsV1().Deployments(foo.Namespace).Update(newDeployment(foo))
	}

	// If an error occurs during Update, we'll requeue the item so we can
	// attempt processing again later. THis could have been caused by a
	// temporary network failure, or any other transient reason.
	if err != nil {
		return err
	}

	// Finally, we update the status block of the Foo resource to reflect the
	// current state of the world
	err = c.updateFooStatus(foo, deployment)
	if err != nil {
		return err
	}

	c.recorder.Event(foo, corev1.EventTypeNormal, SuccessSynced, MessageResourceSynced)
	return nil
}
926592
func (r *FlatChain) ChainHandler(h http.Handler) FlatChains {
	fh := FlatChainHandlerWrap(h, r.log)
	r.Chain(fh)
	return fh
}
1330890
func (ctl *Controller) LaunchJob(c context.Context, j *JobConfig) (JobID, error) {
	disp := ctl.tq()

	if err := j.Validate(); err != nil {
		return 0, errors.Annotate(err, "bad job config").Err()
	}
	if _, err := ctl.getFactory(j.Mapper); err != nil {
		return 0, errors.Annotate(err, "bad job config").Err()
	}

	// Prepare and store the job entity, generate its key. Launch a tq task that
	// subdivides the key space and launches individual shards. We do it
	// asynchronously since this can be potentially slow (for large number of
	// shards).
	var job Job
	err := runTxn(c, func(c context.Context) error {
		now := clock.Now(c).UTC()
		job = Job{
			Config:  *j,
			State:   State_STARTING,
			Created: now,
			Updated: now,
		}
		if err := datastore.Put(c, &job); err != nil {
			return errors.Annotate(err, "failed to store Job entity").Tag(transient.Tag).Err()
		}
		return disp.AddTask(c, &tq.Task{
			Title: fmt.Sprintf("split:job-%d", job.ID),
			Payload: &tasks.SplitAndLaunch{
				JobId: int64(job.ID),
			},
		})
	})
	if err != nil {
		return 0, err
	}
	return job.ID, nil
}
924402
func (b *Bucket) Split(cpl int, target ID) *Bucket {
	b.lk.Lock()
	defer b.lk.Unlock()

	out := list.New()
	newbuck := newBucket()
	newbuck.list = out
	e := b.list.Front()
	for e != nil {
		peerID := ConvertPeerID(e.Value.(peer.ID))
		peerCPL := CommonPrefixLen(peerID, target)
		if peerCPL > cpl {
			cur := e
			out.PushBack(e.Value)
			e = e.Next()
			b.list.Remove(cur)
			continue
		}
		e = e.Next()
	}
	return newbuck
}
1230675
func (cs *CSCloud) LoadBalancer() (cloudprovider.LoadBalancer, bool) {
	if cs.client == nil {
		return nil, false
	}

	return cs, true
}
405066
func TaskQueueHandler(ctx *router.Context) {
	c, rw, r, p := ctx.Context, ctx.Writer, ctx.Request, ctx.Params
	defer r.Body.Close()

	cfgName := p.ByName("cfgName")
	dist, _, err := GetRegistry(c).MakeDistributor(c, cfgName)
	if err != nil {
		logging.Fields{"error": err, "cfg": cfgName}.Errorf(c, "Failed to make distributor")
		http.Error(rw, "bad distributor", http.StatusBadRequest)
		return
	}
	notifications, err := dist.HandleTaskQueueTask(r)
	if err != nil {
		logging.Fields{"error": err, "cfg": cfgName}.Errorf(c, "Failed to handle taskqueue task")
		http.Error(rw, "failure to execute handler", http.StatusInternalServerError)
		return
	}
	if len(notifications) > 0 {
		muts := make([]tumble.Mutation, 0, len(notifications))
		for _, notify := range notifications {
			if notify != nil {
				muts = append(muts, &NotifyExecution{cfgName, notify})
			}
		}
		err = tumble.AddToJournal(c, muts...)
		if err != nil {
			logging.Fields{"error": err, "cfg": cfgName}.Errorf(c, "Failed to handle notifications")
			http.Error(rw, "failure to handle notifications", http.StatusInternalServerError)
			return
		}
	}
	rw.WriteHeader(http.StatusOK)
}
1229354
func (p *ChainMonitor) ReorgHandler() {
	defer p.wg.Done()
out:
	for {
		//keepon:
		select {
		case reorgData, ok := <-p.reorgChan:
			p.mtx.Lock()
			if !ok {
				p.mtx.Unlock()
				log.Warnf("Reorg channel closed.")
				break out
			}

			newHeight, oldHeight := reorgData.NewChainHeight, reorgData.OldChainHeight
			newHash, oldHash := reorgData.NewChainHead, reorgData.OldChainHead

			log.Infof("Reorganize started. NEW head block %v at height %d.",
				newHash, newHeight)
			log.Infof("Reorganize started. OLD head block %v at height %d.",
				oldHash, oldHeight)

			// Switch to the side chain.
			stakeDBTipHeight, stakeDBTipHash, err := p.switchToSideChain(reorgData)
			if err != nil {
				log.Errorf("switchToSideChain failed: %v", err)
			}
			if stakeDBTipHeight != newHeight {
				log.Errorf("stakeDBTipHeight is %d, expected %d",
					stakeDBTipHeight, newHeight)
			}
			if *stakeDBTipHash != newHash {
				log.Errorf("stakeDBTipHash is %d, expected %d",
					stakeDBTipHash, newHash)
			}

			p.mtx.Unlock()

			reorgData.WG.Done()

		case <-p.ctx.Done():
			log.Debugf("Got quit signal. Exiting stakedb reorg notification handler.")
			break out
		}
	}
}