// Spin all jobs
430655
func (s *Spinner) Spin() error {
	for _, j := range s.jobs {
		go func(j *Job) {
			ticker := time.NewTicker(j.RunsEvery.Duration)
			for range ticker.C {
				cmd := exec.Command(j.Cmd, j.Args...)
				SigChan := make(chan os.Signal)
				p := &Process{id: uuid.NewV4().String(), cmd: cmd, SigChan: SigChan}
				err := j.AddProcess(p)
				if err == nil {
					go p.run()
				}
			}
		}(j)
	}
	return nil
}
1281772
func sortJobs(c context.Context, jobs []*engine.Job) sortedJobs {
	out := make(sortedJobs, len(jobs))
	for i, job := range jobs {
		out[i] = makeJob(c, job, nil)
	}
	sort.Sort(out)
	return out
}
1276766
func (c *CoreScheduler) jobReap(jobs []*structs.Job, leaderACL string) error {
	// Call to the leader to issue the reap
	for _, req := range c.partitionJobReap(jobs, leaderACL) {
		var resp structs.JobBatchDeregisterResponse
		if err := c.srv.RPC("Job.BatchDeregister", req, &resp); err != nil {
			c.logger.Error("batch job reap failed", "error", err)
			return err
		}
	}

	return nil
}
1248257
func Spinners() error {
	pkg := "spin"
	_, err := os.Stat(pkg)
	if err != nil {
		if os.IsNotExist(err) {
			return os.Mkdir(pkg, 0777)
		}
		return err
	}
	b, err := ioutil.ReadFile("cli-spinners/spinners.json")
	if err != nil {
		return err
	}
	o := make(map[string]interface{})
	err = json.Unmarshal(b, &o)
	if err != nil {
		return err
	}
	tpl, err := template.New("spinner").Funcs(helpers()).Parse(spinnersTpl)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	err = tpl.Execute(&buf, o)
	if err != nil {
		return err
	}
	bo, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}
	return ioutil.WriteFile(filepath.Join(pkg, "spinners.go"), bo, 0600)
}
296606
func (m *Matrix) Rotate(deg float32) *Matrix {
	return m.RotateRad(deg * DegToRad)
}
792061
func (j *Jenkins) GetAllJobs() ([]*Job, error) {
	exec := Executor{Raw: new(ExecutorResponse), Jenkins: j}
	_, err := j.Requester.GetJSON("/", exec.Raw, nil)

	if err != nil {
		return nil, err
	}

	jobs := make([]*Job, len(exec.Raw.Jobs))
	for i, job := range exec.Raw.Jobs {
		ji, err := j.GetJob(job.Name)
		if err != nil {
			return nil, err
		}
		jobs[i] = ji
	}
	return jobs, nil
}
392594
func (jm *JobController) syncJob(key string) (bool, error) {
	startTime := time.Now()
	defer func() {
		klog.V(4).Infof("Finished syncing job %q (%v)", key, time.Since(startTime))
	}()

	ns, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		return false, err
	}
	if len(ns) == 0 || len(name) == 0 {
		return false, fmt.Errorf("invalid job key %q: either namespace or name is missing", key)
	}
	sharedJob, err := jm.jobLister.Jobs(ns).Get(name)
	if err != nil {
		if errors.IsNotFound(err) {
			klog.V(4).Infof("Job has been deleted: %v", key)
			jm.expectations.DeleteExpectations(key)
			return true, nil
		}
		return false, err
	}
	job := *sharedJob

	// if job was finished previously, we don't want to redo the termination
	if IsJobFinished(&job) {
		return true, nil
	}

	// retrieve the previous number of retry
	previousRetry := jm.queue.NumRequeues(key)

	// Check the expectations of the job before counting active pods, otherwise a new pod can sneak in
	// and update the expectations after we've retrieved active pods from the store. If a new pod enters
	// the store after we've checked the expectation, the job sync is just deferred till the next relist.
	jobNeedsSync := jm.expectations.SatisfiedExpectations(key)

	pods, err := jm.getPodsForJob(&job)
	if err != nil {
		return false, err
	}

	activePods := controller.FilterActivePods(pods)
	active := int32(len(activePods))
	succeeded, failed := getStatus(pods)
	conditions := len(job.Status.Conditions)
	// job first start
	if job.Status.StartTime == nil {
		now := metav1.Now()
		job.Status.StartTime = &now
		// enqueue a sync to check if job past ActiveDeadlineSeconds
		if job.Spec.ActiveDeadlineSeconds != nil {
			klog.V(4).Infof("Job %s have ActiveDeadlineSeconds will sync after %d seconds",
				key, *job.Spec.ActiveDeadlineSeconds)
			jm.queue.AddAfter(key, time.Duration(*job.Spec.ActiveDeadlineSeconds)*time.Second)
		}
	}

	var manageJobErr error
	jobFailed := false
	var failureReason string
	var failureMessage string

	jobHaveNewFailure := failed > job.Status.Failed
	// new failures happen when status does not reflect the failures and active
	// is different than parallelism, otherwise the previous controller loop
	// failed updating status so even if we pick up failure it is not a new one
	exceedsBackoffLimit := jobHaveNewFailure && (active != *job.Spec.Parallelism) &&
		(int32(previousRetry)+1 > *job.Spec.BackoffLimit)

	if exceedsBackoffLimit || pastBackoffLimitOnFailure(&job, pods) {
		// check if the number of pod restart exceeds backoff (for restart OnFailure only)
		// OR if the number of failed jobs increased since the last syncJob
		jobFailed = true
		failureReason = "BackoffLimitExceeded"
		failureMessage = "Job has reached the specified backoff limit"
	} else if pastActiveDeadline(&job) {
		jobFailed = true
		failureReason = "DeadlineExceeded"
		failureMessage = "Job was active longer than specified deadline"
	}

	if jobFailed {
		errCh := make(chan error, active)
		jm.deleteJobPods(&job, activePods, errCh)
		select {
		case manageJobErr = <-errCh:
			if manageJobErr != nil {
				break
			}
		default:
		}

		// update status values accordingly
		failed += active
		active = 0
		job.Status.Conditions = append(job.Status.Conditions, newCondition(batch.JobFailed, failureReason, failureMessage))
		jm.recorder.Event(&job, v1.EventTypeWarning, failureReason, failureMessage)
	} else {
		if jobNeedsSync && job.DeletionTimestamp == nil {
			active, manageJobErr = jm.manageJob(activePods, succeeded, &job)
		}
		completions := succeeded
		complete := false
		if job.Spec.Completions == nil {
			// This type of job is complete when any pod exits with success.
			// Each pod is capable of
			// determining whether or not the entire Job is done.  Subsequent pods are
			// not expected to fail, but if they do, the failure is ignored.  Once any
			// pod succeeds, the controller waits for remaining pods to finish, and
			// then the job is complete.
			if succeeded > 0 && active == 0 {
				complete = true
			}
		} else {
			// Job specifies a number of completions.  This type of job signals
			// success by having that number of successes.  Since we do not
			// start more pods than there are remaining completions, there should
			// not be any remaining active pods once this count is reached.
			if completions >= *job.Spec.Completions {
				complete = true
				if active > 0 {
					jm.recorder.Event(&job, v1.EventTypeWarning, "TooManyActivePods", "Too many active pods running after completion count reached")
				}
				if completions > *job.Spec.Completions {
					jm.recorder.Event(&job, v1.EventTypeWarning, "TooManySucceededPods", "Too many succeeded pods running after completion count reached")
				}
			}
		}
		if complete {
			job.Status.Conditions = append(job.Status.Conditions, newCondition(batch.JobComplete, "", ""))
			now := metav1.Now()
			job.Status.CompletionTime = &now
		}
	}

	forget := false
	// Check if the number of jobs succeeded increased since the last check. If yes "forget" should be true
	// This logic is linked to the issue: https://github.com/kubernetes/kubernetes/issues/56853 that aims to
	// improve the Job backoff policy when parallelism > 1 and few Jobs failed but others succeed.
	// In this case, we should clear the backoff delay.
	if job.Status.Succeeded < succeeded {
		forget = true
	}

	// no need to update the job if the status hasn't changed since last time
	if job.Status.Active != active || job.Status.Succeeded != succeeded || job.Status.Failed != failed || len(job.Status.Conditions) != conditions {
		job.Status.Active = active
		job.Status.Succeeded = succeeded
		job.Status.Failed = failed

		if err := jm.updateHandler(&job); err != nil {
			return forget, err
		}

		if jobHaveNewFailure && !IsJobFinished(&job) {
			// returning an error will re-enqueue Job after the backoff period
			return forget, fmt.Errorf("failed pod(s) detected for job key %q", key)
		}

		forget = true
	}

	return forget, manageJobErr
}
858275
func Rotate(cfg *config.Config) (io.Writer, error) {
	format, found := cfg.Attrs["filename"]
	if !found {
		return nil, argNotFoundErr("rotate", "filename")
	}

	dir, found := cfg.Attrs["dir"]
	if !found {
		return nil, argNotFoundErr("rotate", "dir")
	}

	sizeStr, found := cfg.Attrs["size"]
	if !found {
		return nil, argNotFoundErr("rotate", "size")
	}

	size, err := toByte(sizeStr)
	if err != nil {
		return nil, err
	}

	return rotate.New(format, dir, size)
}
962444
func (c *Client) RetryJobs(page uint) ([]*RetryJob, int64, error) {
	key := redisKeyRetry(c.namespace)
	jobsWithScores, count, err := c.getZsetPage(key, page)
	if err != nil {
		logError("client.retry_jobs.get_zset_page", err)
		return nil, 0, err
	}

	jobs := make([]*RetryJob, 0, len(jobsWithScores))

	for _, jws := range jobsWithScores {
		jobs = append(jobs, &RetryJob{RetryAt: jws.Score, Job: jws.job})
	}

	return jobs, count, nil
}
127057
func (c *cluster) job(id int64) *resizeJob {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.jobs[id]
}
877480
func (m *ManualGenerator) Spindle(enabled, clockwise bool, speed float64) {
	if m.sguard > 0 {
		return
	}
	m.sguard++
	defer func() {
		m.sguard--
	}()

	if !*manualSpindle {
		return
	}

	if enabled {
		if clockwise {
			fmt.Fprintf(os.Stderr, "Set spindle to clockwise rotation at %.2f RPM. Confirm with <ENTER>", speed)
		} else {
			fmt.Fprintf(os.Stderr, "Set spindle to counter clockwise rotation at %.2f RPM. Confirm with <ENTER>", speed)
		}
	} else {
		fmt.Fprintf(os.Stderr, "Disable spindle. Confirm with <ENTER>")
	}
	reader := bufio.NewReader(os.Stdin)
	_, _ = reader.ReadString('\n')
}
666902
func (worker *Worker) RunJob(jobID string) error {
	qorJob, err := worker.GetJob(jobID)

	if qorJob != nil && err == nil {
		defer func() {
			if r := recover(); r != nil {
				qorJob.AddLog(string(debug.Stack()))
				qorJob.SetProgressText(fmt.Sprint(r))
				qorJob.SetStatus(JobStatusException)
			}
		}()

		if qorJob.GetStatus() != JobStatusNew && qorJob.GetStatus() != JobStatusScheduled {
			return errors.New("invalid job status, current status: " + qorJob.GetStatus())
		}

		if err = qorJob.SetStatus(JobStatusRunning); err == nil {
			if err = qorJob.GetJob().GetQueue().Run(qorJob); err == nil {
				return qorJob.SetStatus(JobStatusDone)
			}

			qorJob.SetProgressText(err.Error())
			qorJob.SetStatus(JobStatusException)
		}
	}

	return err
}
1178129
func (m *Manta) handleJobs(w http.ResponseWriter, r *http.Request, _ httprouter.Params) error {
	var live = false
	switch r.Method {
	case "GET":
		if strings.HasSuffix(r.URL.Path, "jobs") {
			// ListJobs
			if state := r.FormValue("state"); state == "running" {
				live = true
			}
			jobs, err := m.ListJobs(live)
			if err != nil {
				return err
			}
			if len(jobs) == 0 {
				jobs = []manta.Entry{}
			}
			resp := jobs
			return sendJSON(http.StatusOK, resp, w, r)
		} else if strings.HasSuffix(r.URL.Path, "status") {
			// GetJob
			job, err := m.GetJob(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			if job == nil {
				job = &manta.Job{}
			}
			resp := job
			return sendJSON(http.StatusOK, resp, w, r)
		} else if strings.HasSuffix(r.URL.Path, "out") {
			// GetJobOutput
			out, err := m.GetJobOutput(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			resp := out
			return sendJSON(http.StatusOK, resp, w, r)
		} else if strings.HasSuffix(r.URL.Path, "in") {
			// GetJobInput
			in, err := m.GetJobInput(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			resp := in
			return sendJSON(http.StatusOK, resp, w, r)
		} else if strings.HasSuffix(r.URL.Path, "fail") {
			// GetJobFailures
			fail, err := m.GetJobFailures(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			resp := fail
			return sendJSON(http.StatusOK, resp, w, r)
		} else if strings.HasSuffix(r.URL.Path, "err") {
			// GetJobErrors
			jobErr, err := m.GetJobErrors(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			//if jobErr == nil {
			//	jobErr = []manta.JobError{}
			//}
			resp := jobErr
			return sendJSON(http.StatusOK, resp, w, r)
		} else {
			return ErrNotAllowed
		}
	case "POST":
		if strings.HasSuffix(r.URL.Path, "jobs") {
			body, errb := ioutil.ReadAll(r.Body)
			if errb != nil {
				return errb
			}
			if len(body) == 0 {
				return ErrBadRequest
			}
			jobId, err := m.CreateJob(body)
			if err != nil {
				return err
			}
			w.Header().Add("Location", jobId)
			return sendJSON(http.StatusCreated, nil, w, r)
		} else if strings.HasSuffix(r.URL.Path, "cancel") {
			// CancelJob
			err := m.CancelJob(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			return sendJSON(http.StatusAccepted, nil, w, r)
		} else if strings.HasSuffix(r.URL.Path, "end") {
			// EndJobInputs
			err := m.EndJobInput(getJobId(r.URL.Path))
			if err != nil {
				return err
			}
			return sendJSON(http.StatusAccepted, nil, w, r)
		} else if strings.HasSuffix(r.URL.Path, "in") {
			// AddJobInputs
			body, err := ioutil.ReadAll(r.Body)
			if err != nil {
				return err
			}
			if len(body) == 0 {
				return ErrBadRequest
			}
			err = m.AddJobInputs(getJobId(r.URL.Path), body)
			if err != nil {
				return err
			}
			return sendJSON(http.StatusNoContent, nil, w, r)
		} else {
			return ErrNotAllowed
		}
	case "PUT":
		return ErrNotAllowed
	case "DELETE":
		return ErrNotAllowed
	}
	return fmt.Errorf("unknown request method %q for %s", r.Method, r.URL.Path)
}
428735
func (w *worker) finishDDLJob(t *meta.Meta, job *model.Job) (err error) {
	startTime := time.Now()
	defer func() {
		metrics.DDLWorkerHistogram.WithLabelValues(metrics.WorkerFinishDDLJob, job.Type.String(), metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
	}()

	if !job.IsCancelled() {
		switch job.Type {
		case model.ActionAddIndex:
			if job.State != model.JobStateRollbackDone {
				break
			}

			// After rolling back an AddIndex operation, we need to use delete-range to delete the half-done index data.
			err = w.deleteRange(job)
		case model.ActionDropSchema, model.ActionDropTable, model.ActionTruncateTable, model.ActionDropIndex, model.ActionDropTablePartition, model.ActionTruncateTablePartition:
			err = w.deleteRange(job)
		}
	}
	switch job.Type {
	case model.ActionRecoverTable:
		err = finishRecoverTable(w, t, job)
	}
	if err != nil {
		return errors.Trace(err)
	}

	_, err = t.DeQueueDDLJob()
	if err != nil {
		return errors.Trace(err)
	}

	job.BinlogInfo.FinishedTS = t.StartTS
	logutil.Logger(w.logCtx).Info("[ddl] finish DDL job", zap.String("job", job.String()))
	err = t.AddHistoryDDLJob(job)
	return errors.Trace(err)
}
183024
func (process *TeleportProcess) rotate(conn *Connector, localState auth.StateV2, remote services.Rotation) (*rotationStatus, error) {
	id := conn.ClientIdentity.ID
	local := localState.Spec.Rotation

	additionalPrincipals, dnsNames, err := process.getAdditionalPrincipals(id.Role)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	additionalPrincipals = utils.ReplaceInSlice(
		additionalPrincipals,
		defaults.AnyAddress,
		defaults.Localhost,
	)

	principalsOrDNSNamesChanged := (len(additionalPrincipals) != 0 && !conn.ServerIdentity.HasPrincipals(additionalPrincipals)) ||
		(len(dnsNames) != 0 && !conn.ServerIdentity.HasDNSNames(dnsNames))

	if local.Matches(remote) && !principalsOrDNSNamesChanged {
		// nothing to do, local state and rotation state are in sync
		return &rotationStatus{}, nil
	}

	storage := process.storage

	const outOfSync = "%v and cluster rotation state (%v) is out of sync with local (%v). Clear local state and re-register this %v."

	writeStateAndIdentity := func(name string, identity *auth.Identity) error {
		err = storage.WriteIdentity(name, *identity)
		if err != nil {
			return trace.Wrap(err)
		}
		localState.Spec.Rotation = remote
		err = storage.WriteState(id.Role, localState)
		if err != nil {
			return trace.Wrap(err)
		}
		return nil
	}

	switch remote.State {
	case "", services.RotationStateStandby:
		switch local.State {
		// There is nothing to do, it could happen
		// that the old node came up and missed the whole rotation
		// rollback cycle.
		case "", services.RotationStateStandby:
			if principalsOrDNSNamesChanged {
				process.Infof("Service %v has updated principals to %q, DNS Names to %q, going to request new principals and update.", id.Role, additionalPrincipals, dnsNames)
				identity, err := process.reRegister(conn, additionalPrincipals, dnsNames, remote)
				if err != nil {
					return nil, trace.Wrap(err)
				}
				err = storage.WriteIdentity(auth.IdentityCurrent, *identity)
				if err != nil {
					return nil, trace.Wrap(err)
				}
				return &rotationStatus{needsReload: true}, nil
			}
			return &rotationStatus{}, nil
		case services.RotationStateInProgress:
			// Rollback phase has been completed, all services
			// will receive new identities.
			if local.Phase != services.RotationPhaseRollback && local.CurrentID != remote.CurrentID {
				return nil, trace.CompareFailed(outOfSync, id.Role, remote, local, id.Role)
			}
			identity, err := process.reRegister(conn, additionalPrincipals, dnsNames, remote)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			err = writeStateAndIdentity(auth.IdentityCurrent, identity)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			return &rotationStatus{needsReload: true}, nil
		default:
			return nil, trace.BadParameter("unsupported state: %q", localState)
		}
	case services.RotationStateInProgress:
		switch remote.Phase {
		case services.RotationPhaseStandby, "":
			// There is nothing to do.
			return &rotationStatus{}, nil
		case services.RotationPhaseInit:
			// Only allow transition in case if local rotation state is standby
			// so this server is in the "clean" state.
			if local.State != services.RotationStateStandby && local.State != "" {
				return nil, trace.CompareFailed(outOfSync, id.Role, remote, local, id.Role)
			}
			// only update local phase, there is no need to reload
			localState.Spec.Rotation = remote
			err = storage.WriteState(id.Role, localState)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			return &rotationStatus{phaseChanged: true}, nil
		case services.RotationPhaseUpdateClients:
			// Allow transition to this phase only if the previous
			// phase was "Init".
			if local.Phase != services.RotationPhaseInit && local.CurrentID != remote.CurrentID {
				return nil, trace.CompareFailed(outOfSync, id.Role, remote, local, id.Role)
			}
			identity, err := process.reRegister(conn, additionalPrincipals, dnsNames, remote)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			process.Debugf("Re-registered, received new identity %v.", identity)
			err = writeStateAndIdentity(auth.IdentityReplacement, identity)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			// Require reload of teleport process to update client and servers.
			return &rotationStatus{needsReload: true}, nil
		case services.RotationPhaseUpdateServers:
			// Allow transition to this phase only if the previous
			// phase was "Update clients".
			if local.Phase != services.RotationPhaseUpdateClients && local.CurrentID != remote.CurrentID {
				return nil, trace.CompareFailed(outOfSync, id.Role, remote, local, id.Role)
			}
			// Write the replacement identity as a current identity and reload the server.
			replacement, err := storage.ReadIdentity(auth.IdentityReplacement, id.Role)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			err = writeStateAndIdentity(auth.IdentityCurrent, replacement)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			// Require reload of teleport process to update servers.
			return &rotationStatus{needsReload: true}, nil
		case services.RotationPhaseRollback:
			// Allow transition to this phase from any other local phase
			// because it will be widely used to recover cluster state to
			// the previously valid state, client will re-register to receive
			// credentials signed by the "old" CA.
			identity, err := process.reRegister(conn, additionalPrincipals, dnsNames, remote)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			err = writeStateAndIdentity(auth.IdentityCurrent, identity)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			// Require reload of teleport process to update servers.
			return &rotationStatus{needsReload: true}, nil
		default:
			return nil, trace.BadParameter("unsupported phase: %q", remote.Phase)
		}
	default:
		return nil, trace.BadParameter("unsupported state: %q", remote.State)
	}
}
391354
func (jm *JobController) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer jm.queue.ShutDown()

	klog.Infof("Starting job controller")
	defer klog.Infof("Shutting down job controller")

	if !controller.WaitForCacheSync("job", stopCh, jm.podStoreSynced, jm.jobStoreSynced) {
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(jm.worker, time.Second, stopCh)
	}

	<-stopCh
}
130961
func (c *converter) Handle(_ <-chan struct{}) error {
	results, err := c.machine.Jobs()
	if err != nil {
		return errors.Annotate(err, "can't get jobs for machine")
	}
	if !multiwatcher.AnyJobNeedsState(results.Jobs...) {
		return nil
	}

	return &util.FatalError{"bounce agent to pick up new jobs"}
}