Create step execution item for a job reference

@param jobIdentifier
@param args
@param nodeStep
@param handler
@param keepgoingOnSuccess

@return

@deprecated
186255
public static StepExecutionItem createJobRef(
            final String jobIdentifier,
            final String[] args,
            final boolean nodeStep,
            final StepExecutionItem handler,
            final boolean keepgoingOnSuccess,
            final String label
    )
    {
        return createJobRef(
                jobIdentifier,
                args,
                nodeStep,
                handler,
                keepgoingOnSuccess,
                null,
                null,
                null,
                null,
                null,
                label,
                false,
                null,
                false,
                false,
                null,
                false
        );
    }
987434
@Override
    public R visitIdentifier(IdentifierTree node, P p) {
        return defaultAction(node, p);
    }
1076902
@Override
  public void sourceConfInit(FlowProcess<JobConf> process, JobConf conf) {
    try {
      Path root = getQualifiedPath(conf);
      if (_options.attrs != null && _options.attrs.length > 0) {
        Pail pail = new Pail(_pailRoot, conf);
        for (List<String> attr : _options.attrs) {
          String rel = Utils.join(attr, "/");
          pail.getSubPail(rel); //ensure the path exists
          Path toAdd = new Path(root, rel);
          LOG.info("Adding input path " + toAdd.toString());
          FileInputFormat.addInputPath(conf, toAdd);
        }
      } else {
        FileInputFormat.addInputPath(conf, root);
      }

      getScheme().sourceConfInit(process, this, conf);
      makeLocal(conf, getQualifiedPath(conf), "forcing job to local mode, via source: ");
      TupleSerialization.setSerializations(conf);
    } catch (IOException e) {
      throw new TapException(e);
    }
  }
656277
function traceFromNode(name) {
    const job = {
      node: name,
      isValid: true
    };

    invalidatePendingJobsForNode(pendingJobs, name);

    // Ensure the job can be tracked
    pendingJobs.push(job);

    // Force an asynchronous start to the tracing so that other parts of a
    // codebase can synchronously trigger the job to be invalidated. This
    // helps to avoid any unnecessary work that may no longer be relevant
    process.nextTick(startTracingNode);

    if (!hasSignalledStart) {
      hasSignalledStart = true;
      events.emit('started', {state: state});
    }

    function startTracingNode() {
      // Handle situations where a job may have been invalidated further
      // down the stack from where the original call originated from
      if (!job.isValid) {
        return;
      }

      getDependencies(name)
        .then(handleDependencies)
        .catch(handleError)
        .then(signalIfCompleted);

      function handleDependencies(dependencies) {
        // If this job has been invalidated, we can ignore anything that may
        // have resulted from it
        if (!job.isValid) {
          return;
        }

        // Indicate that this job is no longer blocking the `completed` stage
        pull(pendingJobs, job);

        // Sanity check
        if (!isArray(dependencies)) {
          return Promise.reject(
            new Error(`Dependencies should be specified in an array. Received ${dependencies}`)
          );
        }

        const previousState = state;

        if (!isNodeDefined(state, name)) {
          state = addNode(state, name);
        }

        // If there are any dependencies encountered that we don't already
        // know about, we start tracing them
        dependencies.forEach(depName => {
          if (
            !isNodeDefined(state, depName) &&
            !isNodePending(pendingJobs, depName)
          ) {
            traceFromNode(depName);
          }

          if (!isNodeDefined(state, depName)) {
            state = addNode(state, depName);
          }

          state = addEdge(state, name, depName);
        });

        // Enable progress updates
        events.emit('traced', {
          node: name,
          diff: Diff({
            from: previousState,
            to: state
          })
        });
      }

      function handleError(err) {
        // Indicate that this job is no longer blocking the `completed` stage
        pull(pendingJobs, job);

        // If the job has been invalidated, we ignore the error
        if (!job.isValid) {
          return;
        }

        const signal = {
          error: err,
          node: name,
          state: state
        };

        errors.push(signal);
        events.emit('error', signal);
      }
    }
  }
1026677
private static Node checkFollowedIdentifier(
			final FormItemList formItemList,
			final CreateFollowResponse createFollowResponse) {
		final String followedId = formItemList
				.getField(ProtocolConstants.Parameters.Create.Follow.FOLLOWED_IDENTIFIER);
		if (followedId != null) {
			final Node followed = NeoUtils.getUserByIdentifier(followedId);
			if (followed != null) {
				return followed;
			}

			createFollowResponse.followedIdentifierInvalid(followedId);
		} else {
			createFollowResponse.followedIdentifierMissing();
		}

		return null;
	}
900963
@Override
	public void acknowledgeCheckpoint(
			final JobID jobID,
			final ExecutionAttemptID executionAttemptID,
			final long checkpointId,
			final CheckpointMetrics checkpointMetrics,
			final TaskStateSnapshot checkpointState) {

		final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
		final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint(
			jobID,
			executionAttemptID,
			checkpointId,
			checkpointMetrics,
			checkpointState);

		if (checkpointCoordinator != null) {
			getRpcService().execute(() -> {
				try {
					checkpointCoordinator.receiveAcknowledgeMessage(ackMessage);
				} catch (Throwable t) {
					log.warn("Error while processing checkpoint acknowledgement message", t);
				}
			});
		} else {
			String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator";
			if (executionGraph.getState() == JobStatus.RUNNING) {
				log.error(errorMessage, jobGraph.getJobID());
			} else {
				log.debug(errorMessage, jobGraph.getJobID());
			}
		}
	}
494414
function ReferencedIdentifier(node, parent, scope, state) {
    // we can't guarantee the purity of arguments
    if (node.name === "arguments") {
      state.deopted = true;
    }

    // is this a referenced identifier and is it referencing the rest parameter?
    if (node.name !== state.name) return;

    if (state.noOptimise) {
      state.deopted = true;
    } else {
      if (this.parentPath.isMemberExpression({ computed: true, object: node })) {
        // if we know that this member expression is referencing a number then we can safely
        // optimise it
        var prop = this.parentPath.get("property");
        if (prop.isBaseType("number")) {
          state.candidates.push(this);
          return;
        }
      }

      // optimise single spread args in calls
      if (this.parentPath.isSpreadElement() && state.offset === 0) {
        var call = this.parentPath.parentPath;
        if (call.isCallExpression() && call.node.arguments.length === 1) {
          state.candidates.push(this);
          return;
        }
      }

      state.references.push(this);
    }
  }
494093
def _serialiseExistingJob(self, jobGraph, jobStore, returnValues):
        
        self._serialiseJobGraph(jobGraph, jobStore, returnValues, False)
        #Drop the completed command, if not dropped already
        jobGraph.command = None
        #Merge any children (follow-ons) created in the initial serialisation
        #with children (follow-ons) created in the subsequent scale-up.
        assert len(jobGraph.stack) >= 4
        combinedChildren = jobGraph.stack[-1] + jobGraph.stack[-3]
        combinedFollowOns = jobGraph.stack[-2] + jobGraph.stack[-4]
        jobGraph.stack = jobGraph.stack[:-4]
        if len(combinedFollowOns) > 0:
            jobGraph.stack.append(combinedFollowOns)
        if len(combinedChildren) > 0:
            jobGraph.stack.append(combinedChildren)
85900
def makeJob(tool, jobobj, step_inputs, runtime_context):
    

    if tool.tool["class"] == "Workflow":
        wfjob = CWLWorkflow(tool, jobobj, runtime_context)
        followOn = ResolveIndirect(wfjob.rv())
        wfjob.addFollowOn(followOn)
        return (wfjob, followOn)
    else:
        # get_requirement
        resourceReq, _ = tool.get_requirement("ResourceRequirement")
        if resourceReq:
            for req in ("coresMin", "coresMax", "ramMin", "ramMax",
                        "tmpdirMin", "tmpdirMax", "outdirMin", "outdirMax"):
                r = resourceReq.get(req)
                if isinstance(r, string_types) and ("$(" in r or "${" in r):
                    # Found a dynamic resource requirement so use a job wrapper
                    job = CWLJobWrapper(tool, jobobj, runtime_context)
                    return (job, job)
        job = CWLJob(tool, jobobj, runtime_context)
        return (job, job)
947807
private void generateAllFailoverRegion(List<ExecutionJobVertex> newJobVerticesTopological) {
		final IdentityHashMap<ExecutionVertex, ArrayList<ExecutionVertex>> vertexToRegion = new IdentityHashMap<>();

		// we use the map (list -> null) to imitate an IdentityHashSet (which does not exist)
		final IdentityHashMap<ArrayList<ExecutionVertex>, Object> distinctRegions = new IdentityHashMap<>();

		// this loop will worst case iterate over every edge in the graph (complexity is O(#edges))
		
		for (ExecutionJobVertex ejv : newJobVerticesTopological) {

			// currently, jobs with a co-location constraint fail as one
			// we want to improve that in the future (or get rid of co-location constraints)
			if (ejv.getCoLocationGroup() != null) {
				makeAllOneRegion(newJobVerticesTopological);
				return;
			}

			// see if this JobVertex one has pipelined inputs at all
			final List<IntermediateResult> inputs = ejv.getInputs();
			final int numInputs = inputs.size();
			boolean hasPipelinedInputs = false;

			for (IntermediateResult input : inputs) {
				if (input.getResultType().isPipelined()) {
					hasPipelinedInputs = true;
					break;
				}
			}

			if (hasPipelinedInputs) {
				// build upon the predecessors
				for (ExecutionVertex ev : ejv.getTaskVertices()) {

					// remember the region in which we are
					ArrayList<ExecutionVertex> thisRegion = null;

					for (int inputNum = 0; inputNum < numInputs; inputNum++) {
						if (inputs.get(inputNum).getResultType().isPipelined()) {

							for (ExecutionEdge edge : ev.getInputEdges(inputNum)) {
								final ExecutionVertex predecessor = edge.getSource().getProducer();
								final ArrayList<ExecutionVertex> predecessorRegion = vertexToRegion.get(predecessor);

								if (thisRegion != null) {
									// we already have a region. see if it is the same as the predecessor's region
									if (predecessorRegion != thisRegion) {

										// we need to merge our region and the predecessor's region
										predecessorRegion.addAll(thisRegion);
										distinctRegions.remove(thisRegion);
										thisRegion = predecessorRegion;

										// remap the vertices from that merged region
										for (ExecutionVertex inPredRegion: predecessorRegion) {
											vertexToRegion.put(inPredRegion, thisRegion);
										}
									}
								}
								else if (predecessor != null) {
									// first case, make this our region
									thisRegion = predecessorRegion;
									thisRegion.add(ev);
									vertexToRegion.put(ev, thisRegion);
								}
								else {
									// throw an uncaught exception here
									// this is a bug and not a recoverable situation
									throw new FlinkRuntimeException(
											"bug in the logic to construct the pipelined failover regions");
								}
							}
						}
					}
				}
			}
			else {
				// no pipelined inputs, start a new region
				for (ExecutionVertex ev : ejv.getTaskVertices()) {
					ArrayList<ExecutionVertex> region = new ArrayList<>(1);
					region.add(ev);
					vertexToRegion.put(ev, region);
					distinctRegions.put(region, null);
				}
			}
		}

		// now that we have all regions, create the failover region objects 
		LOG.info("Creating {} individual failover regions for job {} ({})",
				distinctRegions.size(), executionGraph.getJobName(), executionGraph.getJobID());

		for (List<ExecutionVertex> region : distinctRegions.keySet()) {
			final FailoverRegion failoverRegion = createFailoverRegion(executionGraph, region);
			for (ExecutionVertex ev : region) {
				this.vertexToRegion.put(ev, failoverRegion);
			}
		}
	}
810022
function Job(job, provider, plugins, config) {
  var jobId = job._id.toString();

  debug(`Creating new job "${jobId}" for project "${job.project.display_name}" using provider "${provider.id}".`);

  this.config = _.extend({
    spawn: spawnNormal,
    pty: false,
    env: {}
  }, config);

  // Setup default ENVs to be used in plugin contexts
  this.config.env = _.extend(this.config.env, {
    STRIDER: true,
    STRIDER_TRIGGER: job.trigger.type || 'manual',
    STRIDER_COMMIT: '',
    STRIDER_PR: '',
    STRIDER_BRANCH: job.ref.branch,
    STRIDER_SLUG: job.project.name,
    STRIDER_BUILD: jobId,
    STRIDER_BUILD_URL: `${job.project.display_url}/job/${jobId}`
  });

  if (this.config.pty) {
    debug('PTY has been disabled due to a bug in node core');
    this.config.pty = false;
  }

  this.io = config.io;
  this.job = job;
  this.id = job._id;
  this.provider = provider;
  this.plugins = plugins;
  this.logPrefix = colorize.job(this.id);
  this.phase = null;
  this.testcode = null;
  this.cancelled = false;
  this.commands = [];
  this.listen();
}
788115
@POST
  @Produces(APPLICATION_JSON)
  @Timed
  @ExceptionMetered
  public CreateJobResponse post(@Valid final Job job,
                                @RequestUser final String username) {
    final Job.Builder clone = job.toBuilder()
        .setCreatingUser(username)
        .setCreated(clock.now().getMillis())
        // If the job had a hash coming in, preserve it
        .setHash(job.getId().getHash());
    final Job actualJob = clone.build();
    final Collection<String> errors = jobValidator.validate(actualJob);
    final String jobIdString = actualJob.getId().toString();
    if (!errors.isEmpty()) {
      throw badRequest(new CreateJobResponse(INVALID_JOB_DEFINITION, ImmutableList.copyOf(errors),
          jobIdString));
    }
    try {
      model.addJob(actualJob);
    } catch (JobExistsException e) {
      throw badRequest(new CreateJobResponse(JOB_ALREADY_EXISTS, ImmutableList.<String>of(),
          jobIdString));
    }
    log.info("created job: {}", actualJob);
    return new CreateJobResponse(CreateJobResponse.Status.OK, ImmutableList.<String>of(),
        jobIdString);
  }
1338173
public RuntimeJobExecution createJobStartExecution(final WSJobInstance jobInstance,
                                                       IJobXMLSource jslSource,
                                                       Properties jobParameters,
                                                       long executionId) throws JobStartException {

        // TODO - redesign to avoid cast?
        final JobInstanceEntity jobInstanceImpl = (JobInstanceEntity) jobInstance;
        long instanceId = jobInstance.getInstanceId();

        ModelNavigator<JSLJob> navigator = createFirstExecution(jobInstanceImpl, jslSource, jobParameters);
        JobExecutionEntity jobExec = null;
        try {
            jobExec = getPersistenceManagerService().getJobExecutionMostRecent(instanceId);
            // Check to make sure the executionId belongs to the most recent execution.
            // If not, a restart may have occurred. So, fail the start.
            // Also check to make sure a stop did not come in while the start was on the queue.
            BatchStatus currentBatchStatus = jobExec.getBatchStatus();
            if (jobExec.getExecutionId() != executionId ||
                currentBatchStatus.equals(BatchStatus.STOPPING) ||
                currentBatchStatus.equals(BatchStatus.STOPPED)) {
                throw new JobStartException();
            }

        } catch (IllegalStateException ie) {
            // If no execution exists, request came from old dispatch path.  
            jobExec = getPersistenceManagerService().createJobExecution(instanceId, jobParameters, new Date());
            BatchEventsPublisher eventsPublisher = batchKernelImpl.getBatchEventsPublisher();
            if (eventsPublisher != null) {
                String correlationId = getCorrelationId(jobParameters);
                eventsPublisher.publishJobExecutionEvent(jobExec, BatchEventsPublisher.TOPIC_EXECUTION_STARTING, correlationId);
            }
        }

        TopLevelNameInstanceExecutionInfo topLevelInfo = new TopLevelNameInstanceExecutionInfo(jobInstanceImpl.getJobName(), instanceId, jobExec.getExecutionId());

        return new RuntimeJobExecution(topLevelInfo, jobParameters, navigator);
    }
149243
@Override
  public void addJob(final Job job) throws JobExistsException {
    log.info("adding job: {}", job);
    final JobId id = job.getId();
    final UUID operationId = UUID.randomUUID();
    final String creationPath = Paths.configJobCreation(id, operationId);
    final ZooKeeperClient client = provider.get("addJob");
    try {
      try {
        client.ensurePath(Paths.historyJob(id));
        client.transaction(create(Paths.configJob(id), job),
            create(Paths.configJobRefShort(id), id),
            create(Paths.configJobHosts(id)),
            create(creationPath),
            // Touch the jobs root node so that its version is bumped on every job
            // change down the tree. Effectively, make it that version == cVersion.
            set(Paths.configJobs(), UUID.randomUUID().toString().getBytes()));
      } catch (final NodeExistsException e) {
        if (client.exists(creationPath) != null) {
          // The job was created, we're done here
          return;
        }
        throw new JobExistsException(id.toString());
      }
    } catch (NoNodeException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed due to missing ZK path: "
                                       + e.getPath(), e);
    } catch (final KeeperException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed", e);
    }
  }
1294858
private synchronized RunningJobImpl get(final String jobIdentifier) {
    final RunningJobImpl result = this.jobs.get(jobIdentifier);
    if (null == result) {
      throw new RuntimeException("Trying to get a RunningJob that is unknown: " + jobIdentifier);
    }
    return result;
  }
714087
public StepThreadExecutionEntity updateStepExecutionOnRecovery(PersistenceServiceUnit psu,
                                                                   final long stepExecutionId,
                                                                   final BatchStatus newStepBatchStatus,
                                                                   final String newStepExitStatus,
                                                                   final Date endTime) throws IllegalArgumentException {

        EntityManager em = psu.createEntityManager();
        try {
            return new TranRequest<StepThreadExecutionEntity>(em) {
                @Override
                public StepThreadExecutionEntity call() {

                    StepThreadExecutionEntity stepExec = entityMgr.find(StepThreadExecutionEntity.class, stepExecutionId);
                    if (stepExec == null) {
                        throw new IllegalArgumentException("StepThreadExecEntity with id =" + stepExecutionId + " should be persisted at this point, but didn't find it.");
                    }

                    try {
                        verifyThreadStatusTransitionIsValid(stepExec, newStepBatchStatus);
                    } catch (BatchIllegalJobStatusTransitionException e) {
                        throw new PersistenceException(e);
                    }

                    stepExec.setBatchStatus(newStepBatchStatus);
                    stepExec.setExitStatus(newStepExitStatus);
                    stepExec.setEndTime(endTime);
                    return stepExec;
                }
            }.runInNewOrExistingGlobalTran();
        } finally {
            em.close();
        }
    }
349448
private TopLevelStepExecutionEntity getStepExecutionTopLevelFromJobExecutionIdAndStepName(long jobExecutionId, String stepName) {
        JobExecutionEntity jobExecution = data.executionInstanceData.get(jobExecutionId);
        // Copy list to avoid ConcurrentModificationException
        for (StepThreadExecutionEntity stepExec : new ArrayList<StepThreadExecutionEntity>(jobExecution.getStepThreadExecutions())) {
            if (stepExec.getStepName().equals(stepName)) {
                if (stepExec instanceof TopLevelStepExecutionEntity) {
                    return (TopLevelStepExecutionEntity) stepExec;
                }
            }
        }
        // Bad if we've gotten here.
        throw new IllegalStateException("Couldn't find top-level step execution for jobExecution = " + jobExecutionId + ", and stepName = " + stepName);
    }