Creates {@link TaskDescriptor} for stream combine operations (i.e., join, union, unionAll)
972465
private TaskDescriptor createTaskDescriptorForStreamCombineOperations(DStreamOperation streamOperation){

		TaskDescriptor taskDescriptor = this.createTaskDescriptor(streamOperation.getLastOperationName());
		for (DStreamExecutionGraph dependentOps : streamOperation.getCombinableExecutionGraphs()) {
			TaskDescriptorChainBuilder builder = new TaskDescriptorChainBuilder(executionName, dependentOps, executionConfig);
			List<TaskDescriptor> dependentDescriptors = builder.build();
			taskDescriptor.addDependentTasksChain(dependentDescriptors);
		}
		return taskDescriptor;
	}
498709
static <T extends IncrementalTask> IncrementalTaskState<T> create(
      String taskId, String jobId, long createTime, T initialTask) {
    return new IncrementalTaskState<>(taskId, jobId, createTime, new LockInfo(null, null),
        checkNotNull(initialTask), new Status(StatusCode.RUNNING));
  }
797835
private JobConfig.Builder createJob(List<WorkUnit> workUnits) throws IOException {
    Map<String, TaskConfig> taskConfigMap = Maps.newHashMap();

    try (ParallelRunner stateSerDeRunner = new ParallelRunner(this.stateSerDeRunnerThreads, this.fs)) {
      int multiTaskIdSequence = 0;
      for (WorkUnit workUnit : workUnits) {
        if (workUnit instanceof MultiWorkUnit) {
          workUnit.setId(JobLauncherUtils.newMultiTaskId(this.jobContext.getJobId(), multiTaskIdSequence++));
        }
        addWorkUnit(workUnit, stateSerDeRunner, taskConfigMap);
      }

      Path jobStateFilePath;

      // write the job.state using the state store if present, otherwise serialize directly to the file
      if (this.stateStores.haveJobStateStore()) {
        jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(true, this.appWorkDir, this.jobContext.getJobId());
        this.stateStores.getJobStateStore().put(jobStateFilePath.getParent().getName(), jobStateFilePath.getName(),
            this.jobContext.getJobState());
      } else {
        jobStateFilePath = GobblinClusterUtils.getJobStateFilePath(false, this.appWorkDir, this.jobContext.getJobId());
        SerializationUtils.serializeState(this.fs, jobStateFilePath, this.jobContext.getJobState());
      }

      LOGGER.debug("GobblinHelixJobLauncher.createJob: jobStateFilePath {}, jobState {} jobProperties {}",
          jobStateFilePath, this.jobContext.getJobState().toString(), this.jobContext.getJobState().getProperties());
    }

    JobConfig.Builder jobConfigBuilder = new JobConfig.Builder();

    // Helix task attempts = retries + 1 (fallback to general task retry for backward compatibility)
    jobConfigBuilder.setMaxAttemptsPerTask(this.jobContext.getJobState().getPropAsInt(
        GobblinClusterConfigurationKeys.HELIX_TASK_MAX_ATTEMPTS_KEY,
        this.jobContext.getJobState().getPropAsInt(
            ConfigurationKeys.MAX_TASK_RETRIES_KEY,
            ConfigurationKeys.DEFAULT_MAX_TASK_RETRIES)) + 1);

    // Helix task timeout (fallback to general task timeout for backward compatibility)
    jobConfigBuilder.setTimeoutPerTask(this.jobContext.getJobState().getPropAsLong(
        GobblinClusterConfigurationKeys.HELIX_TASK_TIMEOUT_SECONDS,
        this.jobContext.getJobState().getPropAsLong(
            ConfigurationKeys.TASK_TIMEOUT_SECONDS,
            ConfigurationKeys.DEFAULT_TASK_TIMEOUT_SECONDS)) * 1000);

    jobConfigBuilder.setFailureThreshold(workUnits.size());
    jobConfigBuilder.addTaskConfigMap(taskConfigMap).setCommand(GobblinTaskRunner.GOBBLIN_TASK_FACTORY_NAME);
    jobConfigBuilder.setNumConcurrentTasksPerInstance(ConfigUtils.getInt(jobConfig,
        GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY,
        GobblinClusterConfigurationKeys.HELIX_CLUSTER_TASK_CONCURRENCY_DEFAULT));

    if (this.jobConfig.hasPath(GobblinClusterConfigurationKeys.HELIX_JOB_TAG_KEY)) {
      String jobTag = this.jobConfig.getString(GobblinClusterConfigurationKeys.HELIX_JOB_TAG_KEY);
      log.info("Job {} has tags associated : {}", this.jobContext.getJobId(), jobTag);
      jobConfigBuilder.setInstanceGroupTag(jobTag);
    }

    if (this.jobConfig.hasPath(GobblinClusterConfigurationKeys.HELIX_JOB_TYPE_KEY)) {
      String jobType = this.jobConfig.getString(this.jobConfig.getString(GobblinClusterConfigurationKeys.HELIX_JOB_TYPE_KEY));
      log.info("Job {} has types associated : {}", this.jobContext.getJobId(), jobType);
      jobConfigBuilder.setJobType(jobType);
    }

    if (Task.getExecutionModel(ConfigUtils.configToState(jobConfig)).equals(ExecutionModel.STREAMING)) {
      jobConfigBuilder.setRebalanceRunningTask(true);
    }

    jobConfigBuilder.setExpiry(this.jobContext.getJobState().getPropAsLong(
        GobblinClusterConfigurationKeys.HELIX_WORKFLOW_EXPIRY_TIME_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_EXPIRY_TIME_SECONDS));

    return jobConfigBuilder;
  }
490918
public Stream<T> merge(final Iterable<? extends T> with) {
        return new Stream<T>() {
            @Override
            public Iterator<T> iterator() {
                return new ReadOnlyIterator<T>() {

                    Iterator<T> iterator = Stream.this.iterator();
                    Iterator<? extends T> withIterator = with.iterator();

                    @Override
                    public boolean hasNext() {
                        return iterator.hasNext() || withIterator.hasNext();
                    }

                    @Override
                    public T next() {
                        return iterator.hasNext() ? iterator.next() : withIterator.next();
                    }
                };
            }
        };
    }
1344365
@Override
    public final TaskProcessor create(Task task)
        throws TaskProcessorCreationFailedException {
        if (isSupported(task)) {
            log.debug("creating task processor for " + task);
            AuditTask auditTask = new AuditTask();
            auditTask.readTask(task);
            return createImpl(auditTask);
        }

        throw new TaskProcessorCreationFailedException("Task is not an Audit task");

    }
627262
@Override
  public StreamBuilderImpl<T,U> reduce(BinaryOperatorSync<T> op)
  {
    return new ReduceOpSync<>(this, op);
  }
243204
public TaskExecutorBuilder taskDecorator(TaskDecorator taskDecorator) {
		return new TaskExecutorBuilder(this.queueCapacity, this.corePoolSize,
				this.maxPoolSize, this.allowCoreThreadTimeOut, this.keepAlive,
				this.awaitTermination, this.awaitTerminationPeriod, this.threadNamePrefix,
				taskDecorator, this.customizers);
	}
42594
function createTask(taskId) {
	return function (done) {
		var stream = tasks[taskId].fn(gulp, done);
		if (stream) {
			if (stream.on) stream.on('end', done);
			if (stream.then) stream.then(done);
		}
	};
}
426671
private JobTaskVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException {
		final String taskName = node.getNodeName();
		final DriverStrategy ds = node.getDriverStrategy();
		
		// check, whether chaining is possible
		boolean chaining = false;
		{
			Channel inConn = node.getInput();
			PlanNode pred = inConn.getSource();
			chaining = ds.getPushChainDriverClass() != null &&
					!(pred instanceof NAryUnionPlanNode) &&	// first op after union is stand-alone, because union is merged
					!(pred instanceof BulkPartialSolutionPlanNode) &&	// partial solution merges anyways
					!(pred instanceof WorksetPlanNode) &&	// workset merges anyways
					!(pred instanceof IterationPlanNode) && // cannot chain with iteration heads currently
					inConn.getShipStrategy() == ShipStrategyType.FORWARD &&
					inConn.getLocalStrategy() == LocalStrategy.NONE &&
					pred.getOutgoingChannels().size() == 1 &&
					node.getDegreeOfParallelism() == pred.getDegreeOfParallelism() && 
					node.getSubtasksPerInstance() == pred.getSubtasksPerInstance() &&
					node.getBroadcastInputs().isEmpty();
			
			// cannot chain the nodes that produce the next workset or the next solution set, if they are not the
			// in a tail 
			if (this.currentIteration != null && this.currentIteration instanceof WorksetIterationPlanNode &&
					node.getOutgoingChannels().size() > 0)
			{
				WorksetIterationPlanNode wspn = (WorksetIterationPlanNode) this.currentIteration;
				if (wspn.getSolutionSetDeltaPlanNode() == pred || wspn.getNextWorkSetPlanNode() == pred) {
					chaining = false;
				}
			}
			// cannot chain the nodes that produce the next workset in a bulk iteration if a termination criterion follows
			if (this.currentIteration != null && this.currentIteration instanceof BulkIterationPlanNode)
			{
				BulkIterationPlanNode wspn = (BulkIterationPlanNode) this.currentIteration;
				if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){
					chaining = false;
				}else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred ||
						wspn.getRootOfTerminationCriterion() == pred)) {
					chaining = false;
				}
			}
		}
		
		final JobTaskVertex vertex;
		final TaskConfig config;
		
		if (chaining) {
			vertex = null;
			config = new TaskConfig(new Configuration());
			this.chainedTasks.put(node, new TaskInChain(ds.getPushChainDriverClass(), config, taskName));
		} else {
			// create task vertex
			vertex = new JobTaskVertex(taskName, this.jobGraph);
			vertex.setTaskClass( (this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediatePactTask.class : RegularPactTask.class);
			
			config = new TaskConfig(vertex.getConfiguration());
			config.setDriver(ds.getDriverClass());
		}
		
		// set user code
		config.setStubWrapper(node.getPactContract().getUserCodeWrapper());
		config.setStubParameters(node.getPactContract().getParameters());
		
		// set the driver strategy
		config.setDriverStrategy(ds);
		if (node.getComparator() != null) {
			config.setDriverComparator(node.getComparator(), 0);
		}
		
		// assign memory, file-handles, etc.
		assignDriverResources(node, config);
		return vertex;
	}
672170
public static <T> CombinedDownloadTask<T> single(DownloadTask<T> task) {
		Objects.requireNonNull(task);
		return new SingleCombinedTask<T>(task);
	}
945392
public static TaskGroupSpecification.Builder with(TaskGroupInformation info) {
		return new TaskGroupSpecification.Builder(info.getInputType(), info.getOutputType(), Objects.requireNonNull(info.getLocale()), info.getActivity());
	}
1067913
public Task createTask(String name, Map<String, Object> attributes) {
        return getInstance().create().task(name, this, attributes);
    }
850041
public <T0, T1, T2, T3, T4> SingleOutputStreamOperator<Tuple5<T0, T1, T2, T3, T4>> projectTuple5() {
		TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, dataStream.getType());
		TupleTypeInfo<Tuple5<T0, T1, T2, T3, T4>> tType = new TupleTypeInfo<Tuple5<T0, T1, T2, T3, T4>>(fTypes);

		return dataStream.transform("Projection", tType, new StreamProject<IN, Tuple5<T0, T1, T2, T3, T4>>(fieldIndexes, tType.createSerializer(dataStream.getExecutionConfig())));
	}
1089319
public static <T> JoinObservable<T> from(Observable<T> o) {
        return new JoinObservable<T>(o);
    }
683229
public <T0, T1, T2, T3> SingleOutputStreamOperator<Tuple4<T0, T1, T2, T3>> projectTuple4() {
		TypeInformation<?>[] fTypes = extractFieldTypes(fieldIndexes, dataStream.getType());
		TupleTypeInfo<Tuple4<T0, T1, T2, T3>> tType = new TupleTypeInfo<Tuple4<T0, T1, T2, T3>>(fTypes);

		return dataStream.transform("Projection", tType, new StreamProject<IN, Tuple4<T0, T1, T2, T3>>(fieldIndexes, tType.createSerializer(dataStream.getExecutionConfig())));
	}
598684
public static <T> String join(Iterator<T> iterator, CharSequence conjunction) {
		return join(iterator, conjunction, null, null);
	}
452069
public ApiDescriptionBuilder operations(List<Operation> operations) {
    if (operations != null) {
      this.operations = operations.stream().sorted(operationOrdering).collect(toList());
    }
    return this;
  }