creates {@link Put} for inserting additional info into agg tables
@param {@link AppAggregationKey}
@param {@link JobDetails}
@return {@link Put}
@throws IOException
970257
boolean updateMoreAggInfo(Table aggTable, AppAggregationKey appAggKey,
      JobDetails jobDetails) throws IOException {

    int attempts = 0;
    boolean status = false;
    // update number of runs
    while ((!status) && (attempts < AggregationConstants.RETRY_COUNT)) {
      status = updateNumberRuns(appAggKey, aggTable, jobDetails);
      attempts++;
    }

    // update queue
    attempts = 0;
    status = false;
    while ((!status) && (attempts < AggregationConstants.RETRY_COUNT)) {
      status = updateQueue(appAggKey, aggTable, jobDetails);
      attempts++;
    }

    // update cost
    attempts = 0;
    status = false;
    while ((!status) && (attempts < AggregationConstants.RETRY_COUNT)) {
      status = updateCost(appAggKey, aggTable, jobDetails);
      attempts++;
    }

    // create a put for username, appid
    byte[] rowKey = aggConv.toBytes(appAggKey);
    Put p = new Put(rowKey);
    p.addColumn(Constants.INFO_FAM_BYTES, AggregationConstants.USER_BYTES,
        Bytes.toBytes(appAggKey.getUserName()));
    p.addColumn(Constants.INFO_FAM_BYTES, AggregationConstants.APP_ID_COL_BYTES,
        Bytes.toBytes(appAggKey.getAppId()));
    aggTable.put(p);
    return true;
  }
934253
final protected void common_put(
                                    Object key,
                                    Object value,
                                    Object userMetaData,
                                    int priority,
                                    int timeToLive,
                                    int sharingPolicy,
                                    Object dependencyIds[],
                                    Object aliasIds[]) {
        try {
            internal_putAndGet(key, value, userMetaData, priority, timeToLive, -1, sharingPolicy, dependencyIds, aliasIds, !INCREMENT_REF_COUNT, !SKIP_MEMORY_WRITE_TO_DISK);
        } catch (DynamicCacheException ex) {
            com.ibm.ws.ffdc.FFDCFilter.processException(ex, "com.ibm.ws.cache.DistribtedObjectCacheAdapter", "669", this);
            if (tc.isDebugEnabled())
                Tr.debug(tc, "common_put - exception: " + ExceptionUtility.getStackTrace(ex));
        }
    }
163238
String createQueueListValue(JobDetails jobDetails, String existingQueues) {
    /*
     * check if queue already exists append separator at the end to avoid
     * "false" queue match via substring match
     */
    String queue = jobDetails.getQueue();
    queue = queue.concat(Constants.SEP);

    if (existingQueues == null) {
      return queue;
    }

    if (!existingQueues.contains(queue)) {
      existingQueues = existingQueues.concat(queue);
    }
    return existingQueues;
  }
608689
public void addProperty(JobProperty<? super JobT> jobProp) throws IOException {
        ((JobProperty)jobProp).setOwner(this);
        properties.add(jobProp);
        save();
    }
930552
def put_job(self, id, body, params=None):
        
        for param in (id, body):
            if param in SKIP_IN_PATH:
                raise ValueError("Empty value passed for a required argument.")
        return self.transport.perform_request(
            "PUT", _make_path("_rollup", "job", id), params=params, body=body
        )
910613
@Override
	public void setJobRunning(JobID jobID) throws IOException {
		checkNotNull(jobID, "jobID");
		final Path filePath = createMarkerFilePath(RUNNING_PREFIX, jobID);

		// create the file
		// to avoid an exception if the job already exists, set overwrite=true
		createFile(filePath, true);
	}
801258
@Override
  public void commitJob(JobContext context) throws IOException {
    super.commitJob(context);

    // Get the destination configuration information.
    Configuration conf = context.getConfiguration();
    TableReference destTable = BigQueryOutputConfiguration.getTableReference(conf);
    String destProjectId = BigQueryOutputConfiguration.getProjectId(conf);
    String writeDisposition = BigQueryOutputConfiguration.getWriteDisposition(conf);
    Optional<BigQueryTableSchema> destSchema = BigQueryOutputConfiguration.getTableSchema(conf);
    String kmsKeyName = BigQueryOutputConfiguration.getKmsKeyName(conf);
    BigQueryFileFormat outputFileFormat = BigQueryOutputConfiguration.getFileFormat(conf);
    List<String> sourceUris = getOutputFileURIs();

    try {
      getBigQueryHelper()
          .importFromGcs(
              destProjectId,
              destTable,
              destSchema.isPresent() ? destSchema.get().get() : null,
              kmsKeyName,
              outputFileFormat,
              writeDisposition,
              sourceUris,
              true);
    } catch (InterruptedException e) {
      throw new IOException("Failed to import GCS into BigQuery", e);
    }

    cleanup(context);
  }
1175600
@VisibleForTesting
  static JSONObject handleGetJobDetail(String jobId) {
    ShardedJobService shardedJobService = ShardedJobServiceFactory.getShardedJobService();
    ShardedJobState state = shardedJobService.getJobState(jobId);
    if (state == null) {
      return null;
    }
    JSONObject jobObject = new JSONObject();
    try {
      jobObject.put("name", jobId); // For display
      jobObject.put("mapreduce_id", jobId); // This is the sharedJobId but it needs be be called
                                            // mapreduce_id for python compatibility.
      jobObject.put("start_timestamp_ms", state.getStartTimeMillis());

      if (state.getStatus().isActive()) {
        jobObject.put("active", true);
        jobObject.put("updated_timestamp_ms", System.currentTimeMillis());
      } else {
        jobObject.put("active", false);
        jobObject.put("result_status", String.valueOf(state.getStatus().getStatusCode()));
        jobObject.put("updated_timestamp_ms", state.getMostRecentUpdateTimeMillis());
      }
      jobObject.put("shards", state.getTotalTaskCount());
      jobObject.put("active_shards", state.getActiveTaskCount());

      JSONObject mapperParams = new JSONObject();
      mapperParams.put("Shards completed",
          state.getTotalTaskCount() - state.getActiveTaskCount());
      mapperParams.put("Shards active", state.getActiveTaskCount());
      mapperParams.put("Shards total", state.getTotalTaskCount());
      JSONObject mapperSpec = new JSONObject();
      mapperSpec.put("mapper_params", mapperParams);
      jobObject.put("mapper_spec", mapperSpec);

      JSONArray shardArray = new JSONArray();
      Counters totalCounters = new CountersImpl();
      int i = 0;
      long[] workerCallCounts = new long[state.getTotalTaskCount()];
      Iterator<IncrementalTaskState<IncrementalTask>> tasks = shardedJobService.lookupTasks(state);
      while (tasks.hasNext()) {
        IncrementalTaskState<?> taskState = tasks.next();
        JSONObject shardObject = new JSONObject();
        shardObject.put("shard_number", i);
        shardObject.put("shard_description", taskState.getTaskId());
        shardObject.put("updated_timestamp_ms", taskState.getMostRecentUpdateMillis());
        if (taskState.getStatus().isActive()) {
          shardObject.put("active", true);
        } else {
          shardObject.put("active", false);
          shardObject.put("result_status", taskState.getStatus().getStatusCode());
        }
        IncrementalTask task = taskState.getTask();
        if (task instanceof IncrementalTaskWithContext) {
          IncrementalTaskContext context = ((IncrementalTaskWithContext) task).getContext();
          totalCounters.addAll(context.getCounters());
          workerCallCounts[i] = context.getWorkerCallCount();
          shardObject.put("last_work_item", context.getLastWorkItemString());
        }
        shardArray.put(shardObject);
        i++;
      }
      jobObject.put("counters", toJson(totalCounters));
      jobObject.put("shards", shardArray);
      jobObject.put("chart_width", getChartWidth(state.getTotalTaskCount()));
      jobObject.put("chart_url", getChartUrl(workerCallCounts));
      jobObject.put("chart_data", workerCallCounts);
    } catch (JSONException e) {
      throw new RuntimeException("Hard coded string is null", e);
    }

    return jobObject;
  }
790755
@Override
  public final void setupJob(org.apache.hadoop.mapreduce.JobContext jobContext
                             ) throws IOException {
    setupJob((JobContext) jobContext);
  }
127851
@Deprecated
  public void push(CollectorRegistry registry, String job, String instance) throws IOException {
    push(registry, job, Collections.singletonMap("instance", instance));
  }
1184536
public JobDetails getJobByJobID(QualifiedJobId jobId, boolean populateTasks)
      throws IOException {
    JobDetails job = null;
    JobKey key = idService.getJobKeyById(jobId);
    if (key != null) {
      byte[] historyKey = jobKeyConv.toBytes(key);
      Table historyTable =
          hbaseConnection.getTable(TableName.valueOf(Constants.HISTORY_TABLE));
      Result result = historyTable.get(new Get(historyKey));
      historyTable.close();
      if (result != null && !result.isEmpty()) {
        job = new JobDetails(key);
        job.populate(result);
        if (populateTasks) {
          populateTasks(job);
        }
      }
    }
    return job;
  }
526084
public JobBuilder usingJobData (final String dataKey, final String value)
  {
    m_aJobDataMap.put (dataKey, value);
    return this;
  }
129651
public TaskMetricGroup addTaskForJob(
			final JobID jobId,
			final String jobName,
			final JobVertexID jobVertexId,
			final ExecutionAttemptID executionAttemptId,
			final String taskName,
			final int subtaskIndex,
			final int attemptNumber) {
		Preconditions.checkNotNull(jobId);

		String resolvedJobName = jobName == null || jobName.isEmpty()
			? jobId.toString()
			: jobName;

		// we cannot strictly lock both our map modification and the job group modification
		// because it might lead to a deadlock
		while (true) {
			// get or create a jobs metric group
			TaskManagerJobMetricGroup currentJobGroup;
			synchronized (this) {
				currentJobGroup = jobs.get(jobId);

				if (currentJobGroup == null || currentJobGroup.isClosed()) {
					currentJobGroup = new TaskManagerJobMetricGroup(registry, this, jobId, resolvedJobName);
					jobs.put(jobId, currentJobGroup);
				}
			}

			// try to add another task. this may fail if we found a pre-existing job metrics
			// group and it is closed concurrently
			TaskMetricGroup taskGroup = currentJobGroup.addTask(
				jobVertexId,
				executionAttemptId,
				taskName,
				subtaskIndex,
				attemptNumber);

			if (taskGroup != null) {
				// successfully added the next task
				return taskGroup;
			}

			// else fall through the loop
		}
	}
367083
@Override
	public <P extends ParaObject> void createAll(String appid, List<P> objects) {
		if (objects == null || objects.isEmpty() || StringUtils.isBlank(appid)) {
			return;
		}

		List<WriteRequest> reqs = new ArrayList<>(objects.size());
		int batchSteps = 1;
		if ((objects.size() > MAX_ITEMS_PER_WRITE)) {
			batchSteps = (objects.size() / MAX_ITEMS_PER_WRITE) +
					((objects.size() % MAX_ITEMS_PER_WRITE > 0) ? 1 : 0);
		}

		Iterator<P> it = objects.iterator();
		String tableName = getTableNameForAppid(appid);
		int j = 0;

		for (int i = 0; i < batchSteps; i++) {
			while (it.hasNext() && j < MAX_ITEMS_PER_WRITE) {
				ParaObject object = it.next();
				if (StringUtils.isBlank(object.getId())) {
					object.setId(Utils.getNewId());
				}
				if (object.getTimestamp() == null) {
					object.setTimestamp(Utils.timestamp());
				}
				//if (updateOp) object.setUpdated(Utils.timestamp());
				object.setAppid(appid);
				Map<String, AttributeValue> row = toRow(object, null);
				setRowKey(getKeyForAppid(object.getId(), appid), row);
				reqs.add(new WriteRequest().withPutRequest(new PutRequest().withItem(row)));
				j++;
			}
			batchWrite(Collections.singletonMap(tableName, reqs), 1);
			reqs.clear();
			j = 0;
		}
		logger.debug("DAO.createAll() {}->{}", appid, (objects == null) ? 0 : objects.size());
	}
1221101
protected Job postJobWithDescription(JSONObject jobDescription, long notBeforeRealtimeElapsed) {
        String jobId = UUID.randomUUID().toString();
        InternalJob job = new InternalJob(jobId, jobDescription, notBeforeRealtimeElapsed);
        return post(job);
    }
132904
@Override
	public JobInstance createSubJobInstance(String name, String apptag) {
		Connection conn = null;
		PreparedStatement statement = null;
		ResultSet rs = null;
		JobInstanceImpl jobInstance = null;

		try {
			conn = getConnection();
			statement = conn.prepareStatement("INSERT INTO jobinstancedata (name, apptag) VALUES(?, ?)", new String[] { "JOBINSTANCEID" } );
			statement.setString(1, name);
			statement.setString(2, apptag); 
			statement.executeUpdate();
			rs = statement.getGeneratedKeys();
			if(rs.next()) {
				long jobInstanceID = rs.getLong(1);
				jobInstance = new JobInstanceImpl(jobInstanceID);
				jobInstance.setJobName(name);
			}
		} catch (SQLException e) {
			throw new PersistenceException(e);
		} finally {
			cleanupConnection(conn, rs, statement);
		}
		return jobInstance;
	}
921048
@Override
    public Put createPut(Record record, String tableName,
            String famName, String quaName) throws IOException {
        HTable hTable = getTable(tableName);
        Put put = null;
        if (hTable != null) {
            // Transforming data model record into an Avro record
            AvroSerializer serializer = getSerializer();
            final byte[] bytes = serializer.toBytes(record);
            // Resource's Key
            put = new Put(Bytes.toBytes(record.getID().toString()));
            // Resource's Value
            put.add(Bytes.toBytes(famName), Bytes.toBytes(quaName), bytes);
        }
        return put;
    }