Sets the Bigquery access related fields in the JobConf for input connector.

@param config the job configuration.
@param fullyQualifiedInputTableId input-table id of the form
[optional projectId]:[datasetId].[tableId]
493638
public static void configureBigQueryInput(
      Configuration config, String fullyQualifiedInputTableId)
      throws IOException {
    TableReference parsedTable = BigQueryStrings.parseTableReference(fullyQualifiedInputTableId);
    configureBigQueryInput(
        config, parsedTable.getProjectId(), parsedTable.getDatasetId(), parsedTable.getTableId());
  }
599717
public void configureJobQuery(JobQueryImpl query) {
    configureQuery(query);
    CompositePermissionCheck permissionCheck = new PermissionCheckBuilder()
        .disjunctive()
        .atomicCheck(PROCESS_INSTANCE, "RES.PROCESS_INSTANCE_ID_", READ)
        .atomicCheck(PROCESS_DEFINITION, "RES.PROCESS_DEF_KEY_", READ_INSTANCE)
        .build();
    addPermissionCheck(query.getAuthCheck(), permissionCheck);
  }
730542
public Configuration getRawJobConfiguration(QualifiedJobId jobId)
      throws IOException {
    Configuration conf = null;
    byte[] rowKey = idConv.toBytes(jobId);
    Get get = new Get(rowKey);
    get.addColumn(Constants.RAW_FAM_BYTES, Constants.JOBCONF_COL_BYTES);
    Table rawTable = null;
    try {
      rawTable = hbaseConnection
          .getTable(TableName.valueOf(Constants.HISTORY_RAW_TABLE));
      Result result = rawTable.get(get);
      if (result != null && !result.isEmpty()) {
        conf = createConfigurationFromResult(result);
      }
    } catch (MissingColumnInResultException e) {
      LOG.error(
          "Failed to retrieve configuration from row returned for " + jobId, e);
    } finally {
      if (rawTable != null) {
        rawTable.close();
      }
    }
    return conf;
  }
939133
public static void setInputChoiceStrategy(Job job, Class<? extends FromCobolChoiceStrategy> choiceStrategyClass) {
      job.getConfiguration().setClass(CONF_INPUT_RECORD_CHOICE_STRATEGY_CLASS, choiceStrategyClass, FromCobolChoiceStrategy.class);
    }
63035
public static void setOutput(JobConf job, String tableName, String... fieldNames) {
    job.setOutputFormat(DBOutputFormat.class);
    job.setReduceSpeculativeExecution(false);

    DBConfiguration dbConf = new DBConfiguration(job);
    
    dbConf.setOutputTableName(tableName);
    dbConf.setOutputFieldNames(fieldNames);
  }
801258
@Override
  public void commitJob(JobContext context) throws IOException {
    super.commitJob(context);

    // Get the destination configuration information.
    Configuration conf = context.getConfiguration();
    TableReference destTable = BigQueryOutputConfiguration.getTableReference(conf);
    String destProjectId = BigQueryOutputConfiguration.getProjectId(conf);
    String writeDisposition = BigQueryOutputConfiguration.getWriteDisposition(conf);
    Optional<BigQueryTableSchema> destSchema = BigQueryOutputConfiguration.getTableSchema(conf);
    String kmsKeyName = BigQueryOutputConfiguration.getKmsKeyName(conf);
    BigQueryFileFormat outputFileFormat = BigQueryOutputConfiguration.getFileFormat(conf);
    List<String> sourceUris = getOutputFileURIs();

    try {
      getBigQueryHelper()
          .importFromGcs(
              destProjectId,
              destTable,
              destSchema.isPresent() ? destSchema.get().get() : null,
              kmsKeyName,
              outputFileFormat,
              writeDisposition,
              sourceUris,
              true);
    } catch (InterruptedException e) {
      throw new IOException("Failed to import GCS into BigQuery", e);
    }

    cleanup(context);
  }
758925
func (s *DocumentClassifierProperties) SetInputDataConfig(v *DocumentClassifierInputDataConfig) *DocumentClassifierProperties {
	s.InputDataConfig = v
	return s
}
777147
public static void setInputPathFilter(JobConf conf,
                                        Class<? extends PathFilter> filter) {
    conf.setClass("mapred.input.pathFilter.class", filter, PathFilter.class);
  }
585566
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
    conf.setIfUnset("hbase.client.connection.impl",
      BigtableConfiguration.getConnectionClass().getName());
    conf.setIfUnset(BigtableOptionsFactory.BIGTABLE_RPC_TIMEOUT_MS_KEY, "60000");
    conf.setBoolean(TableInputFormat.SHUFFLE_MAPS, true);

    String tableName = args[0];
    Path outputDir = new Path(args[1]);
    Job job = Job.getInstance(conf, NAME + "_" + tableName);
    job.setJobName(NAME + "_" + tableName);
    job.setJarByClass(Export.class);
    // Set optional scan parameters
    Scan s = getConfiguredScanForJob(conf, args);
    TableMapReduceUtil.initTableMapperJob(tableName, s, IdentityTableMapper.class, ImmutableBytesWritable.class, Result.class, job, false);
    // No reducers.  Just write straight to output files.
    job.setNumReduceTasks(0);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(ImmutableBytesWritable.class);
    job.setOutputValueClass(Result.class);
    FileOutputFormat.setOutputPath(job, outputDir); // job conf doesn't contain the conf so doesn't have a default fs.
    return job;
  }
1340878
public static void writeConfig(BufferedWriter writer, DatabaseFieldConfig config, String tableName)
			throws IOException {
		writer.append(CONFIG_FILE_START_MARKER);
		writer.newLine();
		if (config.getFieldName() != null) {
			writer.append(FIELD_NAME_FIELD_NAME).append('=').append(config.getFieldName());
			writer.newLine();
		}
		if (config.getColumnName() != null) {
			writer.append(FIELD_NAME_COLUMN_NAME).append('=').append(config.getColumnName());
			writer.newLine();
		}
		if (config.getDataPersister() != DEFAULT_DATA_PERSISTER) {
			boolean found = false;
			for (DataType dataType : DataType.values()) {
				if (dataType.getDataPersister() == config.getDataPersister()) {
					writer.append(FIELD_NAME_DATA_PERSISTER).append('=').append(dataType.name());
					writer.newLine();
					found = true;
					break;
				}
			}
			if (!found) {
				throw new IllegalArgumentException("Unknown data persister field: " + config.getDataPersister());
			}
		}
		if (config.getDefaultValue() != null) {
			writer.append(FIELD_NAME_DEFAULT_VALUE).append('=').append(config.getDefaultValue());
			writer.newLine();
		}
		if (config.getWidth() != 0) {
			writer.append(FIELD_NAME_WIDTH).append('=').append(Integer.toString(config.getWidth()));
			writer.newLine();
		}
		if (config.isCanBeNull() != DatabaseFieldConfig.DEFAULT_CAN_BE_NULL) {
			writer.append(FIELD_NAME_CAN_BE_NULL).append('=').append(Boolean.toString(config.isCanBeNull()));
			writer.newLine();
		}
		if (config.isId()) {
			writer.append(FIELD_NAME_ID).append('=').append("true");
			writer.newLine();
		}
		if (config.isGeneratedId()) {
			writer.append(FIELD_NAME_GENERATED_ID).append('=').append("true");
			writer.newLine();
		}
		if (config.getGeneratedIdSequence() != null) {
			writer.append(FIELD_NAME_GENERATED_ID_SEQUENCE).append('=').append(config.getGeneratedIdSequence());
			writer.newLine();
		}
		if (config.isForeign()) {
			writer.append(FIELD_NAME_FOREIGN).append('=').append("true");
			writer.newLine();
		}
		if (config.isUseGetSet()) {
			writer.append(FIELD_NAME_USE_GET_SET).append('=').append("true");
			writer.newLine();
		}
		if (config.getUnknownEnumValue() != null) {
			/**
			 * NOTE: we need to use getDeclaringClass() below. If we use getClass() then for anonymous enum types, we
			 * get an extraneous $1 in the class name. Did not know that. See Enum#getDeclaringClass().  See bug #197.
			 */
			writer.append(FIELD_NAME_UNKNOWN_ENUM_VALUE)
					.append('=')
					.append(config.getUnknownEnumValue().getDeclaringClass().getName())
					.append('#')
					.append(config.getUnknownEnumValue().name());
			writer.newLine();
		}
		if (config.isThrowIfNull()) {
			writer.append(FIELD_NAME_THROW_IF_NULL).append('=').append("true");
			writer.newLine();
		}
		if (config.getFormat() != null) {
			writer.append(FIELD_NAME_FORMAT).append('=').append(config.getFormat());
			writer.newLine();
		}
		if (config.isUnique()) {
			writer.append(FIELD_NAME_UNIQUE).append('=').append("true");
			writer.newLine();
		}
		if (config.isUniqueCombo()) {
			writer.append(FIELD_NAME_UNIQUE_COMBO).append('=').append("true");
			writer.newLine();
		}
		String indexName = config.getIndexName(tableName);
		if (indexName != null) {
			writer.append(FIELD_NAME_INDEX_NAME).append('=').append(indexName);
			writer.newLine();
		}
		String uniqueIndexName = config.getUniqueIndexName(tableName);
		if (uniqueIndexName != null) {
			writer.append(FIELD_NAME_UNIQUE_INDEX_NAME).append('=').append(uniqueIndexName);
			writer.newLine();
		}
		if (config.isForeignAutoRefresh()) {
			writer.append(FIELD_NAME_FOREIGN_AUTO_REFRESH).append('=').append("true");
			writer.newLine();
		}
		if (config.getMaxForeignAutoRefreshLevel() != DatabaseFieldConfig.NO_MAX_FOREIGN_AUTO_REFRESH_LEVEL_SPECIFIED) {
			writer.append(FIELD_NAME_MAX_FOREIGN_AUTO_REFRESH_LEVEL)
					.append('=')
					.append(Integer.toString(config.getMaxForeignAutoRefreshLevel()));
			writer.newLine();
		}
		if (config.getPersisterClass() != DatabaseFieldConfig.DEFAULT_PERSISTER_CLASS) {
			writer.append(FIELD_NAME_PERSISTER_CLASS).append('=').append(config.getPersisterClass().getName());
			writer.newLine();
		}
		if (config.isAllowGeneratedIdInsert()) {
			writer.append(FIELD_NAME_ALLOW_GENERATED_ID_INSERT).append('=').append("true");
			writer.newLine();
		}
		if (config.getColumnDefinition() != null) {
			writer.append(FIELD_NAME_COLUMN_DEFINITION).append('=').append(config.getColumnDefinition());
			writer.newLine();
		}
		if (config.getFullColumnDefinition() != null) {
			writer.append(FIELD_NAME_FULL_COLUMN_DEFINITION).append('=').append(config.getFullColumnDefinition());
			writer.newLine();
		}
		if (config.isForeignAutoCreate()) {
			writer.append(FIELD_NAME_FOREIGN_AUTO_CREATE).append('=').append("true");
			writer.newLine();
		}
		if (config.isVersion()) {
			writer.append(FIELD_NAME_VERSION).append('=').append("true");
			writer.newLine();
		}
		String foreignColumnName = config.getForeignColumnName();
		if (foreignColumnName != null) {
			writer.append(FIELD_NAME_FOREIGN_COLUMN_NAME).append('=').append(foreignColumnName);
			writer.newLine();
		}
		if (config.isReadOnly()) {
			writer.append(FIELD_NAME_READ_ONLY).append('=').append("true");
			writer.newLine();
		}

		/*
		 * Foreign collection settings:
		 */
		if (config.isForeignCollection()) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION).append('=').append("true");
			writer.newLine();
		}
		if (config.isForeignCollectionEager()) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION_EAGER).append('=').append("true");
			writer.newLine();
		}
		if (config.getForeignCollectionMaxEagerLevel() != DEFAULT_MAX_EAGER_FOREIGN_COLLECTION_LEVEL) {
			writer.append(FIELD_NAME_MAX_EAGER_FOREIGN_COLLECTION_LEVEL)
					.append('=')
					.append(Integer.toString(config.getForeignCollectionMaxEagerLevel()));
			writer.newLine();
		}
		if (config.getForeignCollectionColumnName() != null) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION_COLUMN_NAME)
					.append('=')
					.append(config.getForeignCollectionColumnName());
			writer.newLine();
		}
		if (config.getForeignCollectionOrderColumnName() != null) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION_ORDER_COLUMN_NAME)
					.append('=')
					.append(config.getForeignCollectionOrderColumnName());
			writer.newLine();
		}
		if (config.isForeignCollectionOrderAscending() != DatabaseFieldConfig.DEFAULT_FOREIGN_COLLECTION_ORDER_ASCENDING) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION_ORDER_ASCENDING)
					.append('=')
					.append(Boolean.toString(config.isForeignCollectionOrderAscending()));
			writer.newLine();
		}
		if (config.getForeignCollectionForeignFieldName() != null) {
			writer.append(FIELD_NAME_FOREIGN_COLLECTION_FOREIGN_FIELD_NAME)
					.append('=')
					.append(config.getForeignCollectionForeignFieldName());
			writer.newLine();
		}

		writer.append(CONFIG_FILE_END_MARKER);
		writer.newLine();
	}
228219
public void configure(JobConf job) {
    iconf = new IndexUpdateConfiguration(job);
    mapredTempDir = iconf.getMapredTempDir();
    mapredTempDir = Shard.normalizePath(mapredTempDir);
  }
1321911
public void loadConfig(String config)
        throws IOException
    {
        Properties properties = new Properties();
        Resource resource=Resource.newResource(config);
        properties.load(resource.getInputStream());
        
        _jdbcDriver = properties.getProperty("jdbcdriver");
        _url = properties.getProperty("url");
        _userName = properties.getProperty("username");
        _password = properties.getProperty("password");
        _userTable = properties.getProperty("usertable");
        _userTableKey = properties.getProperty("usertablekey");
        _userTableUserField = properties.getProperty("usertableuserfield");
        _userTablePasswordField = properties.getProperty("usertablepasswordfield");
        _roleTable = properties.getProperty("roletable");
        _roleTableKey = properties.getProperty("roletablekey");
        _roleTableRoleField = properties.getProperty("roletablerolefield");
        _userRoleTable = properties.getProperty("userroletable");
        _userRoleTableUserKey = properties.getProperty("userroletableuserkey");
        _userRoleTableRoleKey = properties.getProperty("userroletablerolekey");
        _cacheTime = new Integer(properties.getProperty("cachetime")).intValue();
        
        if (_jdbcDriver == null || _jdbcDriver.equals("")
            || _url == null || _url.equals("")
            || _userName == null || _userName.equals("")
            || _password == null
            || _cacheTime < 0)
        {
            if(log.isDebugEnabled())log.debug("UserRealm " + getName()
                        + " has not been properly configured");
        }
        _cacheTime *= 1000;
        _lastHashPurge = 0;
        _userSql = "select " + _userTableKey + ","
            + _userTablePasswordField + " from "
            + _userTable + " where "
            + _userTableUserField + " = ?";
        _roleSql = "select r." + _roleTableRoleField
            + " from " + _roleTable + " r, "
            + _userRoleTable + " u where u."
            + _userRoleTableUserKey + " = ?"
            + " and r." + _roleTableKey + " = u."
            + _userRoleTableRoleKey;
    }
903159
private static void assignConfigField(int configNum, DatabaseFieldConfig config, Field field, Object value) {
		switch (configNum) {
			case COLUMN_NAME:
				config.setColumnName(valueIfNotBlank((String) value));
				break;
			case DATA_TYPE:
				config.setDataType((DataType) value);
				break;
			case DEFAULT_VALUE:
				String defaultValue = (String) value;
				if (!(defaultValue == null || defaultValue.equals(DatabaseField.DEFAULT_STRING))) {
					config.setDefaultValue(defaultValue);
				}
				break;
			case WIDTH:
				config.setWidth((Integer) value);
				break;
			case CAN_BE_NULL:
				config.setCanBeNull((Boolean) value);
				break;
			case ID:
				config.setId((Boolean) value);
				break;
			case GENERATED_ID:
				config.setGeneratedId((Boolean) value);
				break;
			case GENERATED_ID_SEQUENCE:
				config.setGeneratedIdSequence(valueIfNotBlank((String) value));
				break;
			case FOREIGN:
				config.setForeign((Boolean) value);
				break;
			case USE_GET_SET:
				config.setUseGetSet((Boolean) value);
				break;
			case UNKNOWN_ENUM_NAME:
				config.setUnknownEnumValue(DatabaseFieldConfig.findMatchingEnumVal(field, (String) value));
				break;
			case THROW_IF_NULL:
				config.setThrowIfNull((Boolean) value);
				break;
			case PERSISTED:
				config.setPersisted((Boolean) value);
				break;
			case FORMAT:
				config.setFormat(valueIfNotBlank((String) value));
				break;
			case UNIQUE:
				config.setUnique((Boolean) value);
				break;
			case UNIQUE_COMBO:
				config.setUniqueCombo((Boolean) value);
				break;
			case INDEX:
				config.setIndex((Boolean) value);
				break;
			case UNIQUE_INDEX:
				config.setUniqueIndex((Boolean) value);
				break;
			case INDEX_NAME:
				config.setIndexName(valueIfNotBlank((String) value));
				break;
			case UNIQUE_INDEX_NAME:
				config.setUniqueIndexName(valueIfNotBlank((String) value));
				break;
			case FOREIGN_AUTO_REFRESH:
				config.setForeignAutoRefresh((Boolean) value);
				break;
			case MAX_FOREIGN_AUTO_REFRESH_LEVEL:
				config.setMaxForeignAutoRefreshLevel((Integer) value);
				break;
			case PERSISTER_CLASS:
				@SuppressWarnings("unchecked")
				Class<? extends DataPersister> clazz = (Class<? extends DataPersister>) value;
				config.setPersisterClass(clazz);
				break;
			case ALLOW_GENERATED_ID_INSERT:
				config.setAllowGeneratedIdInsert((Boolean) value);
				break;
			case COLUMN_DEFINITON:
				config.setColumnDefinition(valueIfNotBlank((String) value));
				break;
			case FULL_COLUMN_DEFINITON:
				config.setFullColumnDefinition(valueIfNotBlank((String) value));
				break;
			case FOREIGN_AUTO_CREATE:
				config.setForeignAutoCreate((Boolean) value);
				break;
			case VERSION:
				config.setVersion((Boolean) value);
				break;
			case FOREIGN_COLUMN_NAME:
				config.setForeignColumnName(valueIfNotBlank((String) value));
				break;
			case READ_ONLY:
				config.setReadOnly((Boolean) value);
				break;
			default:
				throw new IllegalStateException("Could not find support for DatabaseField number " + configNum);
		}
	}
918483
public static void validateAwsConfiguration(Properties config) {
		if (config.containsKey(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER)) {
			String credentialsProviderType = config.getProperty(AWSConfigConstants.AWS_CREDENTIALS_PROVIDER);

			// value specified for AWSConfigConstants.AWS_CREDENTIALS_PROVIDER needs to be recognizable
			CredentialProvider providerType;
			try {
				providerType = CredentialProvider.valueOf(credentialsProviderType);
			} catch (IllegalArgumentException e) {
				StringBuilder sb = new StringBuilder();
				for (CredentialProvider type : CredentialProvider.values()) {
					sb.append(type.toString()).append(", ");
				}
				throw new IllegalArgumentException("Invalid AWS Credential Provider Type set in config. Valid values are: " + sb.toString());
			}

			// if BASIC type is used, also check that the Access Key ID and Secret Key is supplied
			if (providerType == CredentialProvider.BASIC) {
				if (!config.containsKey(AWSConfigConstants.AWS_ACCESS_KEY_ID)
					|| !config.containsKey(AWSConfigConstants.AWS_SECRET_ACCESS_KEY)) {
					throw new IllegalArgumentException("Please set values for AWS Access Key ID ('" + AWSConfigConstants.AWS_ACCESS_KEY_ID + "') " +
						"and Secret Key ('" + AWSConfigConstants.AWS_SECRET_ACCESS_KEY + "') when using the BASIC AWS credential provider type.");
				}
			}
		}

		if (config.containsKey(AWSConfigConstants.AWS_REGION)) {
			// specified AWS Region name must be recognizable
			if (!AWSUtil.isValidRegion(config.getProperty(AWSConfigConstants.AWS_REGION))) {
				StringBuilder sb = new StringBuilder();
				for (Regions region : Regions.values()) {
					sb.append(region.getName()).append(", ");
				}
				throw new IllegalArgumentException("Invalid AWS region set in config. Valid values are: " + sb.toString());
			}
		}
	}
416116
public void setJobType(int jobType) {
	String type = null;
	switch(jobType) {
	case JOBTYPE_SINGLE:
	    type = "single"; break;
	case JOBTYPE_MULTIPLE:
	    type = "multiple"; break;
	case JOBTYPE_MPI:
	    type = "mpi"; break;
	case JOBTYPE_CONDOR:
	    type = "condor"; break;
	}
	if (type != null) {
	    set("jobtype", type);
	}
    }
426091
public static void setInputPaths(Job job, 
                                   Path... inputPaths) throws IOException {
    Configuration conf = job.getConfiguration();
    FileSystem fs = FileSystem.get(conf);
    Path path = inputPaths[0].makeQualified(fs);
    StringBuffer str = new StringBuffer(StringUtils.escapeString(path.toString()));
    for(int i = 1; i < inputPaths.length;i++) {
      str.append(StringUtils.COMMA_STR);
      path = inputPaths[i].makeQualified(fs);
      str.append(StringUtils.escapeString(path.toString()));
    }
    conf.set("mapred.input.dir", str.toString());
  }
1088023
public void setConfig(Configuration config) {
		if (config == null) {
			throw new IllegalArgumentException("Parameter config must not be null.");
		}
	    this.config = config;
	    this.loadFrom(config);
	}