We read settings in ~/.fscrawler/{job_name}/_settings.json
@param jobname is the job_name
@return Settings settings
@throws IOException in case of error while reading
966373
public FsSettings readAsJson(String jobname) throws IOException {
        return FsSettingsParser.fromJson(readFile(jobname, FILENAME_JSON));
    }
359896
public Job getJob(String name) {
        //FIXME: Check for null of jobs? Can that happen?
        return jobs.stream()
            .map(SET_CLIENT(this.client))
            .filter(item -> item.getName().equals(name))
            .findAny()
            .orElseThrow(() -> new IllegalArgumentException("Job with name " + name + " does not exist."));
    }
1113401
public JobWithDetails getJob(String jobName) throws IOException {
        return getJob(null, UrlUtils.toFullJobPath(jobName));
    }
1052811
private SimpleJob addBigJoinJob(SimpleJob job)
                throws IOException {
        Configuration conf = job.getConfiguration();
        String[] labels = conf.getStrings(SimpleJob.LABELS);
        String separator = conf.get(SimpleJob.SEPARATOR);
        boolean regex = conf.getBoolean(SimpleJob.SEPARATOR_REGEX, false);
        boolean formatIgnored = conf.getBoolean(SimpleJob.FORMAT_IGNORED, false);

        SimpleJob joinJob = new SimpleJob(conf, jobName, true);
        setConfiguration(joinJob, labels, separator, formatIgnored, regex);

        int type = conf.getInt(SimpleJob.READER_TYPE, -1);
        Configuration joinConf = joinJob.getConfiguration();
        joinConf.setInt(SimpleJob.READER_TYPE, type);
        joinConf.setStrings(SimpleJob.MASTER_LABELS, conf.getStrings(SimpleJob.MASTER_LABELS));
        if (type == SimpleJob.SINGLE_COLUMN_JOIN_READER) {
            joinConf.set(SimpleJob.JOIN_MASTER_COLUMN, conf.get(SimpleJob.JOIN_MASTER_COLUMN));
            joinConf.set(SimpleJob.JOIN_DATA_COLUMN, conf.get(SimpleJob.JOIN_DATA_COLUMN));
        } else if (type == SimpleJob.SOME_COLUMN_JOIN_READER) {
            joinConf.setStrings(SimpleJob.JOIN_MASTER_COLUMN, conf.getStrings(SimpleJob.JOIN_MASTER_COLUMN));
            joinConf.setStrings(SimpleJob.JOIN_DATA_COLUMN, conf.getStrings(SimpleJob.JOIN_DATA_COLUMN));
        }
        joinConf.set(SimpleJob.MASTER_PATH, conf.get(SimpleJob.MASTER_PATH));
        joinConf.set(SimpleJob.MASTER_SEPARATOR, conf.get(SimpleJob.MASTER_SEPARATOR));

        joinJob.setMapOutputKeyClass(Key.class);
        joinJob.setMapOutputValueClass(Value.class);

        joinJob.setPartitionerClass(SimplePartitioner.class);
        joinJob.setGroupingComparatorClass(SimpleGroupingComparator.class);
        joinJob.setSortComparatorClass(SimpleSortComparator.class);

        joinJob.setSummarizer(JoinSummarizer.class);

        if (!job.isMapper() && !job.isReducer()) {
            joinConf.setBoolean(SimpleJob.ONLY_JOIN, true);
            joinJob.setOutputKeyClass(Value.class);
            joinJob.setOutputValueClass(NullWritable.class);
        } else {
            joinJob.setOutputKeyClass(Key.class);
            joinJob.setOutputValueClass(Value.class);
        }

        return joinJob;
    }
529555
public FsSettings readAsYaml(String jobname) throws IOException {
        return FsSettingsParser.fromYaml(readFile(jobname, SETTINGS_YAML));
    }
473195
public void setLocation(String location, Job job) throws IOException
    {
        conf = job.getConfiguration();
        setLocationFromUri(location);

        if (username != null && password != null)
        {
            ConfigHelper.setInputKeyspaceUserNameAndPassword(conf, username, password);
            CqlConfigHelper.setUserNameAndPassword(conf, username, password);
        }
        if (splitSize > 0)
            ConfigHelper.setInputSplitSize(conf, splitSize);
        if (partitionerClass!= null)
            ConfigHelper.setInputPartitioner(conf, partitionerClass);
        if (initHostAddress != null)
            ConfigHelper.setInputInitialAddress(conf, initHostAddress);
        if (rpcPort != null)
            ConfigHelper.setInputRpcPort(conf, rpcPort);
        if (nativePort != null)
            CqlConfigHelper.setInputNativePort(conf, nativePort);
        if (nativeCoreConnections != null)
            CqlConfigHelper.setInputCoreConnections(conf, nativeCoreConnections);
        if (nativeMaxConnections != null)
            CqlConfigHelper.setInputMaxConnections(conf, nativeMaxConnections);
        if (nativeMinSimultReqs != null)
            CqlConfigHelper.setInputMinSimultReqPerConnections(conf, nativeMinSimultReqs);
        if (nativeMaxSimultReqs != null)
            CqlConfigHelper.setInputMaxSimultReqPerConnections(conf, nativeMaxSimultReqs);
        if (nativeConnectionTimeout != null)
            CqlConfigHelper.setInputNativeConnectionTimeout(conf, nativeConnectionTimeout);
        if (nativeReadConnectionTimeout != null)
            CqlConfigHelper.setInputNativeReadConnectionTimeout(conf, nativeReadConnectionTimeout);
        if (nativeReceiveBufferSize != null)
            CqlConfigHelper.setInputNativeReceiveBufferSize(conf, nativeReceiveBufferSize);
        if (nativeSendBufferSize != null)
            CqlConfigHelper.setInputNativeSendBufferSize(conf, nativeSendBufferSize);
        if (nativeSolinger != null)
            CqlConfigHelper.setInputNativeSolinger(conf, nativeSolinger);
        if (nativeTcpNodelay != null)
            CqlConfigHelper.setInputNativeTcpNodelay(conf, nativeTcpNodelay);
        if (nativeReuseAddress != null)
            CqlConfigHelper.setInputNativeReuseAddress(conf, nativeReuseAddress);
        if (nativeKeepAlive != null)
            CqlConfigHelper.setInputNativeKeepAlive(conf, nativeKeepAlive);
        if (nativeAuthProvider != null)
            CqlConfigHelper.setInputNativeAuthProvider(conf, nativeAuthProvider);
        if (nativeSSLTruststorePath != null)
            CqlConfigHelper.setInputNativeSSLTruststorePath(conf, nativeSSLTruststorePath);
        if (nativeSSLKeystorePath != null)
            CqlConfigHelper.setInputNativeSSLKeystorePath(conf, nativeSSLKeystorePath);
        if (nativeSSLTruststorePassword != null)
            CqlConfigHelper.setInputNativeSSLTruststorePassword(conf, nativeSSLTruststorePassword);
        if (nativeSSLKeystorePassword != null)
            CqlConfigHelper.setInputNativeSSLKeystorePassword(conf, nativeSSLKeystorePassword);
        if (nativeSSLCipherSuites != null)
            CqlConfigHelper.setInputNativeSSLCipherSuites(conf, nativeSSLCipherSuites);

        ConfigHelper.setInputColumnFamily(conf, keyspace, column_family);
        setConnectionInformation();

        CqlConfigHelper.setInputCQLPageRowSize(conf, String.valueOf(pageSize));
        if (inputCql != null)
            CqlConfigHelper.setInputCql(conf, inputCql);
        if (columns != null)
            CqlConfigHelper.setInputColumns(conf, columns);
        if (whereClause != null)
            CqlConfigHelper.setInputWhereClauses(conf, whereClause);

        String whereClauseForPartitionFilter = getWhereClauseForPartitionFilter();
        String wc = whereClause != null && !whereClause.trim().isEmpty() 
                               ? whereClauseForPartitionFilter == null ? whereClause: String.format("%s AND %s", whereClause.trim(), whereClauseForPartitionFilter)
                               : whereClauseForPartitionFilter;

        if (wc != null)
        {
            logger.debug("where clause: {}", wc);
            CqlConfigHelper.setInputWhereClauses(conf, wc);
        } 
        if (System.getenv(PIG_INPUT_SPLIT_SIZE) != null)
        {
            try
            {
                ConfigHelper.setInputSplitSize(conf, Integer.parseInt(System.getenv(PIG_INPUT_SPLIT_SIZE)));
            }
            catch (NumberFormatException e)
            {
                throw new IOException("PIG_INPUT_SPLIT_SIZE is not a number", e);
            }           
        }

        if (ConfigHelper.getInputInitialAddress(conf) == null)
            throw new IOException("PIG_INPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
        if (ConfigHelper.getInputPartitioner(conf) == null)
            throw new IOException("PIG_INPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");
        if (loadSignature == null)
            loadSignature = location;

        initSchema(loadSignature);
    }
979711
@Override public List<InputSplit> getSplits(JobContext job)
		throws IOException
	{
		if (this.conf == null)
			this.conf = job.getConfiguration();

		final List<InputSplit> origSplits = super.getSplits(job);

		// We have to partition the splits by input format and hand the BCF ones
		// over to getBCFSplits().

		final List<FileSplit>
			bcfOrigSplits = new ArrayList<FileSplit>(origSplits.size());
		final List<InputSplit>
			newSplits     = new ArrayList<InputSplit>(origSplits.size());

		for (final InputSplit iSplit : origSplits) {
			final FileSplit split = (FileSplit)iSplit;

			if (VCFFormat.BCF.equals(getFormat(split.getPath())))
				bcfOrigSplits.add(split);
			else
				newSplits.add(split);
		}
		fixBCFSplits(bcfOrigSplits, newSplits);
		return filterByInterval(newSplits, conf);
	}
485149
private void read() throws IOException {
        lastModified = jobFile.lastModified();
        if (jobFile.exists()) {
            try (final InputStream in = new FileInputStream(jobFile)) {
                final XMLReader parser = XMLUtils.getXMLReader();
                parser.setContentHandler(new JobHandler(prop, files));

                parser.parse(new InputSource(in));
            } catch (final SAXException e) {
                throw new IOException("Failed to read job file: " + e.getMessage());
            }
        } else {
            // defaults
            prop.put(PROPERTY_GENERATE_COPY_OUTER, Generate.NOT_GENERATEOUTTER.toString());
            prop.put(PROPERTY_ONLY_TOPIC_IN_MAP, Boolean.toString(false));
            prop.put(PROPERTY_OUTER_CONTROL, OutterControl.WARN.toString());
        }
    }
547882
@SuppressWarnings({"unchecked"})
    public <T> T read(File jsonFile) throws IOException {
        return read(jsonFile, Configuration.defaultConfiguration());
    }
807963
public Counters readCounters(JobID jobId) {
    Counters counters = null;
    if (active) {
      try {
        FSDataInputStream dataIn = getJobInfoFile(jobId);
        if (dataIn != null) {
          readJobStatus(dataIn);
          readJobProfile(dataIn);
          counters = readCounters(dataIn);
          dataIn.close();
        }
      } catch (IOException ex) {
        LOG.warn("Could not read [" + jobId + "] job counters : " + ex, ex);
      }
    }
    return counters;
  }
1049680
public OvhSettings telephony_settings_GET() throws IOException {
		String qPath = "/me/telephony/settings";
		StringBuilder sb = path(qPath);
		String resp = exec(qPath, "GET", sb.toString(), null);
		return convertTo(resp, OvhSettings.class);
	}
1340646
public static RunningJob runJob(JobConf job) throws IOException {
    JobClient jc = new JobClient(job);
    RunningJob rj = jc.submitJob(job);
    try {
      if (!jc.monitorAndPrintJob(job, rj)) {
        throw new IOException("Job failed!");
      }
    } catch (InterruptedException ie) {
      Thread.currentThread().interrupt();
    }
    return rj;
  }
1055509
public void setLocation(String location, Job job) throws IOException
    {
        conf = HadoopCompat.getConfiguration(job);
        setLocationFromUri(location);

        if (ConfigHelper.getInputSlicePredicate(conf) == null)
        {
            SliceRange range = new SliceRange(slice_start, slice_end, slice_reverse, limit);
            SlicePredicate predicate = new SlicePredicate().setSlice_range(range);
            ConfigHelper.setInputSlicePredicate(conf, predicate);
        }
        if (System.getenv(PIG_WIDEROW_INPUT) != null)
            widerows = Boolean.parseBoolean(System.getenv(PIG_WIDEROW_INPUT));
        if (System.getenv(PIG_USE_SECONDARY) != null)
            usePartitionFilter = Boolean.parseBoolean(System.getenv(PIG_USE_SECONDARY));
        if (System.getenv(PIG_INPUT_SPLIT_SIZE) != null)
        {
            try
            {
                ConfigHelper.setInputSplitSize(conf, Integer.parseInt(System.getenv(PIG_INPUT_SPLIT_SIZE)));
            }
            catch (NumberFormatException e)
            {
                throw new IOException("PIG_INPUT_SPLIT_SIZE is not a number", e);
            }           
        } 

        if (usePartitionFilter && getIndexExpressions() != null)
            ConfigHelper.setInputRange(conf, getIndexExpressions());

        if (username != null && password != null)
            ConfigHelper.setInputKeyspaceUserNameAndPassword(conf, username, password);

        if (splitSize > 0)
            ConfigHelper.setInputSplitSize(conf, splitSize);
        if (partitionerClass!= null)
            ConfigHelper.setInputPartitioner(conf, partitionerClass);
        if (rpcPort != null)
            ConfigHelper.setInputRpcPort(conf, rpcPort);
        if (initHostAddress != null)
            ConfigHelper.setInputInitialAddress(conf, initHostAddress);

        ConfigHelper.setInputColumnFamily(conf, keyspace, column_family, widerows);
        setConnectionInformation();

        if (ConfigHelper.getInputRpcPort(conf) == 0)
            throw new IOException("PIG_INPUT_RPC_PORT or PIG_RPC_PORT environment variable not set");
        if (ConfigHelper.getInputInitialAddress(conf) == null)
            throw new IOException("PIG_INPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
        if (ConfigHelper.getInputPartitioner(conf) == null)
            throw new IOException("PIG_INPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");
        if (loadSignature == null)
            loadSignature = location;
        initSchema(loadSignature);
    }
712300
protected List<FileStatus> listStatus(JobContext job) throws IOException
    {
        List<FileStatus> result = new ArrayList<FileStatus>();
        Path[] dirs = getInputPaths(job);
        if (dirs.length == 0) {
            throw new IOException("No input paths specified in job");
        }

        // Get tokens for all the required FileSystems..
        TokenCache.obtainTokensForNamenodes(job.getCredentials(), dirs, job.getConfiguration());

        List<IOException> errors = new ArrayList<IOException>();
        for (Path p : dirs) {
            FileSystem fs = p.getFileSystem(job.getConfiguration());
            final SmilePathFilter filter = new SmilePathFilter();
            FileStatus[] matches = fs.globStatus(p, filter);
            if (matches == null) {
                errors.add(new IOException("Input path does not exist: " + p));
            }
            else if (matches.length == 0) {
                errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
            }
            else {
                for (FileStatus globStat : matches) {
                    if (globStat.isDir()) {
                        Collections.addAll(result, fs.listStatus(globStat.getPath(), filter));
                    }
                    else {
                        result.add(globStat);
                    }
                }
            }
        }

        if (!errors.isEmpty()) {
            throw new InvalidInputException(errors);
        }

        return result;
    }
517976
public static Reader openFileReader(final String fileName) throws IOException
	{
		BufferedReader bufferedReader = null;
		final File file = new File(fileName);
		bufferedReader = (BufferedReader)StreamExtensions.getReader(file);
		return bufferedReader;
	}
1184316
public com.google.protobuf.ByteString
      getJobNameBytes() {
    java.lang.Object ref = jobName_;
    if (ref instanceof java.lang.String) {
      com.google.protobuf.ByteString b = 
          com.google.protobuf.ByteString.copyFromUtf8(
              (java.lang.String) ref);
      jobName_ = b;
      return b;
    } else {
      return (com.google.protobuf.ByteString) ref;
    }
  }
1360562
public R setJobTitle(String jobTitle) {
        mBodyMap.put(BoxUser.FIELD_JOB_TITLE, jobTitle);
        return (R) this;
    }