Create symlinks for the files needed for the jobs in current directory
@param job
@throws IOException
979062
private void symLinkAndConfigureFiles(JobConf job) throws IOException {
    if (!(job.getBoolean("mapred.used.genericoptionsparser", false))) {
      LOG.warn("Use GenericOptionsParser for parsing the arguments. " +
               "Applications should implement Tool for the same.");
    }

    // get all the command line arguments into the
    // jobconf passed in by the user conf
    String files = job.get("tmpfiles");
    String archives = job.get("tmparchives");
    // "tmpjars" are not needed because its in the classpath

    List<String> filesToSymLink = new ArrayList<String>();
    splitAndAdd(files, filesToSymLink);
    splitAndAdd(archives, filesToSymLink);

    for (String file : filesToSymLink) {
      String target = new Path(file).toUri().getPath();
      String basename = new File(target).getName();
      String linkName = new File(".").getAbsolutePath() + File.separator + basename;
      File toLink = new File(linkName);
      if (toLink.exists()) {
        LOG.info("Symlink " + linkName + " already exists. Delete it.");
        toLink.delete();
      }
      int ret = FileUtil.symLink(target, linkName);
      LOG.info("Creating symlink " + linkName + " -> " + target +
          " returns " + ret + ".");
    }
    // Configure job name
    String originalJar = job.getJar();
    if (originalJar != null) {
      // use jar name if job is not named.
      if ("".equals(job.getJobName())) {
        job.setJobName(new Path(originalJar).getName());
      }
    }
    // Configure username
    configureUserName(job);
  }
1044370
public void setWorkingDirectory(Path dir) {
    if (!dir.isAbsolute()) {
      FileSystem.LogForCollect
          .info("set job working directory to non absolute path: " + dir
              + " working directory: " + getWorkingDirectory());
    }
    dir = new Path(getWorkingDirectory(), dir);
    set("mapred.working.dir", dir.toString());
  }
574748
public NfsSymlinkResponse symlink(String symbolicLinkData, NfsSetAttributes attributes) throws IOException {
        NfsSymlinkResponse response = getNfs().wrapped_sendSymlink(makeSymlinkRequest(symbolicLinkData, attributes));
        setFileHandle(response.getFileHandle());
        return response;
    }
211817
public void writeToFile(Path loc, JobConf job)
      throws IOException {
    writeToFile(loc, job, new PureJavaCrc32());
  }
187204
def prepare_outputs(self, job):
        
        outputs = flatten(job.output())
        for o in outputs:
            if isinstance(o, FileSystemTarget):
                parent_dir = os.path.dirname(o.path)
                if parent_dir and not o.fs.exists(parent_dir):
                    logger.info("Creating parent directory %r", parent_dir)
                    try:
                        # there is a possible race condition
                        # which needs to be handled here
                        o.fs.mkdir(parent_dir)
                    except FileAlreadyExists:
                        pass
910613
@Override
	public void setJobRunning(JobID jobID) throws IOException {
		checkNotNull(jobID, "jobID");
		final Path filePath = createMarkerFilePath(RUNNING_PREFIX, jobID);

		// create the file
		// to avoid an exception if the job already exists, set overwrite=true
		createFile(filePath, true);
	}
631398
private void appendToQueueInternal (final ScheduledJob job) {
		jobIdQueue.add(job.jobId());
		queuedJobs.put(job.jobId(), job);
	}
1138240
public void setStoreLocation(String location, Job job) throws IOException
    {
        conf = HadoopCompat.getConfiguration(job);
        setLocationFromUri(location);

        if (username != null && password != null)
            ConfigHelper.setOutputKeyspaceUserNameAndPassword(conf, username, password);
        if (splitSize > 0)
            ConfigHelper.setInputSplitSize(conf, splitSize);
        if (partitionerClass!= null)
            ConfigHelper.setOutputPartitioner(conf, partitionerClass);
        if (rpcPort != null)
        {
            ConfigHelper.setOutputRpcPort(conf, rpcPort);
            ConfigHelper.setInputRpcPort(conf, rpcPort);
        }
        if (initHostAddress != null)
        {
            ConfigHelper.setOutputInitialAddress(conf, initHostAddress);
            ConfigHelper.setInputInitialAddress(conf, initHostAddress);
        }

        ConfigHelper.setOutputColumnFamily(conf, keyspace, column_family);
        CqlConfigHelper.setOutputCql(conf, outputQuery);

        setConnectionInformation();

        if (ConfigHelper.getOutputRpcPort(conf) == 0)
            throw new IOException("PIG_OUTPUT_RPC_PORT or PIG_RPC_PORT environment variable not set");
        if (ConfigHelper.getOutputInitialAddress(conf) == null)
            throw new IOException("PIG_OUTPUT_INITIAL_ADDRESS or PIG_INITIAL_ADDRESS environment variable not set");
        if (ConfigHelper.getOutputPartitioner(conf) == null)
            throw new IOException("PIG_OUTPUT_PARTITIONER or PIG_PARTITIONER environment variable not set");

        initSchema(storeSignature);
    }
145106
private void initJobDir (SourceState state) throws IOException {
    String tmpBase = state.getProp(MRCompactor.COMPACTION_TMP_DEST_DIR, MRCompactor.DEFAULT_COMPACTION_TMP_DEST_DIR);
    String jobId;

    if (state instanceof JobState) {
      jobId = ((JobState) state).getJobId();
    } else {
      jobId = UUID.randomUUID().toString();
    }

    this.tmpJobDir = new Path (tmpBase, jobId);
    this.fs.mkdirs(this.tmpJobDir);
    state.setProp (MRCompactor.COMPACTION_JOB_DIR, tmpJobDir.toString());
    log.info ("Job dir is created under {}", this.tmpJobDir);
  }
880001
private static void addInputPath(Job job, Iterable<String> pathStrings, Class<? extends InputFormat> inputFormatClass)
  {
    Configuration conf = job.getConfiguration();
    StringBuilder inputFormats = new StringBuilder(
        StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))
    );

    String[] paths = Iterables.toArray(pathStrings, String.class);
    for (int i = 0; i < paths.length - 1; i++) {
      if (inputFormats.length() > 0) {
        inputFormats.append(',');
      }
      inputFormats.append(paths[i]).append(';').append(inputFormatClass.getName());
    }
    if (inputFormats.length() > 0) {
      conf.set(MultipleInputs.DIR_FORMATS, inputFormats.toString());
    }
    // add last one separately for possible initialization in MultipleInputs
    MultipleInputs.addInputPath(job, new Path(paths[paths.length - 1]), inputFormatClass);
  }
149243
@Override
  public void addJob(final Job job) throws JobExistsException {
    log.info("adding job: {}", job);
    final JobId id = job.getId();
    final UUID operationId = UUID.randomUUID();
    final String creationPath = Paths.configJobCreation(id, operationId);
    final ZooKeeperClient client = provider.get("addJob");
    try {
      try {
        client.ensurePath(Paths.historyJob(id));
        client.transaction(create(Paths.configJob(id), job),
            create(Paths.configJobRefShort(id), id),
            create(Paths.configJobHosts(id)),
            create(creationPath),
            // Touch the jobs root node so that its version is bumped on every job
            // change down the tree. Effectively, make it that version == cVersion.
            set(Paths.configJobs(), UUID.randomUUID().toString().getBytes()));
      } catch (final NodeExistsException e) {
        if (client.exists(creationPath) != null) {
          // The job was created, we're done here
          return;
        }
        throw new JobExistsException(id.toString());
      }
    } catch (NoNodeException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed due to missing ZK path: "
                                       + e.getPath(), e);
    } catch (final KeeperException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed", e);
    }
  }
134975
public void push(CollectorRegistry registry, String job) throws IOException {
    doRequest(registry, job, null, "PUT");
  }
567719
private void prepareHadoopJob(List<WorkUnit> workUnits) throws IOException {
    TimingEvent mrJobSetupTimer = this.eventSubmitter.getTimingEvent(TimingEvent.RunJobTimings.MR_JOB_SETUP);

    // Add dependent jars/files
    addDependencies(this.job.getConfiguration());

    this.job.setJarByClass(MRJobLauncher.class);
    this.job.setMapperClass(TaskRunner.class);

    // The job is mapper-only
    this.job.setNumReduceTasks(0);

    this.job.setInputFormatClass(GobblinWorkUnitsInputFormat.class);
    this.job.setOutputFormatClass(GobblinOutputFormat.class);
    this.job.setMapOutputKeyClass(NullWritable.class);
    this.job.setMapOutputValueClass(NullWritable.class);

    // Set speculative execution

    this.job.setSpeculativeExecution(isSpeculativeExecutionEnabled(this.jobProps));

    this.job.getConfiguration().set("mapreduce.job.user.classpath.first", "true");

    // Job input path is where input work unit files are stored

    // Prepare job input
    prepareJobInput(workUnits);
    FileInputFormat.addInputPath(this.job, this.jobInputPath);

    // Job output path is where serialized task states are stored
    FileOutputFormat.setOutputPath(this.job, this.jobOutputPath);

    // Serialize source state to a file which will be picked up by the mappers
    serializeJobState(this.fs, this.mrJobDir, this.conf, this.jobContext.getJobState(), this.job);

    if (this.jobProps.containsKey(ConfigurationKeys.MR_JOB_MAX_MAPPERS_KEY)) {
      GobblinWorkUnitsInputFormat.setMaxMappers(this.job,
          Integer.parseInt(this.jobProps.getProperty(ConfigurationKeys.MR_JOB_MAX_MAPPERS_KEY)));
    }

    this.job.getConfiguration().set(GOBBLIN_JOB_INTERRUPT_PATH_KEY, this.interruptPath.toString());

    mrJobSetupTimer.stop();
  }
1301026
def worker(job):
    """"""
    ret = False
    try:
        if job.full_url is not None:
            req = requests.get(job.full_url, stream=True)
            ret = save_and_check(req, job.local_file, job.expected_checksum)
            if not ret:
                return ret
        ret = create_symlink(job.local_file, job.symlink_path)
    except KeyboardInterrupt:  # pragma: no cover
        # TODO: Actually test this once I figure out how to do this in py.test
        logging.debug("Ignoring keyboard interrupt.")

    return ret
811801
public void setJobs(java.util.Collection<JobDetail> jobs) {
        if (jobs == null) {
            this.jobs = null;
            return;
        }

        this.jobs = new java.util.ArrayList<JobDetail>(jobs);
    }
1095063
def _symlink_to_workdir(data, key):
    
    orig_file = tz.get_in(key, data)
    if orig_file and not orig_file.startswith(dd.get_work_dir(data)):
        variantcaller = genotype.get_variantcaller(data, require_bam=False)
        if not variantcaller:
            variantcaller = "precalled"
        out_file = os.path.join(dd.get_work_dir(data), variantcaller, os.path.basename(orig_file))
        utils.safe_makedir(os.path.dirname(out_file))
        utils.symlink_plus(orig_file, out_file)
        data = tz.update_in(data, key, lambda x: out_file)
    return data
712300
protected List<FileStatus> listStatus(JobContext job) throws IOException
    {
        List<FileStatus> result = new ArrayList<FileStatus>();
        Path[] dirs = getInputPaths(job);
        if (dirs.length == 0) {
            throw new IOException("No input paths specified in job");
        }

        // Get tokens for all the required FileSystems..
        TokenCache.obtainTokensForNamenodes(job.getCredentials(), dirs, job.getConfiguration());

        List<IOException> errors = new ArrayList<IOException>();
        for (Path p : dirs) {
            FileSystem fs = p.getFileSystem(job.getConfiguration());
            final SmilePathFilter filter = new SmilePathFilter();
            FileStatus[] matches = fs.globStatus(p, filter);
            if (matches == null) {
                errors.add(new IOException("Input path does not exist: " + p));
            }
            else if (matches.length == 0) {
                errors.add(new IOException("Input Pattern " + p + " matches 0 files"));
            }
            else {
                for (FileStatus globStat : matches) {
                    if (globStat.isDir()) {
                        Collections.addAll(result, fs.listStatus(globStat.getPath(), filter));
                    }
                    else {
                        result.add(globStat);
                    }
                }
            }
        }

        if (!errors.isEmpty()) {
            throw new InvalidInputException(errors);
        }

        return result;
    }