Called before job is started.

        If output is a `FileSystemTarget`, create parent directories so the hive command won't fail
187204
def prepare_outputs(self, job):
        
        outputs = flatten(job.output())
        for o in outputs:
            if isinstance(o, FileSystemTarget):
                parent_dir = os.path.dirname(o.path)
                if parent_dir and not o.fs.exists(parent_dir):
                    logger.info("Creating parent directory %r", parent_dir)
                    try:
                        # there is a possible race condition
                        # which needs to be handled here
                        o.fs.mkdir(parent_dir)
                    except FileAlreadyExists:
                        pass
945850
public static void createDirectoriesOnWorker(SubProcessConfiguration configuration)
      throws IOException {

    try {

      Path path = Paths.get(configuration.getWorkerPath());

      if (!path.toFile().exists()) {
        Files.createDirectories(path);
        LOG.info(String.format("Created Folder %s ", path.toFile()));
      }
    } catch (FileAlreadyExistsException ex) {
      LOG.warn(String.format(
          " Tried to create folder %s which already existsed, this should not happen!",
          configuration.getWorkerPath()), ex);
    }
  }
534946
@FFDCIgnore(JobStoppingException.class)
    private void executeAndWaitForCompletion(PartitionPlanDescriptor currentPlan) throws JobRestartException {
        if (isStoppingStoppedOrFailed()) {
            logger.fine("Job already in "
                        + runtimeWorkUnitExecution.getWorkUnitJobContext().getBatchStatus().toString()
                        + " state, exiting from executeAndWaitForCompletion() before beginning execution");
            return;
        }

        List<Integer> partitionsToExecute = getPartitionNumbersToExecute(currentPlan);

        logger.fine("Partitions to execute in this run: " + partitionsToExecute
                    + ".  Total number of partitions in step: " + currentPlan.getNumPartitionsInPlan());

        List<Integer> startedPartitions = new ArrayList<Integer>();
        List<Integer> finishedPartitions = new ArrayList<Integer>();
        List<Throwable> analyzerExceptions = new ArrayList<Throwable>();

        // Keep looping until all partitions have finished.
        while (finishedPartitions.size() < partitionsToExecute.size()) {

            startPartitions(partitionsToExecute, startedPartitions, finishedPartitions, currentPlan);

            // Check that there are still un-finished partitions running.
            // If not, break out of the loop.
            if (finishedPartitions.size() >= partitionsToExecute.size()) {
                break;
            }

            //Break this loop when nextPartitionFinished is -1
            try {
                waitForNextPartitionToFinish(analyzerExceptions, finishedPartitions);
            } catch (JobStoppingException e) {
                // break the loop
                break;
            }
        }

        if (!analyzerExceptions.isEmpty()) {
            rollbackPartitionedStep();
            throw new BatchContainerRuntimeException("Exception previously thrown by Analyzer, rolling back step.", analyzerExceptions.get(0));
        }
    }
1156124
private void moveTaskOutputs(TaskAttemptContext context,
                               FileSystem fs,
                               Path jobOutputDir,
                               Path taskOutput) 
  throws IOException {
    TaskAttemptID attemptId = context.getTaskAttemptID();
    context.progress();
    if (fs.isFile(taskOutput)) {
      Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, 
                                          workPath);
      if (!fs.rename(taskOutput, finalOutputPath)) {
        if (!fs.delete(finalOutputPath, true)) {
          throw new IOException("Failed to delete earlier output of task: " + 
                                 attemptId);
        }
        if (!fs.rename(taskOutput, finalOutputPath)) {
          throw new IOException("Failed to save output of task: " + 
        		  attemptId);
        }
      }
      LOG.debug("Moved " + taskOutput + " to " + finalOutputPath);
    } else if(fs.getFileStatus(taskOutput).isDir()) {
      FileStatus[] paths = fs.listStatus(taskOutput);
      Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, workPath);
      fs.mkdirs(finalOutputPath);
      if (paths != null) {
        for (FileStatus path : paths) {
          moveTaskOutputs(context, fs, jobOutputDir, path.getPath());
        }
      }
    }
  }
740289
public static void createParentDirectories(File file) {
		File parent = file.getParentFile();
		if (!parent.isDirectory()) {
			// Test the creation separately as there appears to be a race
			// condition that causes unit tests to sometimes fail.
			boolean created = parent.mkdirs();
			if (!created && !parent.isDirectory()) {
				throw new SystemException(MessageUtils.format(
						MSG_CANNOT_CREATE_OUTPUT_DIRECTORY,
						parent.getAbsolutePath()), parent);
			}
		}

	}
48767
def create_dir_unless_exists(*args):
    """"""
    path = os.path.join(*args)
    if not os.path.isdir(path):
        os.makedirs(path)
1101033
def __create_output_dir(base_dir):
    
    root, tail = os.path.split(base_dir)
    dir = None
    if tail:
        if base_dir.endswith('/'):
            dir = base_dir
        else:
            dir = root
    else:
        if base_dir.endswith('/'):
            dir = base_dir
    
    if dir and not os.path.isdir(dir):
        os.makedirs(dir)
145106
private void initJobDir (SourceState state) throws IOException {
    String tmpBase = state.getProp(MRCompactor.COMPACTION_TMP_DEST_DIR, MRCompactor.DEFAULT_COMPACTION_TMP_DEST_DIR);
    String jobId;

    if (state instanceof JobState) {
      jobId = ((JobState) state).getJobId();
    } else {
      jobId = UUID.randomUUID().toString();
    }

    this.tmpJobDir = new Path (tmpBase, jobId);
    this.fs.mkdirs(this.tmpJobDir);
    state.setProp (MRCompactor.COMPACTION_JOB_DIR, tmpJobDir.toString());
    log.info ("Job dir is created under {}", this.tmpJobDir);
  }
24018
public void initTasks() throws IOException {
    // log job info
    jobHistory.logSubmitted(jobFile.toString(), this.startTime, this.jobTrackerId);
    // log the job priority
    JobClient.RawSplit[] splits = null;
    splits = JobClient.getAndRemoveCachedSplits(jobId);
    if (splits == null) {
      FileSystem fs = jobFile.getFileSystem(jobConf);
      Path splitFile = new Path(jobFile.getParent(), "job.split");
      LOG.info("Reading splits from " + splitFile);
      DataInputStream splitFileIn = fs.open(splitFile);
      try {
        splits = JobClient.readSplitFile(splitFileIn);
      } finally {
        splitFileIn.close();
      }
    }
    initTasksFromSplits(splits);
    jobHistory.logInited(this.launchTime,
                         numMapTasks, numReduceTasks);
  }
498851
@Override
  void initializeJob(JobID jobid) {
    for(String localDir : this.mapredLocalDirs) {
      File jobDirectory = new File(localDir, 
          TaskTracker.getLocalJobDir(jobid.toString()));
      if(!jobDirectory.exists()) {
        if(!jobDirectory.mkdir()) {
          LOG.warn("Unable to create job cache directory : " 
              + jobDirectory.getPath());
          continue;
        }
      }
      //Should be recursive because the jar and work folders might be 
      //present under the job cache directory
      changeDirectoryPermissions(
          jobDirectory.getPath(), FILE_PERMISSIONS, true);
    }
  }
686867
def mkdir(self, src, extra_args=[]):
        ''''''
        return self._getStdOutCmd([self._hadoop_cmd, 'fs', '-mkdir'] + extra_args + [self._full_hdfs_path(src)], True)
346739
public void ensureStarted()
        throws MojoExecutionException
    {
        synchronized ( lock )
        {
            if ( started || starting )
            {
                return;
            }
            starting = true;
            started = false;
            finished = false;
            finishing = false;
        }
        Thread worker = new Thread( new Worker(), "FileSystemServer[" + name + "]" );
        worker.setDaemon( true );
        worker.start();
        try
        {
            synchronized ( lock )
            {
                while ( starting && !started && !finished && !finishing )
                {
                    lock.wait();
                }
                if ( problem != null )
                {
                    throw problem;
                }
            }
        }
        catch ( Exception e )
        {
            throw new MojoExecutionException( e.getMessage(), e );
        }
    }
643838
public static void setOutputPath(JobConf conf, Path outputDir) {
    if (!outputDir.isAbsolute()) {
      FileSystem.LogForCollect.info("set output path to relative path: "
          + outputDir + " working directory: " + conf.getWorkingDirectory());
    }
    outputDir = new Path(conf.getWorkingDirectory(), outputDir);
    conf.set("mapred.output.dir", outputDir.toString());
  }
1300185
def put(self, src, dst):
        '''
        '''
        src = "%s%s" % (self._testfiles_path, src)
        return self._getStdOutCmd([self._hadoop_cmd, 'fs', '-put', src, self._full_hdfs_path(dst)], True)
639661
def _runWorker(self, jobCommand, jobID, environment):
        
        startTime = time.time()  # Time job is started
        if self.debugWorker and "_toil_worker" in jobCommand:
            # Run the worker without forking
            jobName, jobStoreLocator, jobStoreID = jobCommand.split()[1:] # Parse command
            jobStore = Toil.resumeJobStore(jobStoreLocator)
            # TODO: The following does not yet properly populate self.runningJobs so it is not possible to kill
            # running jobs in forkless mode - see the "None" value in place of popen
            info = Info(time.time(), None, killIntended=False)
            try:
                self.runningJobs[jobID] = info
                try:
                    toil_worker.workerScript(jobStore, jobStore.config, jobName, jobStoreID, 
                                             redirectOutputToLogFile=not self.debugWorker) # Call the worker
                finally:
                    self.runningJobs.pop(jobID)
            finally:
                if not info.killIntended:
                    self.outputQueue.put((jobID, 0, time.time() - startTime))
        else:
            with self.popenLock:
                popen = subprocess.Popen(jobCommand,
                                         shell=True,
                                         env=dict(os.environ, **environment))
            info = Info(time.time(), popen, killIntended=False)
            try:
                self.runningJobs[jobID] = info
                try:
                    statusCode = popen.wait()
                    if statusCode != 0 and not info.killIntended:
                        log.error("Got exit code %i (indicating failure) "
                                  "from job %s.", statusCode, self.jobs[jobID])
                finally:
                    self.runningJobs.pop(jobID)
            finally:
                if not info.killIntended:
                    self.outputQueue.put((jobID, statusCode, time.time() - startTime))
1373596
def preparedir(target_dir, remove_content=True):
    
    if os.path.isdir(target_dir):
        if remove_content:
            nukedir(target_dir, False)
        return True
    else:
        try:
            os.makedirs(target_dir)
            return True
        except Exception as e:
            print("Failed to create folder: %s\n%s" % (target_dir, e))
            return False
1172885
def set_workdir(self, workdir, chroot=False):
        """"""
        if not chroot and hasattr(self, "workdir") and self.workdir != workdir:
            raise ValueError("self.workdir != workdir: %s, %s" % (self.workdir,  workdir))

        self.workdir = os.path.abspath(workdir)

        # Files required for the execution.
        self.input_file = File(os.path.join(self.workdir, "run.abi"))
        self.output_file = File(os.path.join(self.workdir, "run.abo"))
        self.files_file = File(os.path.join(self.workdir, "run.files"))
        self.job_file = File(os.path.join(self.workdir, "job.sh"))
        self.log_file = File(os.path.join(self.workdir, "run.log"))
        self.stderr_file = File(os.path.join(self.workdir, "run.err"))
        self.start_lockfile = File(os.path.join(self.workdir, "__startlock__"))
        # This file is produced by Abinit if nprocs > 1 and MPI_ABORT.
        self.mpiabort_file = File(os.path.join(self.workdir, "__ABI_MPIABORTFILE__"))

        # Directories with input|output|temporary data.
        self.wdir = Directory(self.workdir)
        self.indir = Directory(os.path.join(self.workdir, "indata"))
        self.outdir = Directory(os.path.join(self.workdir, "outdata"))
        self.tmpdir = Directory(os.path.join(self.workdir, "tmpdata"))

        # stderr and output file of the queue manager. Note extensions.
        self.qerr_file = File(os.path.join(self.workdir, "queue.qerr"))
        self.qout_file = File(os.path.join(self.workdir, "queue.qout"))