/*
Create Job directories across disks and set their permissions to 777
This way when tasks are run we just need to setup permissions for
task folder.
498851
@Override
  void initializeJob(JobID jobid) {
    for(String localDir : this.mapredLocalDirs) {
      File jobDirectory = new File(localDir, 
          TaskTracker.getLocalJobDir(jobid.toString()));
      if(!jobDirectory.exists()) {
        if(!jobDirectory.mkdir()) {
          LOG.warn("Unable to create job cache directory : " 
              + jobDirectory.getPath());
          continue;
        }
      }
      //Should be recursive because the jar and work folders might be 
      //present under the job cache directory
      changeDirectoryPermissions(
          jobDirectory.getPath(), FILE_PERMISSIONS, true);
    }
  }
890126
public boolean mkdirs(String src, PermissionStatus permissions
  ) throws IOException {
    INode newNode = mkdirsInternal(src, permissions);
    getEditLog().logSync(false);
    if (newNode != null && auditLog.isInfoEnabled()) {
      logAuditEvent(getCurrentUGI(),
        Server.getRemoteIp(),
        "mkdirs", src, null, newNode);
    }
    return newNode != null;
  }
196891
def build(self, *args, **kwargs):
        
        # Create dirs for input, output and tmp data.
        self.indir.makedirs()
        self.outdir.makedirs()
        self.tmpdir.makedirs()

        # Write files file and input file.
        if not self.files_file.exists:
            self.files_file.write(self.filesfile_string)

        self.input_file.write(self.make_input())
        self.manager.write_jobfile(self)
822432
function makeDirs(dirs) {
	_.castArray(dirs).forEach(dir => {
		if (fs.existsSync(dir)) {
			return;
		}

		log.added(`Create folder ${dir}`);
		fs.ensureDirSync(dir);
	});
}
783963
protected function setSitePermissions() {
    $taskFilesystemStack = $this->taskFilesystemStack();
    $multisite_dir = $this->getConfigValue('docroot') . '/sites/' . $this->getConfigValue('site');
    $finder = new Finder();
    $dirs = $finder
      ->in($multisite_dir)
      ->directories()
      ->depth('< 1')
      ->exclude('files');
    foreach ($dirs->getIterator() as $dir) {
      $taskFilesystemStack->chmod($dir->getRealPath(), 0755);
    }
    $files = $finder
      ->in($multisite_dir)
      ->files()
      ->depth('< 1')
      ->exclude('files');
    foreach ($files->getIterator() as $file) {
      $taskFilesystemStack->chmod($file->getRealPath(), 0644);
    }

    $taskFilesystemStack->setVerbosityThreshold(VerbosityThresholdInterface::VERBOSITY_VERBOSE);
    $result = $taskFilesystemStack->run();

    if (!$result->wasSuccessful()) {
      throw new BltException("Unable to set permissions for site directories and files.");
    }
  }
645211
@Override
	public void setWorkingDirectory(String workingDir) {
		this.workingDir = workingDir;
		if (!this.workingDir.endsWith(URL_SEPARATOR)) {
			this.workingDir = this.workingDir + URL_SEPARATOR;
		}
	}
869196
def create_base_dirs
      BASE_DIR.each do |dir|
        FileUtils.mkdir "#{@project_name}/#{dir}"
        $stdout.puts "\e[1;32m \tcreate\e[0m\t#{dir}"
      end
      FileUtils.mkdir "#{@project_name}/app/apis/#{@project_name}"
      $stdout.puts "\e[1;32m \tcreate\e[0m\tapp/apis/#{@project_name}"
    end
570360
function create(options, cb) {

	options.force = (options.force) ? options.force : false;
	options.preserve = (options.preserve) ? options.preserve : true;
	options.password = getPwHash(options.password);  
	console.log('Gernerating directories with folling options:\n', options, '\n');
	if(options.test) {return;}
	//create directory
	if(options.force || !fs.existsSync(options.dest)) {
		wrench.mkdirSyncRecursive(options.dest);
		wrench.copyDirSyncRecursive(options.src, options.dest, { preserve: options.preserve }); 
	}
	if(options.files.length) {
		_.each(options.files, function(file) {
			fs.exists(path.join(options.dest,options.pwfile), function(exists) {
				if(!exists || options.force || file.force) {
					fs.writeFile(path.join(options.dest,strReplace(file.name, options)),  strReplace(file.data, options), function(err){
						if(err) {throw err;}
					});
				}
			});
		});
	}
	//Search and replace 
	replace(options.dest, {
		filters: ['index\.php$', '\.htaccess']
		, type: 'replace' 
		, values: [{
			user: options.user
			, root: options.root
			, project: options.project
			, title: options.title
			, userDir: options.userDir
			, pwfile: options.pwfile
		}]
	}, function(err, data) {
		if(cb) {
			cb(err, data);
			}
	});
}
685651
function setupByLink() {
    console.warn("symlink doesn't resolve __dirname.");

    const srcTaskDir = path.join(__dirname, 'tasks');

    // detect target package
    const targets = [];
    const PACKAGES_DIR = path.join(__dirname, '..');
    fs.readdirSync(PACKAGES_DIR)
    .forEach((filePath) => {
        const absPath = path.join(PACKAGES_DIR, filePath);
        if (fs.statSync(absPath).isDirectory()) {
            if (/^cdp-/.test(filePath)) {
                // check project.config.js
                try {
                    const config = require(path.join(absPath, 'project.config'));
                    if (config.required_tasks) {
                        targets.push({
                            path: absPath,
                            required_tasks: config.required_tasks,
                            task_dir: config.dir.task,
                        });
                    }
                } catch (error) {
                    return; // next
                }
            }
        }
    });

    // link
    targets.forEach((target) => {
        const dstTaskDir = path.join(target.path, target.task_dir);
        if (!fs.existsSync(dstTaskDir)) {
            // create tasks dir
            fs.mkdirSync(dstTaskDir);
        }

        target.required_tasks.forEach((task) => {
            const src = path.join(srcTaskDir, task);
            const dst = path.join(dstTaskDir, task);
            if (!fs.existsSync(src)) {
                console.error('task not found: ' + task);
                process.exit(1);
            }
            if (fs.existsSync(dst)) {
                fs.unlinkSync(dst);
            }
            fs.symlinkSync(src, dst, 'file');
        });
    });
}
633238
protected boolean setupWorkingDirectory() {
    if (!FileUtils.isDirectoryExists(schedulerWorkingDirectory)) {
      FileUtils.createDirectory(schedulerWorkingDirectory);
    }

    String topologyPackageURL = String.format("file://%s", Context.topologyPackageFile(config));
    String topologyPackageDestination = Paths.get(
        schedulerWorkingDirectory, "topology.tar.gz").toString();


    return SchedulerUtils.curlAndExtractPackage(
        schedulerWorkingDirectory, topologyPackageURL,
        topologyPackageDestination, true, Context.verbose(config));
  }
1109912
private static boolean setup(Configuration conf, JobConf jobConf,
                            JobClient client, final Arguments args,
                            boolean useFastCopy)
      throws IOException {
    jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());

    //set boolean values
    final boolean update = args.flags.contains(Options.UPDATE);
    final boolean skipCRCCheck = args.flags.contains(Options.SKIPCRC);
    final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE);
    final boolean skipUnderConstructionFile = 
        args.flags.contains(Options.SKIPUNDERCONSTRUCTION);
    jobConf.setBoolean(Options.UPDATE.propertyname, update);
    jobConf.setBoolean(Options.SKIPCRC.propertyname, skipCRCCheck);
    jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite);
    jobConf.setBoolean(Options.IGNORE_READ_FAILURES.propertyname,
        args.flags.contains(Options.IGNORE_READ_FAILURES));
    jobConf.setBoolean(Options.PRESERVE_STATUS.propertyname,
        args.flags.contains(Options.PRESERVE_STATUS));
    jobConf.setBoolean(Options.USEFASTCOPY.propertyname, useFastCopy);
    jobConf.setBoolean(Options.SKIPUNDERCONSTRUCTION.propertyname, 
              skipUnderConstructionFile);

    final String randomId = getRandomId();
    Path jobDirectory = new Path(client.getSystemDir(), NAME + "_" + randomId);
    jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());

    FileSystem dstfs = args.dst.getFileSystem(conf);
    boolean dstExists = dstfs.exists(args.dst);
    boolean dstIsDir = false;
    if (dstExists) {
      dstIsDir = dstfs.getFileStatus(args.dst).isDir();
    }

    // default logPath
    Path logPath = args.log; 
    if (logPath == null) {
      String filename = "_distcp_logs_" + randomId;
      if (!dstExists || !dstIsDir) {
        Path parent = args.dst.getParent();
        if (!dstfs.exists(parent)) {
          dstfs.mkdirs(parent);
        }
        logPath = new Path(parent, filename);
      } else {
        logPath = new Path(args.dst, filename);
      }
    }
    FileOutputFormat.setOutputPath(jobConf, logPath);

    // create src list, dst list
    FileSystem jobfs = jobDirectory.getFileSystem(jobConf);

    Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
    jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
    SequenceFile.Writer src_writer = SequenceFile.createWriter(jobfs, jobConf,
        srcfilelist, LongWritable.class, FilePairComparable.class,
        jobfs.getConf().getInt("io.file.buffer.size", 4096),
        SRC_FILES_LIST_REPL_DEFAULT, jobfs.getDefaultBlockSize(),
        SequenceFile.CompressionType.NONE, 
        new DefaultCodec(), null, new Metadata());

    Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
    SequenceFile.Writer dst_writer = SequenceFile.createWriter(jobfs, jobConf,
        dstfilelist, Text.class, Text.class,
        SequenceFile.CompressionType.NONE);

    Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
    jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
    SequenceFile.Writer dir_writer = SequenceFile.createWriter(jobfs, jobConf,
        dstdirlist, Text.class, FilePairComparable.class,
        SequenceFile.CompressionType.NONE);

    // handle the case where the destination directory doesn't exist
    // and we've only a single src directory OR we're updating/overwriting
    // the contents of the destination directory.
    final boolean special =
      (args.srcs.size() == 1 && !dstExists) || update || overwrite;
    int srcCount = 0, cnsyncf = 0, dirsyn = 0;
    long fileCount = 0L, dirCount = 0L, byteCount = 0L, cbsyncs = 0L, blockCount = 0L;

    Path basedir = null;
    HashSet<Path> parentDirsToCopy = new HashSet<Path>();
    if (args.basedir != null) {
      FileSystem basefs = args.basedir.getFileSystem(conf);
      basedir = args.basedir.makeQualified(basefs);
      if (!basefs.isDirectory(basedir)) {
        throw new IOException("Basedir " + basedir + " is not a directory.");
      }
    }

    try {
      for(Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
        final Path src = srcItr.next();
        FileSystem srcfs = src.getFileSystem(conf);
        FileStatus srcfilestat = srcfs.getFileStatus(src);
        Path root = special && srcfilestat.isDir()? src: src.getParent();

        if (basedir != null) {
          root = basedir;
          Path parent = src.getParent().makeQualified(srcfs);
          while (parent != null && !parent.equals(basedir)) {
            if (!parentDirsToCopy.contains(parent)){
              parentDirsToCopy.add(parent);
              String dst = makeRelative(root, parent);
              FileStatus pst = srcfs.getFileStatus(parent);
              src_writer.append(new LongWritable(0), new FilePairComparable(pst, dst));
              dst_writer.append(new Text(dst), new Text(parent.toString()));
              dir_writer.append(new Text(dst), new FilePairComparable(pst, dst));
              if (++dirsyn > SYNC_FILE_MAX) {
                dirsyn = 0;
                dir_writer.sync();
              }
            }
            parent = parent.getParent();
          }

          if (parent == null) {
            throw new IOException("Basedir " + basedir +
                " is not a prefix of source path " + src);
          }
        }

        if (srcfilestat.isDir()) {
          ++srcCount;
          ++dirCount;
          final String dst = makeRelative(root,src);
          src_writer.append(new LongWritable(0), new FilePairComparable(srcfilestat, dst));
          dst_writer.append(new Text(dst), new Text(src.toString()));
        }

        Stack<FileStatus> pathstack = new Stack<FileStatus>();
        for(pathstack.push(srcfilestat); !pathstack.empty(); ) {
          FileStatus cur = pathstack.pop();
          FileStatus[] children = srcfs.listStatus(cur.getPath());
          for(int i = 0; i < children.length; i++) {
            boolean skipfile = false;
            final FileStatus child = children[i]; 
            final String dst = makeRelative(root, child.getPath());
            ++srcCount;

            if (child.isDir()) {
              pathstack.push(child);
              ++dirCount;
            }
            else {
              //skip file if it exceed file limit or size limit.
              //check on whether src and dest files are same will be
              //done in mapper. 
              skipfile |= fileCount == args.filelimit
                          || byteCount + child.getLen() > args.sizelimit; 

              if (!skipfile) {
                ++fileCount;
                byteCount += child.getLen();
                blockCount += getBlocks(child.getLen(), child.getBlockSize());

                if (LOG.isTraceEnabled()) {
                  LOG.trace("adding file " + child.getPath());
                }

                ++cnsyncf;
                cbsyncs += child.getLen();
                if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) {
                  src_writer.sync();
                  dst_writer.sync();
                  cnsyncf = 0;
                  cbsyncs = 0L;
                }
              }
            }

            if (!skipfile) {
              src_writer.append(new LongWritable(child.isDir()? 0: child.getLen()),
                  new FilePairComparable(child, dst));
            }

            dst_writer.append(new Text(dst),
                new Text(child.getPath().toString()));
          }

          if (cur.isDir()) {
            String dst = makeRelative(root, cur.getPath());
            dir_writer.append(new Text(dst), new FilePairComparable(cur, dst));
            if (++dirsyn > SYNC_FILE_MAX) {
              dirsyn = 0;
              dir_writer.sync();                
            }
          }
        }
      }
    } finally {
      checkAndClose(src_writer);
      checkAndClose(dst_writer);
      checkAndClose(dir_writer);
    }

    FileStatus dststatus = null;
    try {
      dststatus = dstfs.getFileStatus(args.dst);
    } catch(FileNotFoundException fnfe) {
      LOG.info(args.dst + " does not exist.");
    }

    // create dest path dir if copying > 1 file
    if (dststatus == null) {
      if (srcCount > 1 && !dstfs.mkdirs(args.dst)) {
        throw new IOException("Failed to create" + args.dst);
      }
    }
    
    final Path sorted = new Path(jobDirectory, "_distcp_sorted"); 
    checkDuplication(jobfs, dstfilelist, sorted, conf);

    if (dststatus != null && args.flags.contains(Options.DELETE)) {
      deleteNonexisting(dstfs, dststatus, sorted,
          jobfs, jobDirectory, jobConf, conf);
    }

    Path tmpDir = new Path(
        (dstExists && !dstIsDir) || (!dstExists && srcCount == 1)?
        args.dst.getParent(): args.dst, "_distcp_tmp_" + randomId);
    jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
    LOG.info("sourcePathsCount=" + srcCount);
    LOG.info("filesToCopyCount=" + fileCount);
    LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
    jobConf.setInt(SRC_COUNT_LABEL, srcCount);
    jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
    jobConf.setLong(TOTAL_BLOCKS_LABEL, blockCount);
    setMapCount(fileCount, byteCount, jobConf, client);
    setReducerCount(fileCount, jobConf, client);
    return fileCount > 0 || dirCount > 0;
  }
343395
def _setupCache(self):
        
        # we first check whether the cache directory exists. If it doesn't, create it.
        if not os.path.exists(self.localCacheDir):
            # Create a temporary directory as this worker's private cache. If all goes well, it
            # will be renamed into the cache for this node.
            personalCacheDir = ''.join([os.path.dirname(self.localCacheDir), '/.ctmp-',
                                        str(uuid.uuid4())])
            os.mkdir(personalCacheDir, 0o755)
            self._createCacheLockFile(personalCacheDir)
            try:
                os.rename(personalCacheDir, self.localCacheDir)
            except OSError as err:
                # The only acceptable FAIL case is that the destination is a non-empty directory
                # directory.  Assuming (it's ambiguous) atomic renaming of directories, if the
                # dst is non-empty, it only means that another worker has beaten this one to the
                # rename.
                if err.errno == errno.ENOTEMPTY:
                    # Cleanup your own mess.  It's only polite.
                    shutil.rmtree(personalCacheDir)
                else:
                    raise
        # You can't reach here unless a local cache directory has been created successfully
        with self._CacheState.open(self) as cacheInfo:
            # Ensure this cache is from the correct attempt at the workflow!  If it isn't, we
            # need to reset the cache lock file
            if cacheInfo.attemptNumber != self.workflowAttemptNumber:
                if cacheInfo.nlink == 2:
                    cacheInfo.cached = 0  # cached file sizes are accounted for by job store
                else:
                    allCachedFiles = [os.path.join(self.localCacheDir, x)
                                      for x in os.listdir(self.localCacheDir)
                                      if not self._isHidden(x)]
                    cacheInfo.cached = sum([os.stat(cachedFile).st_size
                                            for cachedFile in allCachedFiles])
                    # TODO: Delete the working directories
                cacheInfo.sigmaJob = 0
                cacheInfo.attemptNumber = self.workflowAttemptNumber
            self.nlinkThreshold = cacheInfo.nlink
518449
private boolean mkdirs(FTPClient client, Path file, FsPermission permission)
      throws IOException {
    boolean created = true;
    Path workDir = new Path(client.printWorkingDirectory());
    Path absolute = makeAbsolute(workDir, file);
    String pathName = absolute.getName();
    if (!exists(client, absolute)) {
      Path parent = absolute.getParent();
      created = (parent == null || mkdirs(client, parent, FsPermission
          .getDefault()));
      if (created) {
        String parentDir = parent.toUri().getPath();
        client.changeWorkingDirectory(parentDir);
        created = created & client.makeDirectory(pathName);
      }
    } else if (isFile(client, absolute)) {
      throw new IOException(String.format(
          "Can't make directory for path %s since it is a file.", absolute));
    }
    return created;
  }
677250
function makeDirectories(dirs, callback) {
  var mkdir = function(dir, mkCallback) {
    exec('mkdir -p ' + dir, function(err, stdout, stderr) {
      return mkCallback(err);
    });
  };

  async.forEach(dirs, mkdir, callback);
}
575518
def _makedirs(self, path):
        
        try:
            oldmask = os.umask(0)
            os.makedirs(path, self._conf['dmode'])
            os.umask(oldmask)
        except OSError as e:
            if(e.errno == errno.EACCES):
                raise Exception('not sufficent permissions to write on fsdb folder: "{0}"'.format(path))
            elif(e.errno == errno.EEXIST):
                fstat = os.stat(path)
                if not stat.S_ISDIR(fstat.st_mode):
                    raise Exception('fsdb folder already exists but it is not a regular folder: "{0}"'.format(path))
                elif not os.access(path, os.R_OK and os.W_OK):
                    raise Exception('not sufficent permissions to write on fsdb folder: "{0}"'.format(path))
            else:
                raise e
887903
function doInstall(settings, destination) {
  // Create application directories if they don't exist.
  var folders = [
    destination,
    path.join(destination, '/public'),
    path.join(destination, '/extensions')
  ];
  folders.forEach(function(folder) {
    if (!fs.existsSync(folder)) {
      fs.mkdirSync(folder);
    }
  });

  // Create settings.json.
  fs.writeFileSync(path.join(destination, 'settings.json'), JSON.stringify(settings, null, '  '), {flag: 'w'});
}
46726
def set_working_directory(working_directory):
    
    logger.debug("starting")

    logger.debug(f"adding {working_directory} to sys.paths")
    sys.path.append(working_directory)

    logger.debug("done")