Generate the list of files and make them into FileSplits.
559175
public List<InputSplit> getSplits(JobContext job
                                    ) throws IOException {
    long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
    long maxSize = getMaxSplitSize(job);

    // generate splits
    List<InputSplit> splits = new ArrayList<InputSplit>();
    for (LocatedFileStatus file: listLocatedStatus(job)) {
      Path path = file.getPath();
      long length = file.getLen();
      BlockLocation[] blkLocations = file.getBlockLocations();

      if ((length != 0) && isSplitable(job, path)) { 
        long blockSize = file.getBlockSize();
        long splitSize = computeSplitSize(blockSize, minSize, maxSize);

        long bytesRemaining = length;
        while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
          int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
          splits.add(new FileSplit(path, length-bytesRemaining, splitSize, 
                                   blkLocations[blkIndex].getHosts()));
          bytesRemaining -= splitSize;
        }
        
        if (bytesRemaining != 0) {
          splits.add(new FileSplit(path, length-bytesRemaining, bytesRemaining, 
                     blkLocations[blkLocations.length-1].getHosts()));
        }
      } else if (length != 0) {
        splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
      } else { 
        //Create empty hosts array for zero length files
        splits.add(new FileSplit(path, 0, length, new String[0]));
      }
    }
    LOG.debug("Total # of splits: " + splits.size());
    return splits;
  }
96745
def build(self, input_path, output_paths):
        """"""
        for output in output_paths:
            shutil.copy(input_path, output_paths)
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]
3331
def make_filenames(self, **kwargs):
        
        out_dict = dict(ft1file=self.ft1file(**kwargs),
                        ltcube=self.ltcube(**kwargs),
                        ccube=self.ccube(**kwargs),
                        bexpcube=self.bexpcube(**kwargs),
                        srcmaps=self.srcmaps(**kwargs),
                        mcube=self.mcube(**kwargs))
        return out_dict
543036
def splits_and_paths(self, data_dir):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    def append_epoch(paths):
      return [
          "{}.{}".format(path, self.current_epoch)
          for path in paths
      ]

    # We set shuffled=True as we don't want to shuffle on disk later.
    return [
        (split["split"], append_epoch(filepath_fns[split["split"]](
            data_dir, split["shards"], shuffled=True
        )))
        for split in self.dataset_splits
    ]
503369
def createList(self,args):
        
        resultList = []
        if len(args.path) == 1 and os.path.isdir(args.path[0]):
            resultList = [os.path.join(args.path[0], f) for f in os.listdir(args.path[0])]
        else:    # If there are multiple items, wildcard expansion has already created the list of files
            resultList = args.path
        return list(set(resultList))
285787
def merge_required_files(dirnames, out_dir):
    
    # The list of files to merge
    fn_to_merge = ("steps_summary.tex", "excluded_markers.txt",
                   "excluded_samples.txt")

    # Merging the files
    for fn in fn_to_merge:
        o_fn = os.path.join(out_dir, fn)
        with open(o_fn, "w") as o_file:
            for dn in dirnames:
                i_fn = os.path.join(dn, fn)
                with open(i_fn, "r") as i_file:
                    o_file.write(i_file.read())

    # Merging the result summary file
    o_fn = os.path.join(out_dir, "results_summary.txt")
    with open(o_fn, "w") as o_file:
        for i, dn in enumerate(dirnames):
            i_fn = os.path.join(dn, "results_summary.txt")
            with open(i_fn, "r") as i_file:
                if i != 0:
                    # We skip the first 4 lines (file descriptions)
                    [i_file.readline() for i in range(4)]
                o_file.write(i_file.read())

    # Merging the graphic paths file
    graphic_paths = set()
    for dn in dirnames:
        fn = os.path.join(dn, "graphic_paths.txt")
        if os.path.isfile(fn):
            with open(fn, "r") as i_file:
                graphic_paths.update({
                    os.path.join(dn, path)
                    for path in i_file.read().splitlines()
                })
    if len(graphic_paths) > 0:
        with open(os.path.join(out_dir, "graphic_paths.txt"), "w") as o_file:
            for path in sorted(graphic_paths):
                print >>o_file, os.path.relpath(path, out_dir)
185778
def allfiles(self):
        
        for f in self.files:
            yield f
        for f in self.extra_files:
            yield f
152168
def join(input_files, output_file):
    '''
    
    '''

    # get feature collections
    final_features = []
    for file in input_files:
        with open(file) as f:
            feat_collection = geojson.load(f)
            final_features += feat_collection['features']

    feat_collection['features'] = final_features

    # write to output file
    with open(output_file, 'w') as f:
        geojson.dump(feat_collection, f)
704042
def build_dirs(files):
    '''
    
    '''
    for i in files:
        if type(i) is list:
            build_dirs(i)
            continue
        else:
            if len(i['path']) > 1:
                addpath = os.path.join(os.getcwd(), *i['path'][:-1])
                subdirs = all_subdirs(os.getcwd())
                if addpath and addpath not in subdirs:
                    os.makedirs(addpath)
                    print 'just made path', addpath
1028740
public InputSplit[] getSplits(JobConf job, int numSplits)
  throws IOException {
    ArrayList<FileSplit> splits = new ArrayList<FileSplit>();
    for (FileStatus status : listLocatedStatus(job)) {
      Path fileName = status.getPath();
      if (status.isDir()) {
        throw new IOException("Not a file: " + fileName);
      }
      FileSystem  fs = fileName.getFileSystem(job);
      LineReader lr = null;
      try {
        FSDataInputStream in  = fs.open(fileName);
        lr = new LineReader(in, job);
        Text line = new Text();
        int numLines = 0;
        long begin = 0;
        long length = 0;
        int num = -1;
        while ((num = lr.readLine(line)) > 0) {
          numLines++;
          length += num;
          if (numLines == N) {
            splits.add(new FileSplit(fileName, begin, length, new String[]{}));
            begin += length;
            length = 0;
            numLines = 0;
          }
        }
        if (numLines != 0) {
          splits.add(new FileSplit(fileName, begin, length, new String[]{}));
        }
   
      } finally {
        if (lr != null) {
          lr.close();
        }
      }
    }
    return splits.toArray(new FileSplit[splits.size()]);
  }
696501
def split_files(files, split_train, split_val, use_test):
    
    files_train = files[:split_train]
    files_val = files[split_train:split_val] if use_test else files[split_train:]

    li = [(files_train, 'train'), (files_val, 'val')]

    # optional test folder
    if use_test:
        files_test = files[split_val:]
        li.append((files_test, 'test'))
    return li
42825
public static void populateFilesList(File dir, List<File> filesListInDir) {
		File[] files = dir.listFiles();
		for(File file : files){
			if(file.isFile()){
				filesListInDir.add(file);
			}else{
				populateFilesList(file, filesListInDir);
			}
		}
	}
283745
protected function _generateFiles() {
        foreach ($this->selectedArtifacts as $artifact) {
            $settings = $artifact->getGeneratorSettings(static::getKey());

            foreach ($this->getFileOrDirectorySettings($settings, "files") as $file)
                $this->_processFileSpecification(
                    $artifact, fphp\Specification\FileSpecification::fromArrayAndSettings($file, $settings));

            foreach ($this->getFileOrDirectorySettings($settings, "directories") as $directory) {
                $directorySpecification = fphp\Specification\DirectorySpecification::fromArrayAndSettings($directory, $settings);
                
                foreach ($directorySpecification->getFileSpecifications() as $fileSpecification)
                    $this->_processFileSpecification($artifact, $fileSpecification);
            }
        }
    }
723451
@Override
	public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException {
		int numAvroFiles = 0;
		final Path path = this.filePath;
		// get all the files that are involved in the splits
		final FileSystem fs = path.getFileSystem();
		final FileStatus pathFile = fs.getFileStatus(path);

		if (!acceptFile(pathFile)) {
			throw new IOException("The given file does not pass the file-filter");
		}
		if (pathFile.isDir()) {
			// input is directory. list all contained files
			final FileStatus[] dir = fs.listStatus(path);
			for (int i = 0; i < dir.length; i++) {
				if (!dir[i].isDir() && acceptFile(dir[i])) {
					numAvroFiles++;
				}
			}
		} else {
			numAvroFiles = 1;
		}
		return super.createInputSplits(numAvroFiles);
	}
488543
def files(files_list, with_options: Elm::Options.new)
      Files.new @make, files_list, with_options
    end
162790
def add_files(session, imagedir, verbose):
  """"""

  def add_file(session, basename, userid, shotid, sessionid):
    """Parse a single filename and add it to the list."""
    session.add(File(userid, basename, sessionid, shotid))

  filenames = os.listdir(imagedir)
  for filename in filenames:
    basename, extension = os.path.splitext(filename)
    if extension == db_file_extension:
      if verbose>1: print("  Adding file '%s'..." % (basename))
      parts = string.split(basename, "_")
      ctype = parts[0]
      shotid = int(parts[2])
      userid = ctype + '_%d' % int(parts[1])
      if parts[0] == "Impostor":
        sessionid = 3
      elif parts[0] == "Genuine" and shotid <= 6:
        sessionid = 1
      elif parts[0] == "Genuine" and shotid > 6:
        sessionid = 2
        shotid = shotid - 6
      add_file(session, basename, userid, shotid, sessionid)