Helper method for {@link #tar(FileSystem, FileSystem, Path, Path)} that adds a file entry to a given
{@link TarArchiveOutputStream} and copies the contents of the file to the new entry.
977209
private static void fileToTarArchiveOutputStream(FileStatus fileStatus, FSDataInputStream fsDataInputStream,
      Path destFile, TarArchiveOutputStream tarArchiveOutputStream) throws IOException {
    TarArchiveEntry tarArchiveEntry = new TarArchiveEntry(formatPathToFile(destFile));
    tarArchiveEntry.setSize(fileStatus.getLen());
    tarArchiveEntry.setModTime(System.currentTimeMillis());
    tarArchiveOutputStream.putArchiveEntry(tarArchiveEntry);

    try {
      IOUtils.copy(fsDataInputStream, tarArchiveOutputStream);
    } finally {
      tarArchiveOutputStream.closeArchiveEntry();
    }
  }
951451
def create_tarfile(files: List[str], tar_path: str) -> None:
    """"""
    with tarfile.open(tar_path, "w:gz") as tar:
        for f in files:
            tar.add(f)
412357
public static void writeStringToFile(String path, String toWrite, JavaSparkContext sc) throws IOException {
        writeStringToFile(path, toWrite, sc.sc());
    }
905144
public static void writeStreamToFile(InputStream isr, String fileName, String outputFolder) throws IOException {
        logger.entering(new Object[] { isr, fileName, outputFolder });
        FileUtils.copyInputStreamToFile(isr, new File(outputFolder + "/" + fileName));
        logger.exiting();
    }
871078
public static String tarFiles(String dir, String fileNamePrefix, List<File> fileList)
        throws IOException, CompressorException {

        //
        OsUtil.makeDirs(dir);

        // 时间
        String curTime = DateUtils.format(new Date(), DataFormatConstants.COMMON_TIME_FORMAT);

        // 文件名
        String outputFilePath = fileNamePrefix + "_" + curTime + ".tar.gz";
        File outputFile = new File(dir, outputFilePath);

        FileOutputStream out = null;
        out = new FileOutputStream(outputFile);

        //
        // 进行打包
        //
        TarArchiveOutputStream os = new TarArchiveOutputStream(out);
        for (File file : fileList) {
            os.putArchiveEntry(new TarArchiveEntry(file, file.getName()));
            IOUtils.copy(new FileInputStream(file), os);
            os.closeArchiveEntry();
        }

        if (os != null) {
            try {
                os.flush();
                os.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return outputFile.getAbsolutePath();
    }
1329472
private void copyAndConfigureFiles(JobConf job, Path uploadFileDir,
      boolean shared)
    throws IOException {

    if (!(job.getBoolean("mapred.used.genericoptionsparser", false))) {
      LOG.warn("Use GenericOptionsParser for parsing the arguments. " +
               "Applications should implement Tool for the same.");
    }

    // get all the command line arguments into the
    // jobconf passed in by the user conf
    String files = job.get("tmpfiles");
    String libjars = job.get("tmpjars");
    String archives = job.get("tmparchives");

    //
    // Figure out what fs the JobTracker is using.  Copy the
    // job to it, under a temporary name.  This allows DFS to work,
    // and under the local fs also provides UNIX-like object loading
    // semantics.  (that is, if the job file is deleted right after
    // submission, we can still run the submission to completion)
    //

    // Create a number of filenames in the JobTracker's fs namespace
    FileSystem fs = getFs();
    LOG.debug("default FileSystem: " + fs.getUri());

    // We know file system of distributed cache, initialize index
    if (filesInCache == null) {
      filesInCache = new DistributedCacheIndex(fs);
    }

    uploadFileDir = fs.makeQualified(uploadFileDir);
    uploadFileDir = new Path(uploadFileDir.toUri().getPath());
    FsPermission mapredSysPerms = new FsPermission(JOB_DIR_PERMISSION);

    if (!fs.exists(uploadFileDir)) {
      FileSystem.mkdirs(fs, uploadFileDir, mapredSysPerms);
    }
    Path filesDir = new Path(uploadFileDir, "files");
    Path archivesDir = new Path(uploadFileDir, "archives");
    Path libjarsDir = new Path(uploadFileDir, "libjars");
    short replication = (short)job.getInt("mapred.submit.replication", 10);

    fileInfo = new HashMap<URI, FileInfo>();

    String originalJar = job.getJar();

    if (originalJar != null) {
      // use jar name if job is not named.
      if ("".equals(job.getJobName())) {
        job.setJobName(new Path(originalJar).getName());
      }

      Path uploadJarPath;

      Path originalJarPath = new Path(originalJar);
      originalJarPath =
        originalJarPath.makeQualified(FileSystem.getLocal(job));

      try {
        // If sharing is turned on, we load the job jar into the distributed
        // cache

        if (shared) {
          if (!fs.exists(libjarsDir)) {
            FileSystem.mkdirs(fs, libjarsDir, mapredSysPerms);
          }

          MD5Hash md5hash = MD5Hash.digest(new
              FileInputStream(originalJarPath.toUri().getPath()));
          uploadJarPath = copyRemoteFiles(fs, libjarsDir, originalJarPath, job,
              replication, md5hash.toString());
          URI pathURI = new URI(uploadJarPath.toUri().toString());

          DistributedCache.addSharedArchiveToClassPath(uploadJarPath, job);
          fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
                md5hash.getFileLength(), 0));
        } else {
          // Otherwise we copy jar to JT's filesystem
          uploadJarPath = new Path(uploadFileDir, "job.jar");
          fs.copyFromLocalFile(originalJarPath, uploadJarPath);
        }
      } catch (URISyntaxException ue) {
        // should not throw an uri exception
        throw new IOException("Failed to create uri for " + originalJar);
      }

      job.setJar(uploadJarPath.toString());
      fs.setReplication(uploadJarPath, replication);

      try {
        fs.setPermission(uploadJarPath, new FsPermission(JOB_FILE_PERMISSION));
      } catch (IOException ioe) {
        LOG.warn("Unable to set job jar permission");
      }
    } else {
      LOG.warn("No job jar file set. User classes may not be found. "+
          "See JobConf(Class) or JobConf#setJar(String).");
    }

    // add all the command line files/ jars and archive
    // first copy them to jobtrackers filesystem

    if (files != null) {
      if (!fs.exists(filesDir)) {
        FileSystem.mkdirs(fs, filesDir, mapredSysPerms);
      }

      String[] fileArr = files.split(",");
      for (String tmpFile: fileArr) {
        Path tmp = new Path(tmpFile);

        Path newPath;
        FileStatus fStatus = null;
        MD5Hash md5hash = null;
        try {
          if (shared) {
            md5hash
              = MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
            newPath = copyRemoteFiles(fs, filesDir, tmp, job, replication,
                md5hash.toString());

            URI pathURI = new URI(newPath.toUri().toString() + "#" +
                newPath.getName());

            DistributedCache.addSharedCacheFile(pathURI, job);
            fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
                  md5hash.getFileLength(),
                  0));

          } else {
            newPath = copyRemoteFiles(fs, filesDir, tmp, job, replication);
            fStatus = DistributedCache.getFileStatus(job, newPath.toUri());

            URI pathURI = new URI(newPath.toUri().toString() + "#" +
                newPath.getName());

            DistributedCache.addCacheFile(pathURI, job);
            fileInfo.put(pathURI, new FileInfo(null,
                  fStatus.getLen(),
                  fStatus.getModificationTime()));
          }

        } catch(URISyntaxException ue) {
          //should not throw a uri exception
          throw new IOException("Failed to create uri for " + tmpFile);
        }
        DistributedCache.createSymlink(job);
      }
    }

    if (libjars != null) {
      if (!fs.exists(libjarsDir)) {
        FileSystem.mkdirs(fs, libjarsDir, mapredSysPerms);
      }

      String[] libjarsArr = libjars.split(",");
      for (String tmpjars: libjarsArr) {
        Path tmp = new Path(tmpjars);

        Path newPath;
        if (shared) {
          MD5Hash md5hash
            = MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
          newPath = copyRemoteFiles(fs, libjarsDir, tmp, job, replication,
              md5hash.toString());
          DistributedCache.addSharedArchiveToClassPath(newPath, job);

          fileInfo.put(newPath.makeQualified(newPath.getFileSystem(job)).toUri(),
                       new FileInfo(md5hash.toString(),
                                    md5hash.getFileLength(),
                                    0));
        } else {
          newPath = copyRemoteFiles(fs, libjarsDir, tmp, job, replication);
          DistributedCache.addArchiveToClassPath(newPath, job);

          FileStatus fStatus = DistributedCache.getFileStatus(job,
              newPath.toUri());
          fileInfo.put(newPath.makeQualified(newPath.getFileSystem(job)).toUri(),
                       new FileInfo(null,
                                    fStatus.getLen(),
                                    fStatus.getModificationTime()));
        }
      }
    }

    if (archives != null) {
      if (!fs.exists(archivesDir)) {
        FileSystem.mkdirs(fs, archivesDir, mapredSysPerms);
      }

     String[] archivesArr = archives.split(",");
     for (String tmpArchives: archivesArr) {
       Path tmp = new Path(tmpArchives);

       Path newPath;
       MD5Hash md5hash = null;
       FileStatus fStatus = null;
       try {
         if (shared) {
           md5hash
             = MD5Hash.digest(new FileInputStream(tmp.toUri().getPath()));
           newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication,
               md5hash.toString());
           URI pathURI = new URI(newPath.toUri().toString() + "#" +
               newPath.getName());

           DistributedCache.addSharedCacheArchive(pathURI, job);
           fileInfo.put(pathURI, new FileInfo(md5hash.toString(),
                 md5hash.getFileLength(),
                 0));

         } else {
           newPath = copyRemoteFiles(fs, archivesDir, tmp, job, replication);
           fStatus = DistributedCache.getFileStatus(job, newPath.toUri());

           URI pathURI = new URI(newPath.toUri().toString() + "#" +
               newPath.getName());

           DistributedCache.addCacheArchive(pathURI, job);
           fileInfo.put(pathURI, new FileInfo(null,
                 fStatus.getLen(),
                 fStatus.getModificationTime()));
         }
       } catch(URISyntaxException ue) {
         //should not throw an uri excpetion
         throw new IOException("Failed to create uri for " + tmpArchives);
       }
       DistributedCache.createSymlink(job);
     }
    }

    // set the timestamps and md5 of the archives and files
    URI[] tarchives = DistributedCache.getSharedCacheArchives(job);
    if (tarchives != null) {
      StringBuffer archiveLength = new StringBuffer();
      FileStatus fStatus;
      FileInfo info;
      long fileLength;

      for (int i = 0; i < tarchives.length; i++) {
        if (i != 0) archiveLength.append(',');

        info = fileInfo.get(tarchives[i]);
        if (info == null) {
          fStatus = DistributedCache.getFileStatus(job, tarchives[i]);
          fileLength = fStatus.getLen();
        } else {
          fileLength = info.fileLength;
        }

        archiveLength.append(fileLength);
      }

      DistributedCache.setSharedArchiveLength(job, archiveLength.toString());
    }

    URI[] tfiles = DistributedCache.getSharedCacheFiles(job);
    if (tfiles != null) {
      StringBuffer fileLength = new StringBuffer();
      FileStatus fStatus;
      FileInfo info;
      long len;

      for (int i = 0; i < tfiles.length; i++) {
        if (i != 0) fileLength.append(',');

        info = fileInfo.get(tfiles[i]);
        if (info == null) {
          fStatus = DistributedCache.getFileStatus(job, tfiles[i]);
          len = fStatus.getLen();
        } else {
          len = info.fileLength;
        }

        fileLength.append(len);
      }

      DistributedCache.setSharedFileLength(job, fileLength.toString());
    }

    tarchives = DistributedCache.getCacheArchives(job);
    if (tarchives != null) {
      StringBuffer archiveTimestamps = new StringBuffer();
      FileInfo info;
      long timeStamp;

      for (int i = 0; i < tarchives.length; i++) {
        if (i != 0) archiveTimestamps.append(',');

        info = fileInfo.get(tarchives[i]);
        if (info == null) {
          timeStamp = DistributedCache.getTimestamp(job, tarchives[i]);
        } else {
          timeStamp = info.timeStamp;
        }
        archiveTimestamps.append(timeStamp);
      }

      DistributedCache.setArchiveTimestamps(job, archiveTimestamps.toString());
    }

    tfiles = DistributedCache.getCacheFiles(job);
    if (tfiles != null) {
      StringBuffer fileTimestamps = new StringBuffer();
      FileInfo info;
      long timeStamp;

      for (int i = 0; i < tfiles.length; i++) {
        if (i != 0) fileTimestamps.append(',');

        info = fileInfo.get(tfiles[i]);
        if (info == null) {
          timeStamp = DistributedCache.getTimestamp(job, tfiles[i]);
        } else {
          timeStamp = info.timeStamp;
        }
        fileTimestamps.append(timeStamp);
      }

      DistributedCache.setFileTimestamps(job, fileTimestamps.toString());
    }

    configureUserName(job);

  }
376164
public static OutputStream asOututStream(Path path) throws IOException {
		Validate.notNull(path, "path is null");
		return Files.newOutputStream(path);
	}
504035
public String copyFileStream(final ExecutionContext executionContext, final InputStream inputStream,
                                 final INodeEntry node, String destination) throws FileCopierException {

        return copyFile(executionContext, null, inputStream, null, node, destination);
    }
680504
public static OutputStream newOutputStream(String path) throws IOException
    {
        if (IOAdapter == null) return new FileOutputStream(path);
        return IOAdapter.create(path);
    }
239378
public void addRecord(String key,
                          String value)
                          throws TarMalformatException, IOException {

        if (key == null || value == null || key.length() < 1
                || value.length() < 1) {
            throw new TarMalformatException(
                RB.singleton.getString(RB.ZERO_WRITE));
        }

        int lenWithoutIlen = key.length() + value.length() + 3;

        // "Ilen" means Initial Length field.  +3 = SPACE + = + \n
        int lenW = 0;    // lenW = Length With initial-length-field

        if (lenWithoutIlen < 8) {
            lenW = lenWithoutIlen + 1;    // Takes just 1 char to report total
        } else if (lenWithoutIlen < 97) {
            lenW = lenWithoutIlen + 2;    // Takes 2 chars to report this total
        } else if (lenWithoutIlen < 996) {
            lenW = lenWithoutIlen + 3;    // Takes 3...
        } else if (lenWithoutIlen < 9995) {
            lenW = lenWithoutIlen + 4;    // ditto
        } else if (lenWithoutIlen < 99994) {
            lenW = lenWithoutIlen + 5;
        } else {
            throw new TarMalformatException(
                RB.singleton.getString(RB.PIF_TOOBIG, 99991));
        }

        writer.write(Integer.toString(lenW));
        writer.write(' ');
        writer.write(key);
        writer.write('=');
        writer.write(value);
        writer.write('\n');
        writer.flush();    // Does this do anything with a BAOS?
    }
247531
private static void addDir(File dirObj, ZipOutputStream out)
            throws IOException {
        File[] dirList = dirObj.listFiles();
        byte[] tmpBuf = new byte[1024];

        for (int i = 0; i < dirList.length; i++) {
            if (dirList[i].isDirectory()) {
                addDir(dirList[i], out);
                continue;
            }

            FileInputStream in = new FileInputStream(
                    dirList[i].getAbsolutePath());
            System.out.println(" Adding: " + dirList[i].getAbsolutePath());

            out.putNextEntry(new ZipEntry(dirList[i].getAbsolutePath()));

            // Transfer from the file to the ZIP file
            int len;
            while ((len = in.read(tmpBuf)) > 0) {
                out.write(tmpBuf, 0, len);
            }

            // Complete the entry
            out.closeEntry();
            in.close();
        }
    }
1354521
private void createOutputFile() throws IOException {
        this.fileChannel = Files.newByteChannel(Paths.get(filePath), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
    }
936381
private void upload(String fileOrDir, String uploadRootDir) throws Exception {
        if (uploadRootDir.isEmpty())
            uploadRootDir = ".";
        File origin = new File(fileOrDir);

        if (fileOrDir.endsWith(".tar") || fileOrDir.endsWith(".tar.gz")) {
            upload(new File(fileOrDir), uploadRootDir);
            untar(uploadRootDir);
        } else if (origin.isFile()) {
            upload(new File(fileOrDir), uploadRootDir);
        } else {
            File[] childFiles = origin.listFiles();
            if (childFiles != null)
                upload(Arrays.asList(childFiles), uploadRootDir);

        }
    }
1265636
public Jar addEntry(String path, InputStream is) throws IOException {
        beginWriting();
        addEntry(jos, path, is);
        return this;
    }
456007
@Override
    TarGzInputStream getInputStreamForRawStream(final InputStream in) throws IOException {
        assert in != null : "Specified inputstream was null";
        return new TarGzInputStream(in);
    }
9129
@Override
  public void addEntry(Archivable arch) throws IOException {
    final String fn = arch.getFileName();
    TarArchiveEntry entry = new TarArchiveEntry(fn);
    byte[] cbytes = arch.getBytes();
    entry.setSize(cbytes.length);
    this.tos.putArchiveEntry(entry);
    try (ByteArrayInputStream bis = new ByteArrayInputStream(cbytes)) {
      IOUtils.copy(bis, tos);
      tos.closeArchiveEntry();
    }
  }
13545
def create_tar_file(source_files, target=None):
    
    if target:
        filename = target
    else:
        _, filename = tempfile.mkstemp()

    with tarfile.open(filename, mode='w:gz') as t:
        for sf in source_files:
            # Add all files from the directory into the root of the directory structure of the tar
            t.add(sf, arcname=os.path.basename(sf))
    return filename