Tar(gz) archive processing
        :param fname:
        :return:
546990
def process_tar(self, fname):
        
        import tarfile  # lazy import, only when needed
        ret = []
        with tarfile.open(fname) as tr:
            members = tr.getmembers()
            for member in members:
                if not member.isfile():
                    continue
                fh = tr.extractfile(member)
                sub = self.process_file(fh.read(), member.name)
                ret.append(sub)
        return ret
316316
def create_tar (archive, compression, cmd, verbosity, interactive, filenames):
    """"""
    cmdlist = [cmd, '--create']
    add_tar_opts(cmdlist, compression, verbosity)
    cmdlist.extend(["--file", archive, '--'])
    cmdlist.extend(filenames)
    return cmdlist
75282
def get_memory_tar(image_path):
    '''
    '''
    byte_array = Client.image.export(image_path)
    file_object = io.BytesIO(byte_array)
    tar = tarfile.open(mode="r|*", fileobj=file_object)
    return (file_object,tar)
371519
def _is_bugged_tarfile(self):
        
        try:
            output = subprocess.check_output(['file', '-z', self.destination]).decode('utf8')
            return 'tar archive' in output and 'gzip compressed data' in output
        except subprocess.CalledProcessError:
            return False
1029096
def tar_open(f):
    
    if isinstance(f, six.string_types):
        return tarfile.open(name=f)
    else:
        return tarfile.open(fileobj=f)
300643
def extract_tar (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    cmdlist = [cmd, '--extract']
    add_tar_opts(cmdlist, compression, verbosity)
    cmdlist.extend(["--file", archive, '--directory', outdir])
    return cmdlist
179235
def create_tar (archive, compression, cmd, verbosity, interactive, filenames):
    """"""
    cmdlist = [cmd, '-c']
    add_star_opts(cmdlist, compression, verbosity)
    cmdlist.append("file=%s" % archive)
    cmdlist.extend(filenames)
    return cmdlist
877795
def make_tar(tfn, source_dirs, ignore_path=[], optimize_python=True):
    '''
    
    '''

    # selector function
    def select(fn):
        rfn = realpath(fn)
        for p in ignore_path:
            if p.endswith('/'):
                p = p[:-1]
            if rfn.startswith(p):
                return False
        if rfn in python_files:
            return False
        return not is_blacklist(fn)

    # get the files and relpath file of all the directory we asked for
    files = []
    for sd in source_dirs:
        sd = realpath(sd)
        compile_dir(sd, optimize_python=optimize_python)
        files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd)
                  if select(x)]

    # create tar.gz of thoses files
    tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT)
    dirs = []
    for fn, afn in files:
        dn = dirname(afn)
        if dn not in dirs:
            # create every dirs first if not exist yet
            d = ''
            for component in split(dn):
                d = join(d, component)
                if d.startswith('/'):
                    d = d[1:]
                if d == '' or d in dirs:
                    continue
                dirs.append(d)
                tinfo = tarfile.TarInfo(d)
                tinfo.type = tarfile.DIRTYPE
                tf.addfile(tinfo)

        # put the file
        tf.add(fn, afn)
    tf.close()
110785
def read_archive(archive, fname):
    

    # Note that deduplication here is equivalent to an upsert/ignore,
    # but avoids requirement for a database-specific implementation.

    zfile = zipfile.ZipFile(archive)
    contents = zfile.open(fname, 'r')
    fobj = io.TextIOWrapper(contents)

    seen = set()
    for line in fobj:
        line = line.rstrip('\t|\n')
        if line not in seen:
            yield line.split('\t|\t')
            seen.add(line)
416069
def unpack_tarfile(filename, extract_dir, progress_filter=default_filter):
    
    try:
        tarobj = tarfile.open(filename)
    except tarfile.TarError:
        raise UnrecognizedFormat(
            "%s is not a compressed or uncompressed tar file" % (filename,)
        )
    with contextlib.closing(tarobj):
        # don't do any chowning!
        tarobj.chown = lambda *args: None
        for member in tarobj:
            name = member.name
            # don't extract absolute paths or ones with .. in them
            if not name.startswith('/') and '..' not in name.split('/'):
                prelim_dst = os.path.join(extract_dir, *name.split('/'))

                # resolve any links and to extract the link targets as normal
                # files
                while member is not None and (member.islnk() or member.issym()):
                    linkpath = member.linkname
                    if member.issym():
                        base = posixpath.dirname(member.name)
                        linkpath = posixpath.join(base, linkpath)
                        linkpath = posixpath.normpath(linkpath)
                    member = tarobj._getmember(linkpath)

                if member is not None and (member.isfile() or member.isdir()):
                    final_dst = progress_filter(name, prelim_dst)
                    if final_dst:
                        if final_dst.endswith(os.sep):
                            final_dst = final_dst[:-1]
                        try:
                            # XXX Ugh
                            tarobj._extract_member(member, final_dst)
                        except tarfile.ExtractError:
                            # chown/chmod/mkfifo/mknode/makedev failed
                            pass
        return True
107798
def tar_to_bigfile(self, fname, outfile):
        """"""
        fnames = []
        tmpdir = mkdtemp()
        
        # Extract files to temporary directory
        with tarfile.open(fname) as tar:
            tar.extractall(path=tmpdir)
        for root, _, files in os.walk(tmpdir):
            fnames += [os.path.join(root, fname) for fname in files]
        
        # Concatenate
        with open(outfile, "w") as out:
            for infile in fnames:
                for line in open(infile):
                    out.write(line)
                os.unlink(infile)
        
        # Remove temp dir
        shutil.rmtree(tmpdir)
565738
function tarDir(dir) {
    var archiveName = path.basename(dir) +'.orig.tar.xz';
    var archivePath = path.join(path.dirname(dir), archiveName);

    return exec('tar cfJ '+ archivePath +' '+ dir);
}
239998
def _extract_archive(file, target_dir):
    
    if file.endswith('.gz') or file.endswith('.tar') or file.endswith('.tgz'):
        archive = tarfile.open(file, 'r')
    elif file.endswith('.zip'):
        archive = zipfile.ZipFile(file, 'r')
    else:
        raise Exception('Unrecognized file type: ' + file)
    archive.extractall(path=target_dir)
    archive.close()
992520
def extract_tar (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    try:
        with tarfile.open(archive) as tfile:
            tfile.extractall(path=outdir)
    except Exception as err:
        msg = "error extracting %s: %s" % (archive, err)
        raise util.PatoolError(msg)
    return None
835417
def create_tar_file(self, full_archive=False):
        
        tar_file_name = os.path.join(self.archive_tmp_dir, self.archive_name)
        ext = "" if self.compressor == "none" else ".%s" % self.compressor
        tar_file_name = tar_file_name + ".tar" + ext
        logger.debug("Tar File: " + tar_file_name)
        subprocess.call(shlex.split("tar c%sfS %s -C %s ." % (
            self.get_compression_flag(self.compressor),
            tar_file_name,
            # for the docker "uber archive,"use archive_dir
            #   rather than tmp_dir for all the files we tar,
            #   because all the individual archives are in there
            self.tmp_dir if not full_archive else self.archive_dir)),
            stderr=subprocess.PIPE)
        self.delete_archive_dir()
        logger.debug("Tar File Size: %s", str(os.path.getsize(tar_file_name)))
        return tar_file_name
187259
def tarfile_xml_generator(fname):
    
    tf = tarfile.open(fname, mode='r')
    try:
        tarinf = tf.next()
        while tarinf is not None:
            if tarinf.isfile() and tarinf.name[-4:] == '.xml':
                fbuf = tf.extractfile(tarinf)
                yield TarXML(name=tarinf.name, xml=fbuf.read())
            tarinf = tf.next()
            # Kludge around tarfile memory leak, cf
            # http://blogs.it.ox.ac.uk/inapickle/2011/06/20/high-memory-usage-when-using-pythons-tarfile-module/
            tf.members = []
    finally:
        tf.close()
341761
def _extract_archive(archive_fpath, archive_file, archive_namelist, output_dir,
                     force_commonprefix=True, prefix=None,
                     dryrun=False, verbose=not QUIET, overwrite=None):
    
    # force extracted components into a subdirectory if force_commonprefix is
    # on return_path = output_diG
    # FIXMpathE doesn't work right
    if prefix is not None:
        output_dir = join(output_dir, prefix)
        util_path.ensurepath(output_dir)

    archive_basename, ext = split_archive_ext(basename(archive_fpath))
    if force_commonprefix and commonprefix(archive_namelist) == '':
        # use the archivename as the default common prefix
        output_dir = join(output_dir, archive_basename)
        util_path.ensurepath(output_dir)

    for member in archive_namelist:
        (dname, fname) = split(member)
        dpath = join(output_dir, dname)
        util_path.ensurepath(dpath)
        if verbose:
            print('[utool] Unarchive ' + fname + ' in ' + dpath)

        if not dryrun:
            if overwrite is False:
                if exists(join(output_dir, member)):
                    continue
            archive_file.extract(member, path=output_dir)
    return output_dir