opens an entry from an openex zip file archive and yields
    sequential chunks of data from the resulting stream.
552385
def chunk_zip_entry(zipfile, name, chunksize=_CHUNKSIZE):
    

    with open_zip_entry(zipfile, name, mode='r') as stream:
        data = stream.read(chunksize)
        while data:
            yield data
            data = stream.read(chunksize)
239998
def _extract_archive(file, target_dir):
    
    if file.endswith('.gz') or file.endswith('.tar') or file.endswith('.tgz'):
        archive = tarfile.open(file, 'r')
    elif file.endswith('.zip'):
        archive = zipfile.ZipFile(file, 'r')
    else:
        raise Exception('Unrecognized file type: ' + file)
    archive.extractall(path=target_dir)
    archive.close()
36095
def _chunk_data(self, data):
        """"""

        assert data[-2:] == "\r\n", "CRLF"

        if self.gzip_decompressor:
            if not self.gzip_header_seen:
                assert data[:2] == '\x1f\x8b', "gzip header"
                self.gzip_header_seen = True

            self.process_input_buffer += self.gzip_decompressor.decompress(data[:-2])
        else:
            self.process_input_buffer += data[:-2]

        self.got_chunk = True

        if self.process_input_buffer:
            # since we now have data in the buffer, enable write events again
            logger.debug('Got data in buffer, interested in writing to process again')
            self.ioloop.update_handler(self.fd_stdin, self.ioloop.WRITE | self.ioloop.ERROR)

        # do NOT call read_chunks directly. This is to give git a chance to consume input.
        # we don't want to grow the buffer unnecessarily.
        # Additionally, this should mitigate the stack explosion mentioned in read_chunks
        self.ioloop.add_callback(self.read_chunks)
451178
def extract_chunks(blob):
    """"""
    chunks = []
    stream = BytesIO(blob.bytes)
    current_pos = stream.tell()
    stream.seek(0, 2)
    length = stream.tell()
    stream.seek(current_pos, 0)
    while stream.tell() < length:
        chunks.append(read_chunk(stream))

    return chunks
551263
def smart_open_read(path=None, mode='rb', encoding=None, try_gzip=False):
    
    assert mode in ('r', 'rb')
    assert path is None or isinstance(path, (str, _oldstr))
    assert isinstance(mode, (str, _oldstr))
    assert encoding is None or isinstance(encoding, (str, _oldstr))
    assert isinstance(try_gzip, bool)

    fh = None
    binfh = None
    gzfh = None
    if path is None:
        # open stdin
        fh = io.open(sys.stdin.fileno(), mode=mode, encoding=encoding)

    else:
        # open an actual file

        if try_gzip:
            # gzip.open defaults to mode 'rb'
            gzfh = try_open_gzip(path)

        if gzfh is not None:
            logger.debug('Opening gzip''ed file.')
            # wrap gzip stream
            binfh = io.BufferedReader(gzfh)
            if 'b' not in mode:
                # add a text wrapper on top
                logger.debug('Adding text wrapper.')
                fh = io.TextIOWrapper(binfh, encoding=encoding)

        else:
            fh = io.open(path, mode=mode, encoding=encoding)

    yield_fh = fh
    if fh is None:
        yield_fh = binfh

    try:
        yield yield_fh

    finally:
        # close all open files
        if fh is not None:
            # make sure we don't close stdin
            if fh.fileno() != sys.stdin.fileno():
                fh.close()

        if binfh is not None:
            binfh.close()

        if gzfh is not None:
            gzfh.close()
127923
def readlines_bytes(self):
        """"""

        with open_zipfile_archive(self.path, self.filename) as file:
            for line in file:
                yield line.rstrip(b'\r\n')
521247
public ZipStreamImpl openReadImpl(PathImpl path) 
    throws IOException
  {
    String pathName = path.getPath();
    
    return openReadImpl(pathName);
  }
905620
@Override
    // TODO: create AssetStreamException ?
    public InputStream openStream() {
        try {
            final ZipFile file = new ZipFile(this.file);
            return new InputStreamWrapper(file, file.getInputStream(entry));
        } catch (final Exception e) {
            throw new RuntimeException("Could not open zip file stream", e);
        }
    }
484306
private void open() {
    // Load existing log segments from disk.
    for (JournalSegment<E> segment : loadSegments()) {
      segments.put(segment.descriptor().index(), segment);
    }

    // If a segment doesn't already exist, create an initial segment starting at index 1.
    if (!segments.isEmpty()) {
      currentSegment = segments.lastEntry().getValue();
    } else {
      JournalSegmentDescriptor descriptor = JournalSegmentDescriptor.builder()
          .withId(1)
          .withIndex(1)
          .withMaxSegmentSize(maxSegmentSize)
          .withMaxEntries(maxEntriesPerSegment)
          .build();

      currentSegment = createSegment(descriptor);
      currentSegment.descriptor().update(System.currentTimeMillis());

      segments.put(1L, currentSegment);
    }
  }
242625
@Override
    public ZipImporter importFrom(InputStream stream, Filter<ArchivePath> filter) throws ArchiveImportException {
        Validate.notNull(stream, "Stream must be specified");
        Validate.notNull(filter, "Filter must be specified");

        try {
            // Wrap in ZipInputStream if we haven't been given one
            final ZipInputStream zipStream = new ZipInputStream(stream);

            ZipEntry entry;
            while ((entry = zipStream.getNextEntry()) != null) {
                // Get the name
                final String entryName = entry.getName();

                if(!filter.include(ArchivePaths.create(entryName))) {
                    zipStream.closeEntry();
                    continue;
                }

                // Get the archive
                final Archive<?> archive = this.getArchive();

                // Handle directories separately
                if (entry.isDirectory()) {
                    archive.addAsDirectory(entryName);
                    continue;
                }

                final ByteArrayOutputStream output = new ByteArrayOutputStream(8192);
                IOUtil.copy(zipStream, output);
                archive.add(new ByteArrayAsset(output.toByteArray()), entryName);
                zipStream.closeEntry();
            }
        } catch (IOException e) {
            throw new ArchiveImportException("Could not import stream", e);
        }
        return this;
    }
1091443
def open(cls, sock, chunk_type, isatty, chunk_eof_type=None, buf_size=None, select_timeout=None):
    """"""
    with cls.open_multi(sock,
                        (chunk_type,),
                        (isatty,),
                        chunk_eof_type,
                        buf_size,
                        select_timeout) as ctx:
      yield ctx
911826
private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
        // Get default form
        in.defaultReadObject();

        // Create new Archive
        final String name = this.name;
        final ZipImporter archive = ShrinkWrap.create(ZipImporter.class, name);

        // Read in
        archive.importFrom(in);

        // Set
        this.archive = archive.as(JavaArchive.class);

        // Log
        if (log.isLoggable(Level.FINER)) {
            log.finer("Read in archive: " + archive.toString());
        }

        /*
         * Leave this bit here.
         *
         * After reading in the ZIP stream contents, we need to also get to the EOF marker (which is not read in by the
         * ZIP import process because it's the ZIP header, not part of the true contents. Putting this loop here ensures
         * we reach the marker, which is *not* the true end of the stream. Object data may be read again after here via
         * something like:
         *
         * in.readObject();
         *
         * Without this loop we'll get an OptionalDataException when trying to read more objects in from the stream. In
         * the future we may add state which needs to be part of the serialization protocol, and things need to stay in
         * order, so they'll be added *after* the archive ZIP contents. Thus we must be able to read them.
         */
        while (in.read() != -1) {
        }

        // We've now added an ID to archives, so add this into the wire protocol
        try {
            final String id = (String) in.readObject();
            this.archive.as(Identifiable.class).setId(id);
        } catch (final OptionalDataException ode) {
            /*
             * Swallow; this occurs when an older wire format (without notion of ID) attempts to send an archive to a
             * new format. The new archive will have the ID assigned to it at creation time, and we'll ignore that we
             * can't read in the remote one from this stream (as the remote one from the old version does not have an ID
             * to send
             */
            if (log.isLoggable(Level.FINER)) {
                log.finer("Detected an older version of the archive sent over the wire; no ID was sent.  "
                    + "Ignoring and using the default ID for this archive: " + archive.toString());
            }
        }

    }
194604
def open_file(loc):
    """"""
    loc = ensure_path(loc)
    if tarfile.is_tarfile(str(loc)):
        return tarfile.open(str(loc), "r:gz")
    elif loc.parts[-1].endswith("gz"):
        return (line.decode("utf8") for line in gzip.open(str(loc), "r"))
    elif loc.parts[-1].endswith("zip"):
        zip_file = zipfile.ZipFile(str(loc))
        names = zip_file.namelist()
        file_ = zip_file.open(names[0])
        return (line.decode("utf8") for line in file_)
    else:
        return loc.open("r", encoding="utf8")
991032
def extract_zip(fileobj):
    """"""
    archive = zipfile.ZipFile(fileobj, mode='r')
    filenames = archive.namelist()

    for src_name, dst_name in filter_tzfiles(filenames):
        modified = archive.getinfo(src_name).date_time
        bytes = archive.read(src_name)

        yield dst_name, modified, bytes
827545
def extract_zipfile(archive_name, destpath):
    ""
    archive = zipfile.ZipFile(archive_name)
    archive.extractall(destpath)
912015
public InputStream open() throws IOException, InterruptedException {
        if (recyclableInflaterInstance != null) {
            throw new IOException("Zip entry already open");
        }
        if (isDeflated) {
            recyclableInflaterInstance = nestedJarHandler.inflaterRecycler.acquire();
        }
        return new InputStream() {
            /** The data start offset within the physical zip file. */
            private final long dataStartOffsetWithinPhysicalZipFile = getEntryDataStartOffsetWithinPhysicalZipFile();

            /** A scratch buffer. */
            private final byte[] scratch = new byte[8192];

            /** The current 2GB chunk of the zip entry. */
            private ByteBuffer currChunkByteBuf;

            /** True if the current 2GB chunk is the last chunk in the zip entry. */
            private boolean isLastChunk;

            /** The index of the current 2GB chunk. */
            private int currChunkIdx;

            /** True if the end of the zip entry has been reached. */
            private boolean eof;

            /** The {@link Inflater} instance, or null if the entry is stored rather than deflated. */
            private final Inflater inflater = isDeflated ? recyclableInflaterInstance.getInflater() : null;

            /** True if this {@link InputStream} has been closed. */
            private final AtomicBoolean closed = new AtomicBoolean(false);

            /** The size of the {@link Inflate} buffer to use. */
            private static final int INFLATE_BUF_SIZE = 1024;

            // Open the first 2GB chunk.
            {
                // Calculate the chunk index for the first chunk
                currChunkIdx = (int) (dataStartOffsetWithinPhysicalZipFile / FileUtils.MAX_BUFFER_SIZE);

                // Get the MappedByteBuffer for the 2GB chunk, and duplicate it
                currChunkByteBuf = parentLogicalZipFile.physicalZipFile.getByteBuffer(currChunkIdx).duplicate();

                // Calculate the start position within the first chunk, and set the position of the slice.
                // N.B. the cast to Buffer is necessary, see:
                // https://github.com/plasma-umass/doppio/issues/497#issuecomment-334740243
                // https://github.com/classgraph/classgraph/issues/284#issuecomment-443612800
                final int chunkPos = (int) (dataStartOffsetWithinPhysicalZipFile
                        - (((long) currChunkIdx) * (long) FileUtils.MAX_BUFFER_SIZE));
                ((Buffer) currChunkByteBuf).position(chunkPos);

                // Calculate end pos for the first chunk, and truncate it if it overflows 2GB
                final long endPos = chunkPos + compressedSize;
                ((Buffer) currChunkByteBuf).limit((int) Math.min(FileUtils.MAX_BUFFER_SIZE, endPos));
                isLastChunk = endPos <= FileUtils.MAX_BUFFER_SIZE;
            }

            /** Advance to the next 2GB chunk. */
            private boolean readNextChunk() throws IOException, InterruptedException {
                currChunkIdx++;
                if (currChunkIdx >= parentLogicalZipFile.physicalZipFile.numMappedByteBuffers) {
                    // Ran out of chunks
                    return false;
                }

                // Calculate how many bytes were consumed in previous chunks
                final long chunkStartOff = ((long) currChunkIdx) * (long) FileUtils.MAX_BUFFER_SIZE;
                final long priorBytes = chunkStartOff - dataStartOffsetWithinPhysicalZipFile;
                final long remainingBytes = compressedSize - priorBytes;
                if (remainingBytes <= 0) {
                    return false;
                }

                // Get the MappedByteBuffer for the next 2GB chunk, and duplicate it
                currChunkByteBuf = parentLogicalZipFile.physicalZipFile.getByteBuffer(currChunkIdx).duplicate();

                // The start position for 2nd and subsequent chunks is 0.
                // N.B. the cast to Buffer is necessary, see:
                // https://github.com/plasma-umass/doppio/issues/497#issuecomment-334740243
                // https://github.com/classgraph/classgraph/issues/284#issuecomment-443612800
                ((Buffer) currChunkByteBuf).position(0);

                // Calculate end pos for the next chunk, and truncate it if it overflows 2GB
                ((Buffer) currChunkByteBuf).limit((int) Math.min(FileUtils.MAX_BUFFER_SIZE, remainingBytes));
                isLastChunk = remainingBytes <= FileUtils.MAX_BUFFER_SIZE;
                return true;
            }

            /**
             * Inflate deflated data.
             *
             * @param buf
             *            the buffer to inflate into.
             * @param off
             *            the offset within buf to start writing.
             * @param len
             *            the number of bytes of uncompressed data to read.
             * @return the number of bytes read.
             * @throws IOException
             *             if an I/O exception occurred.
             * @throws InterruptedException
             *             if the thread was interrupted.
             */
            private int readDeflated(final byte[] buf, final int off, final int len)
                    throws IOException, InterruptedException {
                try {
                    final byte[] inflateBuf = new byte[INFLATE_BUF_SIZE];
                    int numInflatedBytes;
                    while ((numInflatedBytes = inflater.inflate(buf, off, len)) == 0) {
                        if (inflater.finished() || inflater.needsDictionary()) {
                            eof = true;
                            return -1;
                        }
                        if (inflater.needsInput()) {
                            // Check if there's still data left in the current chunk
                            if (!currChunkByteBuf.hasRemaining()
                                    // No more bytes in current chunk -- get next chunk, and then make sure
                                    // that currChunkByteBuf.hasRemaining() subsequently returns true
                                    && !(readNextChunk() && currChunkByteBuf.hasRemaining())) {
                                // Ran out of data in the current chunk, and could not read a new chunk
                                throw new IOException("Unexpected EOF in deflated data");
                            }
                            // Set inflater input for the current chunk

                            // In JDK11+: simply use the following instead of all the lines below:
                            //     inflater.setInput(currChunkByteBuf);
                            // N.B. the ByteBuffer version of setInput doesn't seem to need the extra
                            // padding byte at the end when using the "nowrap" Inflater option.

                            // Copy from the ByteBuffer into a temporary byte[] array (needed for JDK<11). 
                            try {
                                final int remaining = currChunkByteBuf.remaining();
                                if (isLastChunk && remaining < inflateBuf.length) {
                                    // An extra dummy byte is needed at the end of the input stream when
                                    // using the "nowrap" Inflater option.
                                    // See: ZipFile.ZipFileInputStream.fill()
                                    currChunkByteBuf.get(inflateBuf, 0, remaining);
                                    inflateBuf[remaining] = (byte) 0;
                                    inflater.setInput(inflateBuf, 0, remaining + 1);
                                } else if (isLastChunk && remaining == inflateBuf.length) {
                                    // If this is the last chunk to read, and the number of remaining
                                    // bytes is exactly the size of the buffer, read one byte fewer than
                                    // the number of remaining bytes, to cause the last byte to be read
                                    // in an extra pass.
                                    currChunkByteBuf.get(inflateBuf, 0, remaining - 1);
                                    inflater.setInput(inflateBuf, 0, remaining - 1);
                                } else {
                                    // There are more than inflateBuf.length bytes remaining to be read,
                                    // or this is not the last chunk (i.e. read all remaining bytes in
                                    // this chunk, which will trigger the next chunk to be read on the
                                    // next loop iteration)
                                    final int bytesToRead = Math.min(inflateBuf.length, remaining);
                                    currChunkByteBuf.get(inflateBuf, 0, bytesToRead);
                                    inflater.setInput(inflateBuf, 0, bytesToRead);
                                }
                            } catch (final BufferUnderflowException e) {
                                // Should not happen
                                throw new IOException("Unexpected EOF in deflated data");
                            }
                        }
                    }
                    return numInflatedBytes;
                } catch (final DataFormatException e) {
                    throw new ZipException(
                            e.getMessage() != null ? e.getMessage() : "Invalid deflated zip entry data");
                }
            }

            /**
             * Copy stored (non-deflated) data from ByteBuffer to target buffer.
             *
             * @param buf
             *            the buffer to copy the stored entry into.
             * @param off
             *            the offset within buf to start writing.
             * @param len
             *            the number of bytes to read.
             * @return the number of bytes read.
             * @throws IOException
             *             if an I/O exception occurred.
             * @throws InterruptedException
             *             if the thread was interrupted.
             */
            private int readStored(final byte[] buf, final int off, final int len)
                    throws IOException, InterruptedException {
                int read = 0;
                while (read < len) {
                    if (!currChunkByteBuf.hasRemaining() && !readNextChunk()) {
                        return read == 0 ? -1 : read;
                    }
                    final int remainingToRead = len - read;
                    final int remainingInBuf = currChunkByteBuf.remaining();
                    final int numBytesRead = Math.min(remainingToRead, remainingInBuf);
                    try {
                        currChunkByteBuf.get(buf, off + read, numBytesRead);
                    } catch (final BufferUnderflowException e) {
                        // Should not happen
                        throw new EOFException("Unexpected EOF in stored (non-deflated) zip entry data");
                    }
                    read += numBytesRead;
                }
                return read;
            }

            @Override
            public int read(final byte[] buf, final int off, final int len) throws IOException {
                if (closed.get()) {
                    throw new IOException("Stream closed");
                }
                if (buf == null) {
                    throw new NullPointerException();
                } else if (off < 0 || len < 0 || len > buf.length - off) {
                    throw new IndexOutOfBoundsException();
                } else if (len == 0) {
                    return 0;
                } else if (parentLogicalZipFile.physicalZipFile.fileLen == 0) {
                    return -1;
                }
                try {
                    if (isDeflated) {
                        return readDeflated(buf, off, len);
                    } else {
                        return readStored(buf, off, len);
                    }
                } catch (final InterruptedException e) {
                    nestedJarHandler.interruptionChecker.interrupt();
                    throw new IOException("Thread was interrupted");
                }
            }

            @Override
            public int read() throws IOException {
                if (closed.get()) {
                    throw new IOException("Stream closed");
                }
                return read(scratch, 0, 1) == -1 ? -1 : scratch[0] & 0xff;
            }

            @Override
            public int available() throws IOException {
                if (closed.get()) {
                    throw new IOException("Stream closed");
                }
                if (inflater.finished()) {
                    eof = true;
                }
                return eof ? 0 : 1;
            }

            @Override
            public long skip(final long n) throws IOException {
                if (closed.get()) {
                    throw new IOException("Stream closed");
                }
                if (n < 0) {
                    throw new IllegalArgumentException("Invalid skip value");
                }
                long total = 0;
                while (total < n) {
                    final int numSkipped = read(scratch, 0, (int) Math.min(n - total, scratch.length));
                    if (numSkipped == -1) {
                        eof = true;
                        break;
                    }
                    total += numSkipped;
                }
                return total;
            }

            @Override
            public boolean markSupported() {
                return false;
            }

            @Override
            public synchronized void mark(final int readlimit) {
                throw new IllegalArgumentException("Not supported");
            }

            @Override
            public synchronized void reset() throws IOException {
                throw new IllegalArgumentException("Not supported");
            }

            @Override
            public void close() throws IOException {
                if (!closed.getAndSet(true)) {
                    currChunkByteBuf = null;
                    if (recyclableInflaterInstance != null) {
                        // Reset and recycle the Inflater
                        nestedJarHandler.inflaterRecycler.recycle(recyclableInflaterInstance);
                        recyclableInflaterInstance = null;
                    }
                }
            }
        };
    }
95294
public static void pack(ZipEntrySource[] entries, File zip) {
    if (log.isDebugEnabled()) {
      log.debug("Creating '{}' from {}.", zip, Arrays.asList(entries));
    }

    OutputStream out = null;
    try {
      out = new BufferedOutputStream(new FileOutputStream(zip));
      pack(entries, out, true);
    }
    catch (IOException e) {
      throw ZipExceptionUtil.rethrow(e);
    }
    finally {
      IOUtils.closeQuietly(out);
    }
  }