SimpleJSONPathSpec gzHeaderLengthSpec = new SimpleJSONPathSpec("Container.Gzip-Metadata.Header-Length");
197269
public void output(Resource resource) throws IOException {
		OutputStream nullo = ByteStreams.nullOutputStream();
		CountingOutputStream co = new CountingOutputStream(nullo);
		try {
			StreamCopy.copy(resource.getInputStream(), co);
		} catch(GZIPFormatException e) {
			e.printStackTrace();
			return;
		}
		long bytes = co.getCount();
		if(bytes > 0) {
			LOG.info(bytes + " unconsumed bytes in Resource InputStream.");
		}
		try {
			MetaData m = resource.getMetaData().getTopMetaData();
			// URL DATE OURL MIME HTTP-CODE SHA1 META REDIR OFFSET LENGTH FILE

			String filename = getContainerFilename(m);
			String offset = getContainerOffset(m);
			String gzLen = getGZLength(m);
			String format = getEnvelopeFormat(m);
			String origUrl = "TBD";
			String date = "TBD";
			String canUrl = "TBD";

			String mime = "TBD";
			String httpCode = "TBD";
			String digest = "TBD";
			String meta = "TBD";
			String redir = "TBD";
			
			if(format.startsWith("WARC")) {
				origUrl = getWARCURL(m);
				date = getWARCDate(m);
				String type = getWARCType(m);
				if(type.equals("response")) {
					// is it http or DNS:
					String recType = getWARCContentType(m);
					if(recType.equals("text/dns")) {
						// hrmm..
						redir = "-";
						meta = "-";
						httpCode = "-";
						mime = recType;
						digest = getEnvelopeBlockDigest(m);

					} else if(recType.equals("application/http; msgtype=response")) {
						httpCode = getHTTPStatus(m);
						digest = getHTTPEntityDigest(m);
						JSONObject headers = JSONUtils.extractObject(m, "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers");
						mime = normalizeHTTPMime(scanHeadersLC(headers, "content-type", "unk"));
						redir = scanHeadersLC(headers, "location", "-");
						meta = scanHeadersLC(headers, X_ROBOTS_HTTP_HEADER, null);
						if(meta != null) {
							meta = parseRobotInstructions(meta);
						} else {
							meta = "-";
						}
						if(mime.toLowerCase().contains("html")) {
							if(redir.equals("-")) {
								// maybe an obvious meta-refresh?
								redir = extractHTMLMetaRefresh(origUrl,m);
							}
							if(meta.equals("-")) {
								// see if there are HTML robot instructions:
								meta = extractHTMLRobots(m);
							}
						}
					}
				} else if(type.equals("warcinfo")) {
					origUrl = "warcinfo:/" + filename + "/" + IAUtils.COMMONS_VERSION.replaceAll(" ", "_");
					redir = "-";
					meta = "-";
					httpCode = "-";
					mime = "warc-info";
					digest = getEnvelopeBlockDigest(m);
				
				} else if(type.equals("request")) {
					mime = "warc/request";
					redir = "-";
					meta = "-";
					httpCode = "-";
					digest = getEnvelopeBlockDigest(m);
				} else if(type.equals("metadata")) {
					// interesting...?
					mime = "warc/metadata";
					redir = "-";
					meta = "-";
					httpCode = "-";
					digest = getEnvelopeBlockDigest(m);
				} else if(type.equals("revisit")) {
					mime = "warc/revisit";
					redir = "-";
					meta = "-";
					httpCode = "-";
					digest = getWARCPayloadDigest(m);
				}
			} else if(format.equals("ARC")) {
				origUrl = getARCURL(m);
				date = getARCDate(m);
				if(origUrl.startsWith("filedesc:")) {
					// ARC header record:
					origUrl = "filedesc:/" +filename + "/" + IAUtils.COMMONS_VERSION.replaceAll(" ", "_");
					mime = "arc-filedesc";
					redir = "-";
					meta = "-";
					httpCode = "-";
					digest = getEnvelopeBlockDigest(m);
				} else {
					// either an alexa/dat, or an HTTP response (we hope):
					mime = getARCContentType(m);
					if(mime.equals("alexa/dat")) {
						redir = "-";
						meta = "-";
						httpCode = "-";
						digest = getEnvelopeBlockDigest(m);
					} else {

						httpCode = getHTTPStatus(m);
						digest = getHTTPEntityDigest(m);
						JSONObject headers = JSONUtils.extractObject(m, "Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers");
						mime = normalizeHTTPMime(scanHeadersLC(headers, "content-type", "unk"));
						redir = scanHeadersLC(headers, "location", "-");
						meta = scanHeadersLC(headers, X_ROBOTS_HTTP_HEADER, null);
						if(meta != null) {
							meta = parseRobotInstructions(meta);
						} else {
							meta = "-";
						}
						if(mime.toLowerCase().contains("html")) {
							if(redir.equals("-")) {
								// maybe an obvious meta-refresh?
								redir = extractHTMLMetaRefresh(origUrl,m);
							}
							if(meta.equals("-")) {
								// see if there are HTML robot instructions:
								meta = extractHTMLRobots(m);
							}
						}

					}
				}
			}
			if(!redir.equals("-")) {
				redir = resolve(origUrl, redir);
			}
			canUrl = keyMaker.makeKey(origUrl);
			// URL DATE OURL MIME HTTP-CODE SHA1 META REDIR OFFSET LENGTH FILE
			if(dumpJSON) {
				out.format("%s %s %s %s %s %s %s %s %s %s %s %s\n", 
						canUrl, 
						date, 
						origUrl,
						mime,
						httpCode,
						digest,
						redir,
						meta,
						gzLen, 
						offset, 
						filename, 
						m.toString(1));
			} else {
				out.format("%s %s %s %s %s %s %s %s %s %s %s\n", 
						canUrl, 
						date, 
						origUrl,
						mime,
						httpCode,
						digest,
						redir,
						meta,
						gzLen, 
						offset, 
						filename);

			}
//			out.println(filename + " "+resource.getMetaData().getTopMetaData().toString(1));
		} catch (JSONException e) {
			throw new IOException(e);
		} catch (URISyntaxException e) {
			throw new IOException(e);
		}
		out.flush();
	}
90742
@Override
  public void setHeader(String key, String value)
  {
    int tail;

    if (_headerSize > 0) {
      tail = (_headerValues[_headerSize - 1].offset()
              + _headerValues[_headerSize - 1].length());
    }
    else {
      tail = 0;
    }

    int keyLength = key.length();
    int valueLength = value.length();
    char []headerBuffer = _headerBuffer;
    
    for (int i = keyLength - 1; i >= 0; i--) {
      headerBuffer[tail + i] = key.charAt(i);
    }

    _headerKeys[_headerSize].init(headerBuffer, tail, keyLength);

    tail += keyLength;

    for (int i = valueLength - 1; i >= 0; i--) {
      headerBuffer[tail + i] = value.charAt(i);
    }

    _headerValues[_headerSize].init(headerBuffer, tail, valueLength);
    _headerSize++;
    // XXX: size
  }
1047530
@Override
    public void setContentLengthLong (long len) {
        String methodName = "setContentLengthLong";
        if (com.ibm.ejs.ras.TraceComponent.isAnyTracingEnabled()&&logger.isLoggable (Level.FINE)) { 
            logger.entering(CLASS_NAME, methodName, "length --> " + String.valueOf(len) + " ["+this+"]");
        }

        // check the include flag
        WebAppDispatcherContext dispatchContext = (WebAppDispatcherContext) getRequest().getWebAppDispatcherContext();
        if (dispatchContext.isInclude() == true) {
            if (com.ibm.ejs.ras.TraceComponent.isAnyTracingEnabled()&&logger.isLoggable (Level.FINE)) {
                logger.logp(Level.FINE, CLASS_NAME, methodName, nls.getString("Illegal.from.included.servlet", "Illegal from included servlet"), methodName + " length --> " + String.valueOf(len));  //311717
            }
        }
        else
        {
            if (_response!=null) {
                if (!_response.isCommitted()) {
                    ((IResponse31)_response).setContentLengthLong(len);
                }
            }
            _contentLength = len;
            
            _bufferedWriter.setLimitLong(_contentLength);
            //still want to try to set the header even if the response is committed to throw the warning
            setHeader(HEADER_CONTENT_LENGTH, Long.toString(_contentLength)); // TODO SERVLET 3.1 - Need "long" implementation??
        }
        if (com.ibm.ejs.ras.TraceComponent.isAnyTracingEnabled()&&logger.isLoggable (Level.FINE)) {  //306998.15
            logger.exiting(CLASS_NAME, methodName);
        }
       
    }
630317
private long deserializeDataLength(DataInputStream dis) throws IOException {
        int dataLength = dis.readInt();
        if(dataLength == -1) {
            return dis.readLong();
        }
        return dataLength;
    }
221900
function _gpfHttpGenSetHeaders (methodName) {
    return function (httpObj, headers) {
        if (headers) {
            Object.keys(headers).forEach(function (headerName) {
                httpObj[methodName](headerName, headers[headerName]);
            });
        }
    };
}
973920
function s3Init(file) {
  if (file.s3) {
    return;
  }

  file.s3 = {};
  file.s3.headers = {};
  file.s3.path = file.path.replace(file.base, "").replace(new RegExp("\\" + path.sep, "g"), "/");
}
185416
def readlength(self):
        """"""
        logging.info('Estimating read lengths of FASTQ files')
        # Iterate through the samples
        for sample in self.samples:
            sample.run.Date = 'NA'
            sample.run.InvestigatorName = 'NA'
            sample.run.TotalClustersinRun = 'NA'
            sample.run.NumberofClustersPF = 'NA'
            sample.run.PercentOfClusters = 'NA'
            sample.run.SampleProject = 'NA'
            # Only perform this step if the forward and reverse lengths have not been loaded into the metadata
            if not GenObject.isattr(sample.run, 'forwardlength') and not GenObject.isattr(sample.run, 'reverselength'):
                # Initialise the .header attribute for each sample
                sample.header = GenObject()
                sample.commands = GenObject()
                # Set /dev/null
                devnull = open(os.devnull, 'wb')
                # Only process the samples if the file type is a list
                if type(sample.general.fastqfiles) is list:
                    # Set the forward fastq to be the first entry in the list
                    forwardfastq = sorted(sample.general.fastqfiles)[0]
                    # If the files are gzipped, then zcat must be used instead of cat
                    if '.gz' in forwardfastq:
                        command = 'zcat'
                    else:
                        command = 'cat'
                    # Read in the output of the (z)cat of the fastq file piped through head to read only the first 1000
                    # lines. Will make a string of the first 1000 lines in the file
                    forwardreads = subprocess.Popen("{} {} | head -n 1000".format(command, forwardfastq),
                                                    shell=True,
                                                    stdout=subprocess.PIPE,
                                                    stderr=devnull).communicate()[0].rstrip()
                    # Set the length of the reads as follows: the highest value (max) of the length of the sequence. The
                    # sequence was extracted from the rest of the lines in the fastq file. Example of first four lines:
                    """
                    @M02466:126:000000000-AKF4P:1:1101:11875:1838 1:N:0:1
                    TCATAACGCAGTGAAACGCTTTAACAAAAGCGGAGACACGCCACTATTTGTCAATATTTCGTATGATACATTTTTAGAAAATCAAGAAGAGTTGCACGA
                    +
                    AA,B89C,@++B,,,,C,:BFF9,C,,,,,6+++++:,C,8C+BE,EFF9FC,6E,EFGF@F<F@9F9E<FFGGGC8,,,,CC<,,,,,,6CE,C<C,,
                    """
                    # The line with the sequence information occurs every four lines (1, 5, 9, etc). This can be
                    # represented by linenumber % 4 == 1
                    try:
                        # Added due to weird 2to3 conversion issues, was coming
                        forwardreads = forwardreads.decode('utf-8')
                    except UnicodeDecodeError:
                        sample.run.forwardlength = 0
                    # up as a bytes object when we need it as a string.
                    try:
                        forwardlength = max([len(sequence) for iterator, sequence in enumerate(forwardreads.split('\n'))
                                             if iterator % 4 == 1])
                        sample.run.forwardlength = forwardlength
                    except (ValueError, TypeError):
                        sample.run.forwardlength = 0
                    # For paired end analyses, also calculate the length of the reverse reads
                    if len(sample.general.fastqfiles) == 2:
                        reversefastq = sorted(sample.general.fastqfiles)[1]
                        reversereads = subprocess.Popen("{} {} | head -n 1000".format(command, reversefastq),
                                                        shell=True,
                                                        stdout=subprocess.PIPE,
                                                        stderr=devnull).communicate()[0].rstrip()
                        try:
                            reversereads = reversereads.decode('utf-8')
                        except UnicodeDecodeError:
                            sample.run.reverselength = 0
                        try:
                            sample.run.reverselength = max([len(sequence) for iterator, sequence in
                                                            enumerate(reversereads.split('\n')) if iterator % 4 == 1])
                        except (ValueError, TypeError):
                            sample.run.reverselength = 0
                    # Populate metadata of single end reads with 'NA'
                    else:
                        sample.run.reverselength = 0
1190292
private function determineHeaderLength()
    {
        $rawRead = fread($this->fp, 4);
        if ($rawRead === false || strlen($rawRead) < 4) {
            throw new RecAnalystException(
                'Unable to read the header length',
                RecAnalystException::HEADERLEN_READERROR
            );
        }
        list (, $this->headerLen) = unpack('V', $rawRead);
        if (!$this->headerLen) {
            $this->manuallyDetermineHeaderLength();
        }
        if (!$this->headerLen) {
            throw new RecAnalystException(
                'Header length is zero',
                RecAnalystException::EMPTY_HEADER
            );
        }
        // In MGL files, the header starts immediately after the header length
        // bytes. In MGX files, another int32 is stored first, possibly indicating
        // the position of possible further headers(? something for saved chapters,
        // at least, or perhaps saved & restored games).
        // The gzip-compressed header always starts with a series of bytes that
        // decodes to a really high number. So if the next 4 bytes _don't_
        // encode to a really high number, they must refer to the next header.
        $rawRead = fread($this->fp, 4);
        list (, $nextPos) = unpack('V', $rawRead);
        $hasNextPos = $nextPos === 0 || $nextPos < $this->getFileSize();

        $this->headerStart = $hasNextPos ? 8 : 4;
        $this->headerLen -= $this->headerStart;
    }
1106950
public static boolean commonSuffixOfLength(String s, String p, int len) {
        return s != null && p != null && len >= 0 && s.regionMatches(s.length() - len, p, p.length() - len, len);
    }
570978
function (path, minify, encode) {
        minify = minify || false;
        encode = encode || 'utf-8';

        if (this.isFileSync(path)) {
            var raw = fs.readFileSync(path, encode);
            if (minify) {
                raw = jsonminify(raw);
            }

            try {
                return JSON.parse(raw);
            } catch (e1) {
                try {
                    jsonlint.parse(raw);
                } catch (e2) {
                    throw e2;
                }
            }
        }
        return {};
    }
454209
private static void assertArrayLength(final int length, final JBBPNamedFieldInfo name) {
    if (length < 0) {
      throw new JBBPParsingException("Detected negative calculated array length for field '" + (name == null ? "<NO NAME>" : name.getFieldPath()) + "\' [" + JBBPUtils.int2msg(length) + ']');
    }
  }
691869
public void readHeader(InputStream in) throws IOException {
        CRC32 crc = new CRC32();
        crc.reset();
        if (!testGzipMagic(in, crc)) {
            throw new NoGzipMagicException();
        }
        this.length += 2;
        if (readByte(in, crc) != Deflater.DEFLATED) {
            throw new IOException("Unknown compression");
        }
        this.length++;
       
        // Get gzip header flag.
        this.flg = readByte(in, crc);
        this.length++;
        
        // Get MTIME.
        this.mtime = readInt(in, crc);
        this.length += 4;
        
        // Read XFL and OS.
        this.xfl = readByte(in, crc);
        this.length++;
        this.os = readByte(in, crc);
        this.length++;
        
        // Skip optional extra field -- stuff w/ alexa stuff in it.
        final int FLG_FEXTRA = 4;
        if ((this.flg & FLG_FEXTRA) == FLG_FEXTRA) {
            int count = readShort(in, crc);
            this.length +=2;
            this.fextra = new byte[count];
            readByte(in, crc, this.fextra, 0, count);
            this.length += count;
        }   
        
        // Skip file name.  It ends in null.
        final int FLG_FNAME  = 8;
        if ((this.flg & FLG_FNAME) == FLG_FNAME) {
            while (readByte(in, crc) != 0) {
                this.length++;
            }
        }   
        
        // Skip file comment.  It ends in null.
        final int FLG_FCOMMENT = 16;   // File comment
        if ((this.flg & FLG_FCOMMENT) == FLG_FCOMMENT) {
            while (readByte(in, crc) != 0) {
                this.length++;
            }
        }
        
        // Check optional CRC.
        final int FLG_FHCRC  = 2;
        if ((this.flg & FLG_FHCRC) == FLG_FHCRC) {
            int calcCrc = (int)(crc.getValue() & 0xffff);
            if (readShort(in, crc) != calcCrc) {
                throw new IOException("Bad header CRC");
            }
            this.length += 2;
        }
    }
1009305
def _file_size(file_path, uncompressed=False):
    """"""
    _, ext = os.path.splitext(file_path)

    if uncompressed:
        if ext in {".gz", ".gzip"}:
            with gzip.GzipFile(file_path, mode="rb") as fp:
                try:
                    fp.seek(0, os.SEEK_END)
                    return fp.tell()
                except ValueError:
                    # on python2, cannot seek from end and must instead read to end
                    fp.seek(0)
                    while len(fp.read(8192)) != 0:
                        pass
                    return fp.tell()
        elif ext in {".bz", ".bz2", ".bzip", ".bzip2"}:
            with bz2.BZ2File(file_path, mode="rb") as fp:
                fp.seek(0, os.SEEK_END)
                return fp.tell()

    return os.path.getsize(file_path)
853295
def ciphertext_length(self):
        
        return aws_encryption_sdk.internal.formatting.ciphertext_length(
            header=self.header, plaintext_length=self.stream_length
        )
886163
public static void checkIfORC(FileSystem fs, Path path) throws IOException {
    // hardcoded to 40 because "SEQ-org.apache.hadoop.hive.ql.io.RCFile", the header, is of 40 chars
    final int buffLen = 40;
    final byte header[] = new byte[buffLen];
    final FSDataInputStream file = fs.open(path);
    final long fileLength = fs.getFileStatus(path).getLen();
    int sizeToBeRead = buffLen;
    if (buffLen > fileLength) {
      sizeToBeRead = (int)fileLength;
    }

    IOUtils.readFully(file, header, 0, sizeToBeRead);
    file.close();

    final String headerString = new String(header);
    if (headerString.startsWith("ORC")) {
      LOG.error("Error while parsing the footer of the file : " + path);
    } else {
      throw new NotAnORCFileException("Input file = " + path + " , header = " + headerString);
    }
  }
304442
public static Message buildFileHeader(long length) {
    ZabMessage.FileHeader header = ZabMessage.FileHeader.newBuilder()
                                             .setLength(length).build();
    return Message.newBuilder().setType(MessageType.FILE_HEADER)
                               .setFileHeader(header).build();
  }
1118312
private function gzdecode($data,&$filename='',&$error='',$maxlength=null){
	    $len = strlen($data);
	    if ($len < 18 || strcmp(substr($data,0,2),"\x1f\x8b")) {
	        $error = "Not in GZIP format.";
	        return null;  // Not GZIP format (See RFC 1952)
	    }
	    $method = ord(substr($data,2,1));  // Compression method
	    $flags  = ord(substr($data,3,1));  // Flags
	    if ($flags & 31 != $flags) {
	        $error = "Reserved bits not allowed.";
	        return null;
	    }
	    // NOTE: $mtime may be negative (PHP integer limitations)
	    $mtime = unpack("V", substr($data,4,4));
	    $mtime = $mtime[1];
	    $xfl   = substr($data,8,1);
	    $os    = substr($data,8,1);
	    $headerlen = 10;
	    $extralen  = 0;
	    $extra     = "";
	    if ($flags & 4) {
	        // 2-byte length prefixed EXTRA data in header
	        if ($len - $headerlen - 2 < 8) {
	            return false;  // invalid
	        }
	        $extralen = unpack("v",substr($data,8,2));
	        $extralen = $extralen[1];
	        if ($len - $headerlen - 2 - $extralen < 8) {
	            return false;  // invalid
	        }
	        $extra = substr($data,10,$extralen);
	        $headerlen += 2 + $extralen;
	    }
	    $filenamelen = 0;
	    $filename = "";
	    if ($flags & 8) {
	        // C-style string
	        if ($len - $headerlen - 1 < 8) {
	            return false; // invalid
	        }
	        $filenamelen = strpos(substr($data,$headerlen),chr(0));
	        if ($filenamelen === false || $len - $headerlen - $filenamelen - 1 < 8) {
	            return false; // invalid
	        }
	        $filename = substr($data,$headerlen,$filenamelen);
	        $headerlen += $filenamelen + 1;
	    }
	    $commentlen = 0;
	    $comment = "";
	    if ($flags & 16) {
	        // C-style string COMMENT data in header
	        if ($len - $headerlen - 1 < 8) {
	            return false;    // invalid
	        }
	        $commentlen = strpos(substr($data,$headerlen),chr(0));
	        if ($commentlen === false || $len - $headerlen - $commentlen - 1 < 8) {
	            return false;    // Invalid header format
	        }
	        $comment = substr($data,$headerlen,$commentlen);
	        $headerlen += $commentlen + 1;
	    }
	    $headercrc = "";
	    if ($flags & 2) {
	        // 2-bytes (lowest order) of CRC32 on header present
	        if ($len - $headerlen - 2 < 8) {
	            return false;    // invalid
	        }
	        $calccrc = crc32(substr($data,0,$headerlen)) & 0xffff;
	        $headercrc = unpack("v", substr($data,$headerlen,2));
	        $headercrc = $headercrc[1];
	        if ($headercrc != $calccrc) {
	            $error = "Header checksum failed.";
	            return false;    // Bad header CRC
	        }
	        $headerlen += 2;
	    }
	    // GZIP FOOTER
	    $datacrc = unpack("V",substr($data,-8,4));
	    $datacrc = sprintf('%u',$datacrc[1] & 0xFFFFFFFF);
	    $isize = unpack("V",substr($data,-4));
	    $isize = $isize[1];
	    // decompression:
	    $bodylen = $len-$headerlen-8;
	    if ($bodylen < 1) {
	        // IMPLEMENTATION BUG!
	        return null;
	    }
	    $body = substr($data,$headerlen,$bodylen);
	    $data = "";
	    if ($bodylen > 0) {
	        switch ($method) {
	        case 8:
	            // Currently the only supported compression method:
	            $data = gzinflate($body,$maxlength);
	            break;
	        default:
	            $error = "Unknown compression method.";
	            return false;
	        }
	    }  // zero-byte body content is allowed
	    // Verifiy CRC32
	    $crc   = sprintf("%u",crc32($data));
	    $crcOK = $crc == $datacrc;
	    $lenOK = $isize == strlen($data);
	    if (!$lenOK || !$crcOK) {
	        $error = ( $lenOK ? '' : 'Length check FAILED. ') . ( $crcOK ? '' : 'Checksum FAILED.');
	        return false;
	    }
	    return $data;
	}