Extract binary components in the packet.
335859
def _deconstruct_binary(self, data):
        """"""
        attachments = []
        data = self._deconstruct_binary_internal(data, attachments)
        return data, attachments
1355680
def as_binary(self, content, encoding='utf8'):
        ''''''
        if hasattr(content, 'read'):
            return content.read()
        elif isinstance(content, six.text_type):
            return content.encode(encoding)
        else:
            return content
1002922
def extract_frames(self, bpf_buffer):
        """"""  # noqa: E501

        # Ensure that the BPF buffer contains at least the header
        len_bb = len(bpf_buffer)
        if len_bb < 20:  # Note: 20 == sizeof(struct bfp_hdr)
            return

        # Extract useful information from the BPF header
        if FREEBSD or NETBSD:
            # struct bpf_xhdr or struct bpf_hdr32
            bh_tstamp_offset = 16
        else:
            # struct bpf_hdr
            bh_tstamp_offset = 8

        # Parse the BPF header
        bh_caplen = struct.unpack('I', bpf_buffer[bh_tstamp_offset:bh_tstamp_offset + 4])[0]  # noqa: E501
        next_offset = bh_tstamp_offset + 4
        bh_datalen = struct.unpack('I', bpf_buffer[next_offset:next_offset + 4])[0]  # noqa: E501
        next_offset += 4
        bh_hdrlen = struct.unpack('H', bpf_buffer[next_offset:next_offset + 2])[0]  # noqa: E501
        if bh_datalen == 0:
            return

        # Get and store the Scapy object
        frame_str = bpf_buffer[bh_hdrlen:bh_hdrlen + bh_caplen]
        try:
            pkt = self.guessed_cls(frame_str)
        except Exception:
            if conf.debug_dissector:
                raise
            pkt = conf.raw_layer(frame_str)
        self.received_frames.append(pkt)

        # Extract the next frame
        end = self.bpf_align(bh_hdrlen, bh_caplen)
        if (len_bb - end) >= 20:
            self.extract_frames(bpf_buffer[end:])
1058950
def pull(self):
        """"""

        # For each packet in the pcap process the contents
        for item in self.input_stream:

            # Print out the timestamp in UTC
            print('%s -' % item['timestamp'], end='')

            # Transport info
            if item['transport']:
                print(item['transport']['type'], end='')

            # Print out the Packet info
            packet_type = item['packet']['type']
            print(packet_type, end='')
            packet = item['packet']
            if packet_type in ['IP', 'IP6']:
                # Is there domain info?
                if 'src_domain' in packet:
                    print('%s(%s) --> %s(%s)' % (net_utils.inet_to_str(packet['src']), packet['src_domain'],
                                                 net_utils.inet_to_str(packet['dst']), packet['dst_domain']), end='')
                else:
                    print('%s --> %s' % (net_utils.inet_to_str(packet['src']), net_utils.inet_to_str(packet['dst'])), end='')
            else:
                print(str(packet))

            # Only include application if we have it
            if item['application']:
                print('Application: %s' % item['application']['type'], end='')
                print(str(item['application']), end='')

            # Just for newline
            print()
1166798
def _parse (self):
        """"""

        feedparser = FeedParser (Message)
        binarychunks = {}
        sizeinfo = None
        headxml = None
        self.fp.seek (0, 0)

        while True:
            data = self.fp.readline ()
            if not data:
                break

            feedparser.feed (data)

            skip = (data == '\n' and
                    len (feedparser._msgstack) == 3 and
                    feedparser._msgstack[-1].get_content_type () in ('application/octet-stream',
                                                                     'binary/octet-stream'))
            if skip:
                # We just finished reading the headers for a huge binary blob.
                # Time to remember where the data chunk is and pretend it doesn't
                # exist.
                msg = feedparser._msgstack[-1]
                ident = msg['Content-Location']
                assert ident.endswith ('.bin'), 'confusion #1 in hacky MIME parsing!'
                binarychunks[ident] = self.fp.tell ()
                if sizeinfo is None:
                    headxml, sizeinfo, tagpfx = _extract_size_info (feedparser)
                kind = ident.split ('/')[-1]
                assert kind in sizeinfo, 'no size info for binary chunk kind %s in MIME!' % kind
                self.fp.seek (sizeinfo[kind] + 1, 1) # skip ahead by data chunk size
                sample = self.fp.read (16)
                assert sample.startswith ('--MIME'), 'crap, unexpected chunk size in MIME parsing: %r' % sample
                self.fp.seek (-16, 1) # go back

            # check that two major kinds of data are read at least once
            if any([k.split('/')[3] == '3' for k in binarychunks.iterkeys()]):
                break

        if headxml is None:
            raise RuntimeError ('never found any binary data')

        self.mimemsg = feedparser.close ()
        self.headxml = headxml
        self.sizeinfo = sizeinfo
        self.binarychunks = binarychunks

        headsize, intsize = self.calc_intsize()

        # Compute some miscellaneous parameters that we'll need.
#        self.n_integrations = len (self.mimemsg.get_payload ()) - 1
        self.n_integrations = os.stat(self.fp.name).st_size/intsize
        self.n_antennas = int (headxml.find (tagpfx + nanttag).text)
        self.n_baselines = (self.n_antennas * (self.n_antennas - 1)) // 2

        ds = headxml.find (tagpfx + dstag)
        nbb = 0
        nspw = 0
        nchan = 0
        crosspolstr = None

        for bb in ds.findall (tagpfx + basebandtag):
            nbb += 1

            for spw in bb.getchildren ():
                nspw += 1
                nchan += int (spw.get ('numSpectralPoint'))

                if crosspolstr is None:
                    crosspolstr = spw.get ('crossPolProducts')
                elif spw.get ('crossPolProducts') != crosspolstr:
                    raise Exception ('can only handle spectral windows with identical cross pol products')

        self.n_basebands = nbb
        self.n_spws = nspw
        self.n_channels = nchan
        self.crosspols = crosspolstr.split ()
        self.n_pols = len(self.crosspols)

        # if bdf info pkl not present, write it
        if os.path.exists(os.path.dirname(self.pklname)) and self.pklname and (not os.path.exists(self.pklname)):
            logger.info('Writing bdf pkl info to %s...' % (self.pklname))
            with open(self.pklname,'wb') as pkl:
                # Compute some miscellaneous parameters that we'll need.
                pickle.dump( (self.mimemsg, self.headxml, self.sizeinfo, self.binarychunks, self.n_integrations, self.n_antennas, self.n_baselines, self.n_basebands, self.n_spws, self.n_channels, self.crosspols), pkl)

        return self
285212
def frombinary(self, s):
        """"""
        entrylen = struct.calcsize(self.ENTRYSTRUCT)
        p = 0
        while p<len(s):
            (slen, dpos, dlen, ulen, flag, typcd) = struct.unpack(self.ENTRYSTRUCT,
                                                        s[p:p+entrylen])
            nmlen = slen - entrylen
            p = p + entrylen
            (nm,) = struct.unpack(`nmlen`+'s', s[p:p+nmlen])
            p = p + nmlen
            # version 4
            # self.data.append((dpos, dlen, ulen, flag, typcd, nm[:-1]))
            # version 5
            # nm may have up to 15 bytes of padding
            pos = nm.find('\0')
            if pos < 0:
                self.data.append((dpos, dlen, ulen, flag, typcd, nm))
            else:
                self.data.append((dpos, dlen, ulen, flag, typcd, nm[:pos]))
609665
def from_bytes(cls, bitstream, decode_payload=True):
        '''
        
        '''
        packet = cls()

        # Convert to ConstBitStream (if not already provided)
        if not isinstance(bitstream, ConstBitStream):
            if isinstance(bitstream, Bits):
                bitstream = ConstBitStream(auto=bitstream)
            else:
                bitstream = ConstBitStream(bytes=bitstream)

        # Read the version
        version = bitstream.read('uint:4')
        if version != packet.version:
            raise ValueError('Provided bytes do not contain an IPv4 packet')

        # Read the header length
        ihl = bitstream.read('uint:4')
        if ihl < 5:
            raise ValueError('Invalid IPv4 header length')

        # Now that we know the length of the header we store it to be able
        # to easily recalculate the header checksum later
        remaining_header_bits = (ihl * 32) - 8
        header = (BitStream('uint:4=4, uint:4=%d' % ihl) +
                  bitstream.peek(remaining_header_bits))

        # Read the type of service
        packet.tos = bitstream.read('uint:8')

        # Read the total length
        total_length = bitstream.read('uint:16')
        if total_length < ihl * 4:
            raise ValueError('Total length is shorter than the header')

        # Read the identification
        packet.identification = bitstream.read('uint:16')

        # Read the flags
        (reserved,
         packet.dont_fragment,
         packet.more_fragments) = bitstream.readlist('3*bool')

        if reserved:
            raise ValueError('Reserved flag must be 0')

        # Read the fragment offset
        packet.fragment_offset = bitstream.read('uint:13')

        # Read the TTL
        packet.ttl = bitstream.read('uint:8')

        # Read the protocol number
        packet.protocol = bitstream.read('uint:8')

        # Read the header checksum
        header_checksum = bitstream.read('uint:16')

        # Set the checksum bits in the header to 0 and re-calculate
        header[80:96] = BitStream(16)
        my_checksum = checksum.ones_complement(header.bytes)

        if my_checksum != header_checksum:
            raise ValueError('Header checksum does not match')

        # Read the source and destination addresses
        packet.source = IPv4Address(bitstream.read('uint:32'))
        packet.destination = IPv4Address(bitstream.read('uint:32'))

        # Read the options
        option_len = (ihl - 5) * 4
        packet.options = bitstream.read('bytes:%d' % option_len)

        # And the rest is payload
        payload_bytes = (total_length) - (ihl * 4)
        packet.payload = bitstream.read('bytes:%d' % payload_bytes)

        if decode_payload:
            payload_class = protocol_registry.get_type_class(packet.protocol)
            if payload_class:
                packet.payload = payload_class.from_bytes(packet.payload)

        # There should be no remaining bits
        if bitstream.pos != bitstream.len:
            raise ValueError('Bits remaining after processing packet')

        # Verify that the properties make sense
        packet.sanitize()

        return packet
698658
function matchBinaries (input, binary) {
  const regExp = bytecodeToBytecodeRegex(binary);
  return (input.match(regExp) !== null);
}
1290375
def get_binary_property(value, is_bytes=False):
    """"""

    obj = unidata.ascii_binary if is_bytes else unidata.unicode_binary

    if value.startswith('^'):
        negated = value[1:]
        value = '^' + unidata.unicode_alias['binary'].get(negated, negated)
    else:
        value = unidata.unicode_alias['binary'].get(value, value)

    return obj[value]
521236
def unpack_from(cls, payload, expected_parts):
        """"""

        for num_part in iter_range(expected_parts):
            hdr = payload.read(cls.header_size)
            try:
                part_header = PartHeader(*cls.header_struct.unpack(hdr))
            except struct.error:
                raise InterfaceError("No valid part header")

            if part_header.payload_size % 8 != 0:
                part_payload_size = part_header.payload_size + 8 - (part_header.payload_size % 8)
            else:
                part_payload_size = part_header.payload_size
            pl = payload.read(part_payload_size)
            part_payload = io.BytesIO(pl)
            try:
                _PartClass = PART_MAPPING[part_header.part_kind]
            except KeyError:
                raise InterfaceError("Unknown part kind %s" % part_header.part_kind)

            debug('%s (%d/%d): %s', _PartClass.__name__, num_part+1, expected_parts, str(part_header))
            debug('Read %d bytes payload for part %d', part_payload_size, num_part + 1)

            init_arguments = _PartClass.unpack_data(part_header.argument_count, part_payload)
            debug('Part data: %s', init_arguments)
            part = _PartClass(*init_arguments)
            part.header = part_header
            part.attribute = part_header.part_attributes
            part.source = 'server'
            if pyhdb.tracing:
                part.trace_header = humanhexlify(hdr[:part_header.payload_size])
                part.trace_payload = humanhexlify(pl, 30)
            yield part
344886
def _sense_packet_to_data(packet):
        """"""
        data = bytearray(0)
        last = 0
        i = 1
        while (i < len(packet)):
            if not (last == 2 and packet[i] == 0):
                data.append(packet[i])
            last = packet[i]
            i += 1

        return data
1035659
def decode_packet(data):
    """"""
    if _debug: decode_packet._debug("decode_packet %r", data)

    # empty strings are some other kind of pcap content
    if not data:
        return None

    # assume it is ethernet for now
    d = decode_ethernet(data)
    pduSource = Address(d['source_address'])
    pduDestination = Address(d['destination_address'])
    data = d['data']

    # there could be a VLAN header
    if (d['type'] == 0x8100):
        if _debug: decode_packet._debug("    - vlan found")

        d = decode_vlan(data)
        data = d['data']

    # look for IP packets
    if (d['type'] == 0x0800):
        if _debug: decode_packet._debug("    - IP found")

        d = decode_ip(data)
        pduSource, pduDestination = d['source_address'], d['destination_address']
        data = d['data']

        if (d['protocol'] == 'udp'):
            if _debug: decode_packet._debug("    - UDP found")

            d = decode_udp(data)
            data = d['data']

            pduSource = Address((pduSource, d['source_port']))
            pduDestination = Address((pduDestination, d['destination_port']))
            if _debug:
                decode_packet._debug("    - pduSource: %r", pduSource)
                decode_packet._debug("    - pduDestination: %r", pduDestination)
        else:
            if _debug: decode_packet._debug("    - not a UDP packet")
    else:
        if _debug: decode_packet._debug("    - not an IP packet")

    # check for empty
    if not data:
        if _debug: decode_packet._debug("    - empty packet")
        return None

    # build a PDU
    pdu = PDU(data, source=pduSource, destination=pduDestination)

    # check for a BVLL header
    if (pdu.pduData[0] == 0x81):
        if _debug: decode_packet._debug("    - BVLL header found")

        try:
            xpdu = BVLPDU()
            xpdu.decode(pdu)
            pdu = xpdu
        except Exception as err:
            if _debug: decode_packet._debug("    - BVLPDU decoding error: %r", err)
            return pdu

        # make a more focused interpretation
        atype = bvl_pdu_types.get(pdu.bvlciFunction)
        if not atype:
            if _debug: decode_packet._debug("    - unknown BVLL type: %r", pdu.bvlciFunction)
            return pdu

        # decode it as one of the basic types
        try:
            xpdu = pdu
            bpdu = atype()
            bpdu.decode(pdu)
            if _debug: decode_packet._debug("    - bpdu: %r", bpdu)

            pdu = bpdu

            # lift the address for forwarded NPDU's
            if atype is ForwardedNPDU:
                pdu.pduSource = bpdu.bvlciAddress
            # no deeper decoding for some
            elif atype not in (DistributeBroadcastToNetwork, OriginalUnicastNPDU, OriginalBroadcastNPDU):
                return pdu

        except Exception as err:
            if _debug: decode_packet._debug("    - decoding Error: %r", err)
            return xpdu

    # check for version number
    if (pdu.pduData[0] != 0x01):
        if _debug: decode_packet._debug("    - not a version 1 packet: %s...", btox(pdu.pduData[:30], '.'))
        return None

    # it's an NPDU
    try:
        npdu = NPDU()
        npdu.decode(pdu)
    except Exception as err:
        if _debug: decode_packet._debug("    - decoding Error: %r", err)
        return None

    # application or network layer message
    if npdu.npduNetMessage is None:
        if _debug: decode_packet._debug("    - not a network layer message, try as an APDU")

        # decode as a generic APDU
        try:
            xpdu = APDU()
            xpdu.decode(npdu)
            apdu = xpdu
        except Exception as err:
            if _debug: decode_packet._debug("    - decoding Error: %r", err)
            return npdu

        # "lift" the source and destination address
        if npdu.npduSADR:
            apdu.pduSource = npdu.npduSADR
        else:
            apdu.pduSource = npdu.pduSource
        if npdu.npduDADR:
            apdu.pduDestination = npdu.npduDADR
        else:
            apdu.pduDestination = npdu.pduDestination

        # make a more focused interpretation
        atype = apdu_types.get(apdu.apduType)
        if not atype:
            if _debug: decode_packet._debug("    - unknown APDU type: %r", apdu.apduType)
            return apdu

        # decode it as one of the basic types
        try:
            xpdu = apdu
            apdu = atype()
            apdu.decode(xpdu)
        except Exception as err:
            if _debug: decode_packet._debug("    - decoding Error: %r", err)
            return xpdu

        # decode it at the next level
        if isinstance(apdu, ConfirmedRequestPDU):
            atype = confirmed_request_types.get(apdu.apduService)
            if not atype:
                if _debug: decode_packet._debug("    - no confirmed request decoder: %r", apdu.apduService)
                return apdu

        elif isinstance(apdu, UnconfirmedRequestPDU):
            atype = unconfirmed_request_types.get(apdu.apduService)
            if not atype:
                if _debug: decode_packet._debug("    - no unconfirmed request decoder: %r", apdu.apduService)
                return apdu

        elif isinstance(apdu, SimpleAckPDU):
            atype = None

        elif isinstance(apdu, ComplexAckPDU):
            atype = complex_ack_types.get(apdu.apduService)
            if not atype:
                if _debug: decode_packet._debug("    - no complex ack decoder: %r", apdu.apduService)
                return apdu

        elif isinstance(apdu, SegmentAckPDU):
            atype = None

        elif isinstance(apdu, ErrorPDU):
            atype = error_types.get(apdu.apduService)
            if not atype:
                if _debug: decode_packet._debug("    - no error decoder: %r", apdu.apduService)
                return apdu

        elif isinstance(apdu, RejectPDU):
            atype = None

        elif isinstance(apdu, AbortPDU):
            atype = None
        if _debug: decode_packet._debug("    - atype: %r", atype)

        # deeper decoding
        try:
            if atype:
                xpdu = apdu
                apdu = atype()
                apdu.decode(xpdu)
        except Exception as err:
            if _debug: decode_packet._debug("    - decoding error: %r", err)
            return xpdu

        # success
        return apdu

    else:
        # make a more focused interpretation
        ntype = npdu_types.get(npdu.npduNetMessage)
        if not ntype:
            if _debug: decode_packet._debug("    - no network layer decoder: %r", npdu.npduNetMessage)
            return npdu
        if _debug: decode_packet._debug("    - ntype: %r", ntype)

        # deeper decoding
        try:
            xpdu = npdu
            npdu = ntype()
            npdu.decode(xpdu)
        except Exception as err:
            if _debug: decode_packet._debug("    - decoding error: %r", err)
            return xpdu

        # success
        return npdu
368793
def _process(self, data):
        ""
        parts = data.split(':')
        if self._debug:
            self.error('packet: %r' % data)
        if not parts:
            return

        # interpret the packet and update stats
        stats = self._stats
        key = parts[0].translate(KEY_TABLE, KEY_DELETIONS)
        if self._key_prefix:
            key = '.'.join([self._key_prefix, key])
        for part in parts[1:]:
            srate = 1.0
            fields = part.split('|')
            length = len(fields)
            if length < 2:
                continue
            value = fields[0]
            stype = fields[1].strip()

            with stats_lock:
                # timer (milliseconds)
                if stype == 'ms':
                    stats.timers[key].append(float(value if value else 0))

                # counter with optional sample rate
                elif stype == 'c':
                    if length == 3 and fields[2].startswith('@'):
                        srate = float(fields[2][1:])
                    value = float(value if value else 1) * (1 / srate)
                    stats.counts[key] += value
                elif stype == 'g':
                    value = float(value if value else 1)
                    stats.gauges[key] = value
896832
def decode(binary):
    """"""
    encoded = io.BytesIO(binary)
    with gzip.GzipFile(mode='rb', fileobj=encoded) as file_:
        decoded = file_.read()
    return decoded
858654
def reconstruct_binary(self, attachments):
        
        self.data = self._reconstruct_binary_internal(self.data,
                                                      self.attachments)
950211
def list_packets(self, raw_data):
        """"""
        args = ["--list-packets"]
        result = self._result_map['packets'](self)
        self._handle_io(args, _make_binary_stream(raw_data, self._encoding),
                        result)
        return result
994842
def parse_packet(data, format=None):
    
    unpacked = struct.unpack(BASE_FORMAT, data[:_FORMAT_SIZE])
    psize, protocol, mac, gateway, time, ptype = unpacked
    header = Header(psize, protocol, mac, gateway, time, ptype)
    return header, data[_FORMAT_SIZE:]