Breaks data stream (bytes) into a list of bytes objects containing single instructions each.

    Logs warnings for unknown opcodes or raises an exception instead, if raise_exception is set to True.

    returns: list of bytes objects
190210
def chunker(data, raise_exception=False):
    
    instructions = []
    data = bytes(data)
    while True:
        if len(data) == 0: break
        try:
            opcode = match_opcode(data)
        except:
            msg = 'unknown opcode starting with {}...)'.format(hex_format(data[0:4]))
            if raise_exception:
                raise ValueError(msg)
            else:
                logger.warning(msg)
                data = data[1:]
                continue
        opcode_def = OPCODES[opcode]
        num_bytes = len(opcode)
        if opcode_def[1] > 0: num_bytes += opcode_def[1]
        if 'raster' in opcode_def[0]:
            num_bytes += data[2] + 2
        #payload = data[len(opcode):num_bytes]
        instructions.append(data[:num_bytes])
        data = data[num_bytes:]
    return instructions
621662
def parse(self, data: bytes, context=None):
        
        stream = BytesIO(data)
        return self.parse_stream(stream, context)
975850
def encode_list(data, encoding=None, errors='strict', keep=False,
                preserve_dict_class=False, preserve_tuples=False):
    '''
    
    '''
    rv = []
    for item in data:
        if isinstance(item, list):
            item = encode_list(item, encoding, errors, keep,
                               preserve_dict_class, preserve_tuples)
        elif isinstance(item, tuple):
            item = encode_tuple(item, encoding, errors, keep, preserve_dict_class) \
                if preserve_tuples \
                else encode_list(item, encoding, errors, keep,
                                 preserve_dict_class, preserve_tuples)
        elif isinstance(item, Mapping):
            item = encode_dict(item, encoding, errors, keep,
                               preserve_dict_class, preserve_tuples)
        else:
            try:
                item = salt.utils.stringutils.to_bytes(item, encoding, errors)
            except TypeError:
                # to_bytes raises a TypeError when input is not a
                # string/bytestring/bytearray. This is expected and simply
                # means we are going to leave the value as-is.
                pass
            except UnicodeEncodeError:
                if not keep:
                    raise

        rv.append(item)
    return rv
296992
def encode_streaming(self, data):
        
        # Buffer value and size
        buffer = 0
        size = 0
        for s in data:
            b, v = self._table[s]
            # Shift new bits in the buffer
            buffer = (buffer << b) + v
            size += b
            while size >= 8:
                byte = buffer >> (size - 8)
                yield to_byte(byte)
                buffer = buffer - (byte << (size - 8))
                size -= 8

        # Handling of the final sub-byte chunk.
        # The end of the encoded bit stream does not align necessarily with byte boundaries,
        # so we need an "end of file" indicator symbol (_EOF) to guard against decoding
        # the non-data trailing bits of the last byte.
        # As an optimization however, while encoding _EOF, it is only necessary to encode up to
        # the end of the current byte and cut off there.
        # No new byte has to be started for the remainder, saving us one (or more) output bytes.
        if size > 0:
            b, v = self._table[_EOF]
            buffer = (buffer << b) + v
            size += b
            if size >= 8:
                byte = buffer >> (size - 8)
            else:
                byte = buffer << (8 - size)
            yield to_byte(byte)
110866
def get_messages_from_stream(data):
    
    messages = []
    iterator = HEADER_RE.finditer(data)
    last_pos = 0
    for match in iterator:
        pos = match.span()[0]
        header = read_machine_header(data[pos:])
        h_len = __get_machine_header_length(header)
        cur_last_pos = pos + h_len + header['meta_len'] + header['data_len']

        if cur_last_pos > len(data):
            break

        header, meta, bin_data = parse_message(data[pos:])
        messages.append({'header': header, 'meta': meta, 'data': bin_data})

        last_pos = cur_last_pos

    data = data[last_pos:]
    return messages, data
1338953
def build_stream(self, obj, stream: BytesIO, context=None) -> None:
        
        if context is None:
            context = Context()
        if not isinstance(context, Context):
            context = Context(context)
        try:
            self._build_stream(obj, stream, context)
        except Error:
            raise
        except Exception as exc:
            raise BuildingError(str(exc))
119568
def _data_block(stream):
    
    while len(stream) > 0:
        line = stream.popleft()

        if line.startswith('>'):
            yield DataHeader(line[1:].strip())
        else:
            data_item = line.strip()
            if data_item:
                yield DataItem(line)
            else:
                continue
924213
def load_stream(self, stream):
        
        # load the batches
        for batch in self.serializer.load_stream(stream):
            yield batch

        # load the batch order indices
        num = read_int(stream)
        batch_order = []
        for i in xrange(num):
            index = read_int(stream)
            batch_order.append(index)
        yield batch_order
684149
def parse_buffer(stream, separator=None):
    
    rows = []
    lines = []
    for line, row in parse_lines(stream, separator):
        lines.append(line)
        rows.append(row)
    cols = zip(*rows)
    return {
        'rows': rows,
        'lines': lines,
        'cols': cols,
        }
1251377
def opcode_list(self, script):
        """"""
        opcodes = []
        new_pc = 0
        try:
            for opcode, data, pc, new_pc in self.get_opcodes(script):
                opcodes.append(self.disassemble_for_opcode_data(opcode, data))
        except ScriptError:
            opcodes.append(binascii.hexlify(script[new_pc:]).decode("utf8"))

        return opcodes
278163
def _get_info(self):
        
        out_parts = []
        while True:
            line = self.proc.stderr.readline()
            if not line:
                # EOF and data not found.
                raise CommunicationError("stream info not found")

            # In Python 3, result of reading from stderr is bytes.
            if isinstance(line, bytes):
                line = line.decode('utf8', 'ignore')

            line = line.strip().lower()

            if 'no such file' in line:
                raise IOError('file not found')
            elif 'invalid data found' in line:
                raise UnsupportedError()
            elif 'duration:' in line:
                out_parts.append(line)
            elif 'audio:' in line:
                out_parts.append(line)
                self._parse_info(''.join(out_parts))
                break
950211
def list_packets(self, raw_data):
        """"""
        args = ["--list-packets"]
        result = self._result_map['packets'](self)
        self._handle_io(args, _make_binary_stream(raw_data, self._encoding),
                        result)
        return result
126798
def decode_streaming(self, data):
        
        # Reverse lookup table: map (bitsize, value) to symbols
        lookup = dict(((b, v), s) for (s, (b, v)) in self._table.items())

        buffer = 0
        size = 0
        for byte in data:
            for m in [128, 64, 32, 16, 8, 4, 2, 1]:
                buffer = (buffer << 1) + bool(from_byte(byte) & m)
                size += 1
                if (size, buffer) in lookup:
                    symbol = lookup[size, buffer]
                    if symbol == _EOF:
                        return
                    yield symbol
                    buffer = 0
                    size = 0
204074
def from_stream(cls, stream):
        
        chunk_parser = _ChunkParser.from_stream(stream)
        chunks = [chunk for chunk in chunk_parser.iter_chunks()]
        return cls(chunks)
497355
def list2bytecode(l, opc, varnames, consts):
    
    bc = []
    for i, opcodes in enumerate(l):
        opname = opcodes[0]
        operands = opcodes[1:]
        if opname not in opc.opname:
            raise TypeError(
                "error at item %d [%s, %s], opcode not valid" %
                (i, opname, operands))
        opcode = opc.opmap[opname]
        bc.append(opcode)
        print(opname, operands)
        gen = (j for j in operands if operands)
        for j in gen:
            k = (consts if opcode in opc.CONST_OPS else varnames).index(j)
            if k == -1:
                raise TypeError(
                    "operand %s [%s, %s], not found in names" %
                    (i, opname, operands))
            else:
                bc += num2code(k)
                pass
            pass
        pass
    if opc.python_version < 3.0:
        return reduce(lambda a, b: a + chr(b), bc, '')
    else:
        if PYTHON3:
            return bytes(bc)
        else:
            return bytes(bytearray(bc))
516431
def instruction_list_to_easm(instruction_list: list) -> str:
    
    result = ""

    for instruction in instruction_list:
        result += "{} {}".format(instruction["address"], instruction["opcode"])
        if "argument" in instruction:
            result += " " + instruction["argument"]
        result += "\n"

    return result
215756
def dump(self):
        
        org = min(self.memory_bytes.keys())  # Org is the lowest one
        OUTPUT = []
        align = []

        for i in range(org, max(self.memory_bytes.keys()) + 1):
            if gl.has_errors:
                return org, OUTPUT

            try:
                try:
                    a = [x for x in self.orgs[i] if isinstance(x, Asm)]  # search for asm instructions

                    if not a:
                        align.append(0)  # Fill with ZEROes not used memory regions
                        continue

                    OUTPUT += align
                    align = []
                    a = a[0]
                    if a.pending:
                        a.arg = a.argval()
                        a.pending = False
                        tmp = a.bytes()

                        for r in range(len(tmp)):
                            self.memory_bytes[i + r] = tmp[r]
                except KeyError:
                    pass

                OUTPUT.append(self.memory_bytes[i])

            except KeyError:
                OUTPUT.append(0)  # Fill with ZEROes not used memory regions

        return org, OUTPUT