unpack_from requires a buffer of at least 4 bytes

Question

I am receiving a packet from client, consisting of many fields. I read all fields successfully, but when it comes to the last field which is tag_end, python gives me an error:

unpack_from requires a buffer of at least 4 bytes not found.

this is the code:

def set_bin(self, buf):
    """Reads a vector of bytes (probably received from network or
    read from file) and tries to construct the packet structure
    from it, by reading each packet member from the buffer.  This
    is somehow like deserializing the packet.
    """
    assert isinstance(buf, bytearray), 'buffer type is not valid'
    offset = 0

    print("$$$$$$$$$$$$$$$$ set bin $$$$$$$$$$$$$$$$$")

    try:
        (self._tag_start, self._version, self._checksum, self._connection_id,
         self._packet_seq) = Packet.PACKER_1.unpack_from(str(buf), offset)
    except struct.error as e:
        print(e)
        raise DeserializeError(e)
    except ValueError as e:
        print(e)
        raise DeserializeError(e)
          #I=4 H=2 B=1
    offset = Packet.OFFSET_GUID     #14 correct
    self._guid = buf[offset:offset+Packet.UUID_SIZE] #14-16 correct

    offset = Packet.OFFSET_GUID + Packet.UUID_SIZE

    print("$$$$$$$$$$$$$$$$ GUID read successfully  $$$$$$$$$$$$$$$$$")

    try:
       (self._timestamp_sec, self._timestamp_microsec, self._command,
        self._command_seq, self._subcommand, self._data_seq,
        self._data_length) = Packet.PACKER_3.unpack_from(str(buf), offset)
    except struct.error as e:
        print(e)
        raise DeserializeError(e)
    except ValueError as e:
        print(e)
        raise DeserializeError(e)

    print("$$$$$$$$$$$$$$$$ timestamps read successfully $$$$$$$$$$$$$$$$$")
   offset = Packet.OFFSET_AUTHENTICATE
    self._username = buf[offset:offset + self.USERNAME_SIZE]        #Saman
    offset += self.USERNAME_SIZE

    print("$$$$$$$$$$$$$$$$ username read successfully  $$$$$$$$$$$$$$$$$")

    self._password = buf[offset:offset+self.USERNAME_SIZE]
    offset += self.PASSWORD_SIZE

    print("$$$$$$$$$$$$$$$$ password read successfully $$$$$$$$$$$$$$$$$")

    self._data = buf[offset:offset+self._data_length]
    offset = offset + self._data_length

    print("$$$$$$$$$$$$$$$$ data read successfully  $$$$$$$$$$$$$$$$$")

    try:
        (self._tag_end,) = Packet.PACKER_4.unpack_from(str(buf), offset)
    except struct.error as e:
        print(e)
        raise DeserializeError(e)
    except ValueError as e:
        print(e)
        raise DeserializeError(e)

    print("$$$$$$$$$$$$$$$$ tag end read successfully $$$$$$$$$$$$$$$$$")


    if len(buf) != Packet.PACKER.size + self._data_length:
        print('failed to deserialize binary data correctly and construct the packet due to extra data')
    else:
        print('@@@@@@@@@@@@@@@ Deserialized Successfully')

and this is some constants used in the code:

STRUCT_FORMAT_STR   = r'=IHIHH 16B IIHHHHH I 6c 9c' #Saman
STRUCT_FORMAT_STR_1 = r'=IHIHH'
STRUCT_FORMAT_STR_2 = r'=16B'
STRUCT_FORMAT_STR_3 = r'=IIHHHHH'
STRUCT_FORMAT_STR_4 = r'=I'
STRUCT_FORMAT_STR_5 = r'=6c'
STRUCT_FORMAT_STR_6 = r'=9c'
UUID_SIZE   = 16
OFFSET_GUID = 14
#OFFSET_DATA = 48     #shifting offset data by 15 char      
OFFSET_AUTHENTICATE = 48
PACKER   = struct.Struct(str(STRUCT_FORMAT_STR))     #Saman
PACKER_1 = struct.Struct(str(STRUCT_FORMAT_STR_1))
PACKER_2 = struct.Struct(str(STRUCT_FORMAT_STR_2))
PACKER_3 = struct.Struct(str(STRUCT_FORMAT_STR_3))
PACKER_4 = struct.Struct(str(STRUCT_FORMAT_STR_4))
PACKER_5 = struct.Struct(str(STRUCT_FORMAT_STR_5))
PACKER_6 = struct.Struct(str(STRUCT_FORMAT_STR_6))
BYTES_TAG_START = PACKER_4.pack(TAG_START)
BYTES_TAG_END   = PACKER_4.pack(TAG_END)

and initialization of the packet object, where it initializes the fields:

def init(self, **kwargs): if 'buf' in kwargs: self.set_bin(kwargs['buf']) else: assert kwargs['command'] in Packet.RTCINET_COMMANDS.values() and kwargs['subcommand'] in Packet.RTCINET_COMMANDS.values(), 'Undefined protocol command' assert isinstance(kwargs['data'], bytearray), 'invalid type for data field' for field in ('command', 'subcommand', 'data'): setattr(self, '_' + field, kwargs[field])

    self._tag_start = Packet.TAG_START
    self._version = Packet.VERSION_CURRENT % (Packet.USHRT_MAX + 1)
    self._checksum = Packet.CRC_INIT
    self._connection_id = kwargs.get('connection_id', 0) % (Packet.USHRT_MAX + 1)
    self._packet_seq = Packet.PACKET_SEQ
    Packet.PACKET_SEQ = (Packet.PACKET_SEQ + 1) % (Packet.USHRT_MAX + 1)
    self._guid = uuid.uuid4().bytes
    dt = datetime.datetime.now()
    self._timestamp_sec = int(time.mktime(dt.timetuple()))
    self._timestamp_microsec = dt.microsecond
    # self._command = kwargs['command']
    self._command_seq = kwargs.get('command_seq', 0)
    # self._subcommand = kwargs['subcommand']
    self._data_seq = kwargs.get('data_seq', 0)
    self._data_length = len(kwargs['data'])

    self._username = Packet.USERNAME            #Saman
    self._password = Packet.PASSWORD

I have made sure that I read all fields in the right order, as it was written in the packet by the client program. but still I couldn't manage to solve this problem.

Do you have any idea how this could be solved?

Answer 1

The problem seems to be that you're converting things to str all over the place for no good reason.

In some places, like PACKER_1 = struct.Struct(str(STRUCT_FORMAT_STR_1)), it makes your code less readable and understandable, but doesn't affect the actual output. For example, STRUCT_FORMAT_STR_1 is already a str, so str(STRUCT_FORMAT_STR_1) is the same str.

But in other places, it's far worse than that. In particular, look at all the lines like Packet.PACKER_1.unpack_from(str(buf), offset). There, buf is a bytearray. (It has to be, because you assert it.) Calling str on a bytearray gives you the string representation of that bytearray. For example:

>>> b = bytearray(b'abc')
>>> len(b)
3
>>> s = str(b)
>>> s
"bytearray(b'abc')"
>>> len(s)
17

That string representation is obviously not generally going to have the same length as the actual buffer you're representing. So it's no wonder that you get errors about the length being wrong. (And if you got really unlucky and didn't have any such errors, you'd be reading garbage values instead.)

So, what should you do to convert the bytearray into something the struct module can handle? Nothing! As the docs say:

Several struct functions (and methods of Struct) take a buffer argument. This refers to objects that implement the Buffer Protocol and provide either a readable or read-writable buffer. The most common types used for that purpose are bytes and bytearray…