functional_tests: Python epee binary lib and test /get_blocks.bin

Co-authored-by: selsta <selsta@sent.at>

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -229,7 +229,7 @@ namespace epee
       case SERIALIZE_TYPE_OBJECT: return read_ae<section>();
       case SERIALIZE_TYPE_ARRAY:  return read_ae<array_entry>();
       default: 
-        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << type);
+        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << static_cast<int>(type));
       }
       return read_ae<int8_t>(); // unreachable, dummy return to avoid compiler warning
     }
@@ -321,7 +321,7 @@ namespace epee
       case SERIALIZE_TYPE_OBJECT: return read_se<section>();
       case SERIALIZE_TYPE_ARRAY:  return read_se<array_entry>();
       default: 
-        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << ent_type);
+        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << static_cast<int>(ent_type));
       }
       return read_se<int8_t>(); // unreachable, dummy return to avoid compiler warning
     }

tests/functional_tests/blockchain.py

@@ -43,12 +43,14 @@ Test the following RPCs:
 """
 
 from framework.daemon import Daemon
+from framework.wallet import Wallet
 
 class BlockchainTest():
     def run_test(self):
         self.reset()
         self._test_generateblocks(5)
         self._test_alt_chains()
+        self.test_get_blocks_fast()
 
     def reset(self):
         print('Resetting blockchain')
@@ -343,6 +345,102 @@ class BlockchainTest():
         print('Saving blockchain explicitely')
         daemon.save_bc()
 
+    def test_get_blocks_fast(self):
+        print('Testing the /get_blocks.bin RPC endpoint')
+
+        daemon = Daemon()
+
+        n_blocks = daemon.get_height()['height']
+        assert(n_blocks >= 3)
+
+        genesis_block_id = daemon.getblockheaderbyheight(0)['block_header']['hash']
+
+        target_block_index = n_blocks-1
+        target_block_id = daemon.getblockheaderbyheight(target_block_index)['block_header']['hash']
+
+        print('First, mining to wallet and creating 2 transactions in the top block')
+
+        wallet = Wallet()
+        seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
+        main_address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+
+        try: wallet.close_wallet()
+        except: pass
+        wallet.auto_refresh(enable = False)
+        wallet.restore_deterministic_wallet(seed)
+        assert wallet.get_transfers() == {}
+        assert wallet.get_address().address == main_address
+
+        wallet.refresh()
+        res = wallet.get_transfers()
+        assert len(res['in']) > 0
+        first_recv_height = res['in'][0]['height']
+
+        N_TO_MINE = 20 + max(0, first_recv_height + 60 - n_blocks)
+        daemon.generateblocks(main_address, N_TO_MINE)
+        wallet.refresh()
+
+        dst = {'address': '8BQKgTSSqJjP14AKnZUBwnXWj46MuNmLvHfPTpmry52DbfNjjHVvHUk4mczU8nj8yZ57zBhksTJ8kM5xKeJXw55kCMVqyG7', 'amount': 1000000000000}
+
+        res = wallet.get_transfers()
+        assert len(res['in']) >= N_TO_MINE
+        last_recv_height = max(x['height'] for x in res['in'])
+        assert last_recv_height == n_blocks + N_TO_MINE - 1, last_recv_height
+        assert wallet.get_balance().unlocked_balance > dst['amount'] * 2
+
+        res = wallet.transfer([dst])
+        res = wallet.transfer([dst])
+
+        assert len(daemon.get_transaction_pool_hashes()['tx_hashes']) == 2
+
+        daemon.generateblocks(main_address, 1)
+
+        print('Calling /get_blocks.bin (blocks only) and testing response...')
+
+        res = daemon.get_blocks_fast(0, [target_block_id, genesis_block_id])
+        assert len(res.blocks) == N_TO_MINE + 1 + 1
+        assert len(res.blocks) == len(res.output_indices)
+
+        for i in range(len(res.blocks)):
+            is_last = i == len(res.blocks) - 1
+            block = res.blocks[i]
+            assert block.pruned
+            block_indices = res.output_indices[i]['indices']
+            n_txs = len(block['txs']) if 'txs' in block else 0
+            if is_last:
+                assert n_txs == 2
+                assert len(block_indices) == 3
+            else:
+                assert n_txs == 0
+                assert len(block_indices) == 1
+            for tx_idx in range(n_txs):
+                tx = block.txs[tx_idx]
+                tx_indices = block_indices[1 + tx_idx]['indices']
+                assert tx.prunable_hash != (b'\0' * 32) # non-null
+                assert len(tx_indices) == 2
+
+        print('Calling /get_blocks.bin (blocks and incremental pool, no new blocks) and testing response...')
+
+        time.sleep(1)
+        res = daemon.get_blocks_fast(0, [], requested_info = 2)
+        assert res.pool_info_extent == 2
+        pool_info_since = res.daemon_time
+
+        wallet.refresh()
+        pending_txid = wallet.transfer([dst]).tx_hash
+        current_height = daemon.get_height().height
+
+        res = daemon.get_blocks_fast(
+            current_height,
+            [],
+            requested_info = 1,
+            pool_info_since = pool_info_since)
+        assert not res.get('blocks')
+        assert res.pool_info_extent == 1
+        assert [tx.tx_hash.hex() for tx in res.added_pool_txs] == [pending_txid]
+
+        daemon.flush_txpool([pending_txid])
+
 
 if __name__ == '__main__':
     BlockchainTest().run_test()

utils/python-rpc/framework/daemon.py

@@ -29,8 +29,18 @@
 
 """Daemon class to make rpc calls and store state."""
 
+from . import epee_binary
 from .rpc import JSONRPC 
 
+class DaemonBinary:
+    @staticmethod
+    def hash_list_to_blob(hash_list):
+        blob = bytes()
+        for h in hash_list:
+            assert len(h) == 64
+            blob += bytes.fromhex(h)
+        return blob
+
 class Daemon(object):
 
     def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0, restricted_rpc = False, username=None, password=None):
@@ -179,6 +189,24 @@ class Daemon(object):
         return self.rpc.send_json_rpc_request(getblockheadersrange)
     get_block_headers_range = getblockheadersrange
 
+    def get_blocks_fast(self, start_height, block_ids, requested_info = 0,
+            pool_info_since = 0, prune = True, no_miner_tx = False,
+            max_block_count = 0, block_ids_exclusive = False):
+        get_blocks_fast = {
+            'requested_info': requested_info,
+            'block_ids': DaemonBinary.hash_list_to_blob(block_ids),
+            'start_height': start_height,
+            'prune': prune,
+            'no_miner_tx': no_miner_tx,
+            'block_ids_exclusive': block_ids_exclusive,
+            'pool_info_since': pool_info_since,
+            'max_block_count': max_block_count
+        }
+        res = self.rpc.send_binary_request("/getblocks.bin", get_blocks_fast)
+        if 'top_block_hash' in res:
+            res['top_block_hash'] = res['top_block_hash'].hex()
+        return res
+
     def get_connections(self, client = ""):
         get_connections = {
             'client': client,

utils/python-rpc/framework/epee_binary.py

@@ -0,0 +1,319 @@
+from collections.abc import Sequence
+import io
+
+PORTABLE_STORAGE_SIGNATURE = bytes.fromhex('0111010101010201') # bender's nightmare 
+PORTABLE_STORAGE_FORMAT_VER = 1
+
+PORTABLE_RAW_SIZE_MARK_MASK   = 0x03 
+PORTABLE_RAW_SIZE_MARK_BYTE   = 0
+PORTABLE_RAW_SIZE_MARK_WORD   = 1
+PORTABLE_RAW_SIZE_MARK_DWORD  = 2
+PORTABLE_RAW_SIZE_MARK_INT64  = 3
+
+SERIALIZE_TYPE_INT64                = 1
+SERIALIZE_TYPE_INT32                = 2
+SERIALIZE_TYPE_INT16                = 3
+SERIALIZE_TYPE_INT8                 = 4
+SERIALIZE_TYPE_UINT64               = 5
+SERIALIZE_TYPE_UINT32               = 6
+SERIALIZE_TYPE_UINT16               = 7
+SERIALIZE_TYPE_UINT8                = 8
+SERIALIZE_TYPE_DOUBLE               = 9
+SERIALIZE_TYPE_STRING               = 10
+SERIALIZE_TYPE_BOOL                 = 11
+SERIALIZE_TYPE_OBJECT               = 12
+#SERIALIZE_TYPE_ARRAY                = 13
+
+SERIALIZE_FLAG_ARRAY              = 0x80
+
+class Serializer:
+    def __init__(self, outf = None):
+        self.outf = outf if outf is not None else io.BytesIO()
+
+    def serialize_stream(self, obj):
+        self.outf.write(PORTABLE_STORAGE_SIGNATURE)
+        self.outf.write(bytes([PORTABLE_STORAGE_FORMAT_VER]))
+        self.__serialize_section(obj, include_type=False)
+
+    def serialize(self, obj):
+        assert(isinstance(self.outf, io.BytesIO))
+        self.serialize_stream(obj)
+        return self.outf.getvalue()
+
+    @classmethod
+    def __is_maybe_dict_like(cls, x):
+        return hasattr(x, 'keys')
+
+    @classmethod
+    def __is_maybe_array_like(cls, x):
+        return isinstance(x, Sequence) \
+            and not isinstance(x, (str, bytes, bytearray)) \
+            and not cls.__is_maybe_dict_like(x)
+
+    @classmethod
+    def __get_int_serialize_type(cls, signed, byte_size):
+        if byte_size == 8:
+            int_type = 1
+        elif byte_size == 4:
+            int_type = 2
+        elif byte_size == 2:
+            int_type = 3
+        elif byte_size == 1:
+            int_type = 4
+        else:
+            raise ValueError("Unrecognized serialized int byte size: " + str(byte_size))
+        if not signed:
+            int_type += 4
+        return int_type
+
+    def __serialize_fixed_int(self, x, signed, byte_size, include_type):
+        int_type = self.__get_int_serialize_type(signed, byte_size)
+
+        ones_mask = 2**(byte_size*8) - 1
+        if signed:
+            max_val = ones_mask // 2
+            min_val = -(max_val+1)
+        else:
+            max_val = ones_mask
+            min_val = 0
+
+        if not (min_val <= x <= max_val):
+            raise ValueError("Cannot serialize {}integer {} in {} bytes".format('un' if not signed else '', x, byte_size))
+
+        if signed and x < 0:
+            twos_comp = ones_mask+x+1
+        else:
+            twos_comp = x
+
+        if include_type:
+            self.outf.write(bytes([int_type]))
+
+        int_bytes = twos_comp.to_bytes(byte_size, byteorder='little')
+        self.outf.write(int_bytes)
+
+    @classmethod
+    def __get_auto_int_properties(cls, x):
+        signed = x < 0
+        sizes = (1, 2, 4, 8)
+
+        for byte_size in sizes:
+            bits = byte_size * 8
+            if signed:
+                if -(1 << (bits - 1)) <= x <= (1 << (bits - 1)) - 1:
+                    return True, byte_size
+            else:
+                if 0 <= x <= (1 << bits) - 1:
+                    return False, byte_size
+
+        raise ValueError("Integer too large to serialize")
+
+    def __serialize_varint(self, x):
+        if x < 0:
+            raise ValueError("Negative values cannot be serialized as a varint")
+        for size_marker in range(4):
+            byte_size = 1 << size_marker
+            max_val = 2**(byte_size*8-2)-1
+            if x > max_val:
+                continue
+            x_serial = (x << 2) + size_marker
+            self.__serialize_fixed_int(x_serial, False, byte_size, False)
+            return
+        raise ValueError("Variant too large to be serialized")
+
+    def __serialize_bool(self, x, include_type):
+        if include_type:
+            self.outf.write(bytes([SERIALIZE_TYPE_BOOL]))
+        self.outf.write(bytes([1 if x else 0]))
+
+    def __serialize_float(self, x, include_type):
+        raise ValueError("Floating point numbers not supported yet")
+
+    def __serialize_string(self, x, include_type):
+        if isinstance(x, str):
+            x = x.encode()
+
+        if include_type:
+            self.outf.write(bytes([SERIALIZE_TYPE_STRING]))
+
+        self.__serialize_varint(len(x))
+        self.outf.write(x)
+
+    def __dispatch_serialize_scalar(self, x, include_type, signed_override = None, byte_size_override = None):
+        if isinstance(x, bool):
+            self.__serialize_bool(x, include_type)
+        elif isinstance(x, int):
+            if signed_override is None:
+                signed, byte_size = self.__get_auto_int_properties(x)
+                self.__serialize_fixed_int(x, signed, byte_size, include_type)
+            else:
+                self.__serialize_fixed_int(x, signed_override, byte_size_override, include_type)
+        elif isinstance(x, float):
+            self.__serialize_float(x, include_type)
+        elif isinstance(x, bytes) or isinstance(x, str):
+            self.__serialize_string(x, include_type)
+        elif self.__is_maybe_dict_like(x):
+            self.__serialize_section(x, include_type)
+        else:
+            raise ValueError("Cannot decide how to dispatch serialization for type {}".format(type(x)))
+
+    def __serialize_section_key(self, x):
+        if isinstance(x, str):
+            x = x.encode()
+        if len(x) > 255:
+            raise ValueError("Object/section key name cannot be longer than 255 ASCII characters")
+        self.__serialize_fixed_int(len(x), False, 1, False)
+        self.outf.write(x)
+
+    def __serialize_section_value(self, x):
+        if self.__is_maybe_array_like(x):
+            self.__serialize_array(x)
+        else:
+            self.__dispatch_serialize_scalar(x, include_type = True)
+
+    def __serialize_section(self, x, include_type):
+        if include_type:
+            self.outf.write(bytes([SERIALIZE_TYPE_OBJECT]))
+        self.__serialize_varint(len(x))
+        for key, value in x.items():
+            self.__serialize_section_key(key)
+            self.__serialize_section_value(value)
+
+    def __serialize_array(self, x):
+        # Cannot determine the serialization type of a dynamically typed empty array
+        # Thankfully, monero deserialization code allows coercing empty arrays to any type
+        if len(x) == 0:
+            EMPTY_U8_ARRAY_FLAG = SERIALIZE_FLAG_ARRAY | SERIALIZE_TYPE_UINT8
+            self.__serialize_fixed_int(EMPTY_U8_ARRAY_FLAG, signed = False, byte_size = 1, include_type = False) #type
+            self.__serialize_varint(0) # length
+            return
+
+        signed = False
+        byte_size = 1
+        base_type = None
+
+        # TODO: This doesn't assert that arrays are homogenous
+
+        if isinstance(x[0], bool):
+            base_type = SERIALIZE_TYPE_BOOL
+        elif isinstance(x[0], int):
+            # Get automatic int type for entire array
+            for elem in x:
+                s, b = self.__get_auto_int_properties(elem)
+                if s:
+                    signed = True
+                byte_size = max(byte_size, b)
+            base_type = self.__get_int_serialize_type(signed, byte_size)
+        elif isinstance(x[0], float):
+            base_type = SERIALIZE_TYPE_DOUBLE
+        elif isinstance(x[0], bytes) or isinstance(x[0], str):
+            base_type = SERIALIZE_TYPE_STRING
+        elif self.__is_maybe_dict_like(x[0]):
+            base_type = SERIALIZE_TYPE_OBJECT
+        else:
+            raise ValueError("Cannot determine array element type for Python type {}".format(type(x[0])))
+
+        array_type = base_type | SERIALIZE_FLAG_ARRAY
+        self.__serialize_fixed_int(array_type, signed = False, byte_size = 1, include_type = False) #type
+        self.__serialize_varint(len(x)) # length
+
+        for elem in x:
+            self.__dispatch_serialize_scalar(elem, include_type = False,
+                signed_override = signed, byte_size_override = byte_size)
+
+class Deserializer:
+    def __init__(self, inf):
+        if isinstance(inf, bytes):
+            self.inf = io.BytesIO(inf)
+        else:
+            self.inf = inf
+
+    def deserialize(self):
+        assert(self.inf.read(len(PORTABLE_STORAGE_SIGNATURE)) == PORTABLE_STORAGE_SIGNATURE) 
+        assert(self.inf.read(1)[0] == PORTABLE_STORAGE_FORMAT_VER)
+        return self.__deserialize_section()
+
+    def __deserialize_int(self, int_type):
+        assert(int_type != 0)
+        assert(int_type <= SERIALIZE_TYPE_UINT8)
+        signed = int_type <= SERIALIZE_TYPE_INT8
+        if int_type > SERIALIZE_TYPE_INT8:
+            int_type -= 4
+        assert(int_type <= SERIALIZE_TYPE_INT8)
+        byte_size = 1 << (SERIALIZE_TYPE_INT8-int_type)
+        int_bytes = self.inf.read(byte_size)
+        assert(len(int_bytes) == byte_size)
+        val = int.from_bytes(int_bytes, 'little')
+        if signed:
+            sign_bit = 1 << (byte_size * 8 - 1)
+            if val & sign_bit:
+                val -= 1 << (byte_size * 8)
+        return val
+
+    def __deserialize_varint(self):
+        int_bytes = self.inf.read(1)
+        byte_size = 1 << (int_bytes[0] & PORTABLE_RAW_SIZE_MARK_MASK)
+        if byte_size > 1:
+            int_bytes += self.inf.read(byte_size - 1)
+        assert(len(int_bytes) == byte_size)
+        return int.from_bytes(int_bytes, 'little') >> 2
+
+    def __deserialize_float(self):
+        raise ValueError("Floating point numbers not supported yet")
+
+    def __deserialize_string(self):
+        length = self.__deserialize_varint()
+        string = self.inf.read(length)
+        assert(len(string) == length)
+        return string
+
+    def __deserialize_bool(self):
+        b = self.inf.read(1)[0]
+        return b != 0
+
+    def __deserialize_section_key(self):
+        key_length = self.__deserialize_int(SERIALIZE_TYPE_UINT8)
+        key_bytes = self.inf.read(key_length)
+        assert(len(key_bytes) == key_length)
+        key_string = key_bytes.decode()
+        assert(len(key_string) == key_length)
+        return key_string
+
+    def __deserialize_section_value(self):
+        type_tag = self.__deserialize_int(SERIALIZE_TYPE_UINT8)
+        is_array = (type_tag & SERIALIZE_FLAG_ARRAY) != 0
+        base_type = type_tag & ~(SERIALIZE_FLAG_ARRAY)
+        assert(base_type != 0)
+        assert(base_type <= SERIALIZE_TYPE_OBJECT)
+        if is_array:
+            array_length = self.__deserialize_varint()
+            val = []
+            for _ in range(array_length):
+                val.append(self.__deserialize_of_type(base_type))
+            return val
+        else:
+            return self.__deserialize_of_type(base_type)
+
+    def __deserialize_section(self):
+        obj_length = self.__deserialize_varint()
+        obj = {}
+        for _ in range(obj_length):
+            key = self.__deserialize_section_key()
+            value = self.__deserialize_section_value()
+            obj[key] = value
+        return obj
+
+    def __deserialize_of_type(self, serialize_type):
+        assert(serialize_type > 0)
+        assert(serialize_type <= SERIALIZE_TYPE_OBJECT)
+        if serialize_type <= SERIALIZE_TYPE_UINT8:
+            return self.__deserialize_int(serialize_type)
+        elif serialize_type == SERIALIZE_TYPE_DOUBLE:
+            return self.__deserialize_float()
+        elif serialize_type == SERIALIZE_TYPE_STRING:
+            return self.__deserialize_string()
+        elif serialize_type == SERIALIZE_TYPE_BOOL:
+            return self.__deserialize_bool()
+        elif serialize_type == SERIALIZE_TYPE_OBJECT:
+            return self.__deserialize_section()
+        else:
+            raise ValueError("Unrecognized serialize type: " + str(serialize_type))

utils/python-rpc/framework/rpc.py

@@ -31,6 +31,8 @@ import requests
 from requests.auth import HTTPDigestAuth
 import json
 
+from . import epee_binary
+
 class Response(dict):
     def __init__(self, d):
         for k, v in d.items():
@@ -74,5 +76,20 @@ class JSONRPC(object):
     def send_json_rpc_request(self, inputs):
         return self.send_request("/json_rpc", inputs, 'result')
 
+    def send_binary_request(self, path, inputs):
+        binary_req = epee_binary.Serializer().serialize(inputs)
 
+        res = requests.post(
+            self.url + path,
+            data=binary_req,
+            headers={'content-type': 'application/octet-stream'},
+            auth=HTTPDigestAuth(self.username, self.password) if self.username is not None else None,
+            stream=True)
 
+        assert res.status_code == 200, res.status_code
+
+        res = epee_binary.Deserializer(res.raw).deserialize()
+
+        assert res['status'] == b'OK', res['status']
+
+        return Response(res)