cryptonote_core: cache input verification results directly in mempool

This replaces `ver_rct_non_semantics_simple_cached()` with an API that offloads
the responsibility of tracking input verification successes to the caller. The
main caller of this function in the codebase, `cryptonote::Blockchain()` instead
keeps track of the verification results for transaction in the mempool by
storing a "verification ID" in the mempool metadata table (with `txpool_tx_meta_t`).
This has several benefits, including:

* When the mempool is large (>8192 txs), we no longer experience cache misses and unnecessarily re-verify ring signatures. This greatly improves block propagation time for FCMP++ blocks under load
* For the same reason, reorg handling can be sped up by storing verification IDs of transactions popped from the chain
* Speeds up re-validating every mempool transaction on fork change (monerod revalidates the whole tx-pool on HFs #10142)
* Caches results for every single type of Monero transaction, not just latest RCT type
* Cache persists over a node restart
* Uses 512KiB less RAM (8192*2*32B)
* No additional storage or DB migration required since `txpool_tx_meta_t` already had padding allocated
* Moves more verification logic out of `cryptonote::Blockchain`

Furthermore, this opens the door to future multi-threaded block verification
speed-ups. Right now, transactions' input proof verification is limited to one
transaction at a time. However, one can imagine a scenario with verification IDs
where input proofs are optimistically multi-threaded in advance of block
processing. Then, even though ring member fetching and verification is
single-threaded inside of `cryptonote::Blockchain::check_tx_inputs()`, the
single thread can skip the CPU-intensive cryptographic code if the verification
ID allows it.

Also changes the default log category in `tx_verification_utils.cpp` from "blockchain" to "verify".
This commit is contained in:
jeffro256
2025-10-09 15:15:59 -05:00
parent 6bb36309d6
commit 40eb82873e
12 changed files with 822 additions and 306 deletions

View File

@@ -37,6 +37,17 @@ import time
from framework.daemon import Daemon
from framework.wallet import Wallet
def average(a):
return sum(a) / len(a)
def median(a):
a = sorted(a)
i0 = len(a)//2
if len(a) % 2 == 0:
return average(a[i0-1:i0+1])
else:
return a[i0]
class P2PTest():
def run_test(self):
self.reset()
@@ -47,6 +58,7 @@ class P2PTest():
self.test_p2p_block_propagation_shared(txid)
txid = self.test_p2p_tx_propagation()
self.test_p2p_block_propagation_new(txid)
self.bench_p2p_heavy_block_propagation_speed()
def reset(self):
print('Resetting blockchain')
@@ -307,6 +319,140 @@ class P2PTest():
assert ('in_pool' not in tx_details) or (not tx_details.in_pool)
assert tx_details.block_height == block_height
def bench_p2p_heavy_block_propagation_speed(self):
ENABLED = False
if not ENABLED:
print('SKIPPING benchmark of P2P heavy block propagation')
return
print('Benchmarking P2P heavy block propagation')
daemon2 = Daemon(idx = 2)
daemon3 = Daemon(idx = 3)
start_height = daemon2.get_height().height
current_height = start_height
daemon2_address = '{}:{}'.format(daemon2.host, daemon2.port)
print(' Setup: creating new wallet')
wallet = Wallet()
try: wallet.close_wallet()
except: pass
wallet.create_wallet()
wallet.auto_refresh(enable = False)
wallet.set_daemon(daemon2_address)
assert wallet.get_transfers() == {}
main_address = wallet.get_address().address
CURRENT_RING_SIZE = 16
min_height = CURRENT_RING_SIZE + 1 + 60
if start_height < min_height:
print(' Setup: mining to mixable RingCT height: {}'.format(min_height))
n_to_mine = min_height - start_height
daemon2.generateblocks(main_address, n_to_mine)
current_height += n_to_mine
print(' Setup: spamming self-send transactions into mempool to increase size')
update_unlocked_inputs = lambda: \
[x for x in wallet.incoming_transfers().get('transfers', []) if not x.spent
and x.unlocked
and x.amount > 2 * last_fee]
MAX_TX_OUTPUTS = 16
MEMPOOL_TX_TARGET = 2 * 8192 # 2x previous ver_rct_non_semantics_simple_cached() cache size
n_mempool_txs = 0
unlocked_inputs = update_unlocked_inputs()
last_fee = 10000000000 # 0.01 XMR to start off with is an over-estimation for a 1/16 in the penalty-free zone
print_progress = lambda action: \
print(' Progress: {}/{} ({:.1f}%) txs in mempool, {} usable inputs, {} blocks mined, just {} {}'.format(
n_mempool_txs, MEMPOOL_TX_TARGET, n_mempool_txs/MEMPOOL_TX_TARGET*100, len(unlocked_inputs),
current_height - start_height, action, ' ' * 10
), end='\r')
print_progress('started')
while n_mempool_txs < MEMPOOL_TX_TARGET:
try:
if len(unlocked_inputs) == 0:
daemon2.generateblocks(main_address, 1)
wallet.refresh()
current_height += 1
wallet_height = wallet.get_height().height
assert wallet_height == current_height, wallet_height
n_mempool_txs = len(daemon2.get_transaction_pool_hashes().get('tx_hashes', []))
res = wallet.incoming_transfers()
unlocked_inputs = update_unlocked_inputs()
last_action = 'mined'
else:
inp = unlocked_inputs.pop()
res = wallet.sweep_single(main_address, outputs = MAX_TX_OUTPUTS - 1, key_image = inp.key_image)
assert res.spent_key_images.key_images == [inp.key_image]
last_fee = res.fee
n_mempool_txs += 1
last_action = 'swept'
except AssertionError as ae:
print() # Clear carriage return
if 'Transaction sanity check failed' not in str(ae):
raise
# The RingCT output distribution gets so skewed in this test that the wallet
# thinks something is wrong with decoy selection. To recover, try mining a block on
# the next action.
print(' WARNING: caught transaction sanity check, stepping forward chain to try to fix')
unlocked_inputs = []
last_action = 'caught sanity'
print_progress(last_action)
print()
print(' Setup: wait for daemons to reach equilibrium on mempool contents')
assert n_mempool_txs > 0
sync_start = time.time()
sync_deadline = sync_start + 120
while True:
mempool_hashes_2 = daemon2.get_transaction_pool_hashes().get('tx_hashes', [])
assert len(mempool_hashes_2) == n_mempool_txs # Txs were submitted to daemon 2
mempool_hashes_3 = daemon3.get_transaction_pool_hashes().get('tx_hashes', [])
print(' {}/{} mempool txs propagated'.format(len(mempool_hashes_2), len(mempool_hashes_3)), end='\r')
if sorted(mempool_hashes_2) == sorted(mempool_hashes_3):
break
elif time.time() > sync_deadline:
raise RuntimeError('daemons did not sync mempools within deadline')
time.sleep(0.25)
print()
print(' Bench: mine and propagate blocks until mempool is empty of profitable txs')
timings = []
while True:
time1 = time.time()
daemon2.generateblocks(main_address, 1)
current_height += 1
time2 = time.time()
while daemon3.get_height().height != current_height:
time.sleep(0.01)
time3 = time.time()
new_n_mempool_txs = len(daemon2.get_transaction_pool_hashes().get('tx_hashes', []))
assert new_n_mempool_txs <= n_mempool_txs
n_mined_txs = n_mempool_txs - new_n_mempool_txs
elapsed_mining = time2 - time1
elapsed_prop = time3 - time2
timings.append((n_mined_txs, elapsed_mining, elapsed_prop))
n_mempool_txs = new_n_mempool_txs
print(' * Mined {} txs in {:.2f}s, propagated in {:.2f}s'.format(*(timings[-1])))
if n_mempool_txs == 0 or n_mined_txs == 0:
break
print(' Analysis of {}-tx mempool handling:'.format(MEMPOOL_TX_TARGET))
avg_mining = average([x[1] for x in timings])
median_mining = median([x[1] for x in timings])
avg_prop = average([x[2] for x in timings])
median_prop = median([x[2] for x in timings])
print(' Average mining time: {:.2f}'.format(avg_mining))
print(' Median mining time: {:.2f}'.format(median_mining))
print(' Average block propagation time: {:.2f}'.format(avg_prop))
print(' Median block propagation time: {:.2f}'.format(median_prop))
if __name__ == '__main__':
P2PTest().run_test()

View File

@@ -92,7 +92,7 @@ set(unit_tests_sources
uri.cpp
util.cpp
varint.cpp
ver_rct_non_semantics_simple_cached.cpp
verRctNonSemanticsSimple.cpp
ringct.cpp
output_selection.cpp
vercmp.cpp

View File

@@ -0,0 +1,269 @@
// Copyright (c) 2025, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "gtest/gtest.h"
#include "cryptonote_core/cryptonote_tx_utils.h"
#include "cryptonote_core/tx_verification_utils.h"
TEST(tx_verification_utils, make_input_verification_id)
{
rct::key key1, key2, key3;
epee::from_hex::to_buffer(epee::as_mut_byte_span(key1), "e50f476129d40af31e0938743f7f2d60e867aab31294f7acaf6e38f0976f0228");
epee::from_hex::to_buffer(epee::as_mut_byte_span(key2), "e50f476129d40af31e0938743f7f2d60e867aab31294f7acaf6e38f0976f0227");
epee::from_hex::to_buffer(epee::as_mut_byte_span(key3), "d50f476129d40af31e0938743f7f2d60e867aab31294f7acaf6e38f0976f0228");
const crypto::hash hash1 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {});
const crypto::hash hash2 = cryptonote::make_input_verification_id(rct::rct2hash(key2), {});
const crypto::hash hash3 = cryptonote::make_input_verification_id(rct::rct2hash(key3), {});
ASSERT_NE(hash1, hash2);
ASSERT_NE(hash1, hash3);
ASSERT_NE(hash2, hash3);
const crypto::hash hash4 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1}}});
const crypto::hash hash5 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key2}}});
const crypto::hash hash6 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key3}}});
ASSERT_NE(hash4, hash5);
ASSERT_NE(hash4, hash6);
ASSERT_NE(hash5, hash6);
const crypto::hash hash7 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1},{key1, key1}}});
const crypto::hash hash8 = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1}},{{key1, key1}}});
ASSERT_NE(hash7, hash8);
const crypto::hash hash1_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {});
const crypto::hash hash2_eq = cryptonote::make_input_verification_id(rct::rct2hash(key2), {});
const crypto::hash hash3_eq = cryptonote::make_input_verification_id(rct::rct2hash(key3), {});
const crypto::hash hash4_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1}}});
const crypto::hash hash5_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key2}}});
const crypto::hash hash6_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key3}}});
const crypto::hash hash7_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1},{key1, key1}}});
const crypto::hash hash8_eq = cryptonote::make_input_verification_id(rct::rct2hash(key1), {{{key1, key1}},{{key1, key1}}});
ASSERT_EQ(hash1, hash1_eq);
ASSERT_EQ(hash2, hash2_eq);
ASSERT_EQ(hash3, hash3_eq);
ASSERT_EQ(hash4, hash4_eq);
ASSERT_EQ(hash5, hash5_eq);
ASSERT_EQ(hash6, hash6_eq);
ASSERT_EQ(hash7, hash7_eq);
ASSERT_EQ(hash8, hash8_eq);
}
TEST(tx_verification_utils, ver_input_proofs_rings)
{
// constants
static constexpr size_t N_INPUTS = 2;
static constexpr size_t N_OUTPUTS = 10;
static constexpr size_t N_RING_MEMBERS = 16;
static constexpr bool USE_VIEW_TAGS = true;
static constexpr rct::RCTConfig RCT_CONFIG{ rct::RangeProofPaddedBulletproof, 4 }; // CLSAG, BP+
static constexpr uint8_t HF_VERSION = HF_VERSION_VIEW_TAGS + 1; // CLSAG, BP+, after grace period
// generate accounts
hw::device &hwdev = hw::get_device("default");
cryptonote::account_base alice;
alice.generate();
const cryptonote::account_public_address &alice_main_addr = alice.get_keys().m_account_address;
const std::unordered_map<crypto::public_key, cryptonote::subaddress_index> alice_subaddresses{
{alice_main_addr.m_spend_public_key, {}}
};
cryptonote::account_base bob;
bob.generate();
const cryptonote::account_public_address &bob_main_addr = bob.get_keys().m_account_address;
cryptonote::account_base aether;
aether.generate();
// populate inputs
rct::xmr_amount total_input_amounts = 0;
std::vector<cryptonote::tx_source_entry> sources;
sources.reserve(N_INPUTS);
for (size_t i = 0; i < N_INPUTS; ++i)
{
const rct::xmr_amount in_amount = crypto::rand_range<rct::xmr_amount>(0, COIN) + COIN; // [1, 2] XMR
const size_t real_in_ring_idx = crypto::rand_idx(N_RING_MEMBERS);
// generate one-time address from derivation
crypto::secret_key in_main_tx_privkey;
crypto::public_key in_main_tx_pubkey;
crypto::generate_keys(in_main_tx_pubkey, in_main_tx_privkey); // (r, R)
crypto::secret_key_to_public_key(in_main_tx_privkey, in_main_tx_pubkey);
crypto::key_derivation ecdh;
ASSERT_TRUE(hwdev.generate_key_derivation(in_main_tx_pubkey, alice.get_keys().m_view_secret_key, ecdh));
const size_t real_output_in_tx_index = crypto::rand_idx(N_OUTPUTS);
crypto::public_key in_onetime_address;
crypto::view_tag in_view_tag;
std::vector<crypto::public_key> in_additional_tx_public_keys;
std::vector<rct::key> in_amount_keys;
ASSERT_TRUE(hwdev.generate_output_ephemeral_keys(/*tx_version=*/2,
aether.get_keys(), in_main_tx_pubkey, in_main_tx_privkey,
{0, alice_main_addr, false}, /*change_addr=*/boost::none, real_output_in_tx_index,
/*need_additional_txkeys=*/false, /*additional_tx_keys=*/{},
in_additional_tx_public_keys,
in_amount_keys, in_onetime_address,
USE_VIEW_TAGS, in_view_tag));
ASSERT_EQ(1, in_amount_keys.size());
const rct::key in_amount_blinding_factor = rct::genCommitmentMask(in_amount_keys.at(0));
const rct::key in_amount_commitment = rct::commit(in_amount, in_amount_blinding_factor);
// randomly populate decoys and insert real spend
auto &tx_source = sources.emplace_back();
tx_source.outputs.reserve(N_RING_MEMBERS);
for (size_t j = 0; j < N_RING_MEMBERS; ++j)
{
const size_t ring_member_global_output_idx = 20 * j;
if (j == real_in_ring_idx)
{
tx_source.outputs.emplace_back(ring_member_global_output_idx,
rct::ctkey{rct::pk2rct(in_onetime_address), in_amount_commitment});
}
else // decoy
{
tx_source.outputs.emplace_back(ring_member_global_output_idx,
rct::ctkey{rct::pkGen(), rct::pkGen()});
}
}
tx_source.real_output = real_in_ring_idx;
tx_source.real_out_tx_key = in_main_tx_pubkey;
tx_source.real_out_additional_tx_keys = in_additional_tx_public_keys;
tx_source.real_output_in_tx_index = real_output_in_tx_index;
tx_source.amount = in_amount;
tx_source.rct = true;
tx_source.mask = in_amount_blinding_factor;
tx_source.multisig_kLRki = {};
total_input_amounts += in_amount;
}
// populate destinations
const rct::xmr_amount approx_fee = 500000000000; // 0.5 XMR
const rct::xmr_amount dest_amount = (total_input_amounts - approx_fee) / N_OUTPUTS;
std::vector<cryptonote::tx_destination_entry> destinations;
destinations.reserve(N_OUTPUTS);
for (size_t i = 0; i < N_OUTPUTS - 1; ++i)
destinations.push_back(cryptonote::tx_destination_entry(dest_amount, bob_main_addr, false));
destinations.push_back(cryptonote::tx_destination_entry(dest_amount, alice_main_addr, false));
// construct transaction
cryptonote::transaction tx;
crypto::secret_key main_tx_key;
std::vector<crypto::secret_key> additional_tx_keys;
ASSERT_TRUE(cryptonote::construct_tx_and_get_tx_key(alice.get_keys(),
alice_subaddresses,
sources,
destinations,
alice_main_addr,
/*extra=*/{},
tx,
main_tx_key,
additional_tx_keys,
/*rct=*/true,
RCT_CONFIG,
USE_VIEW_TAGS));
ASSERT_EQ(N_INPUTS, tx.vin.size());
ASSERT_EQ(N_OUTPUTS, tx.vout.size());
ASSERT_EQ(N_RING_MEMBERS, boost::get<cryptonote::txin_to_key>(tx.vin.at(0)).key_offsets.size());
ASSERT_GE(tx.rct_signatures.txnFee, approx_fee);
ASSERT_LE(tx.rct_signatures.txnFee, approx_fee + N_OUTPUTS);
// collect mix rings
rct::ctkeyM mixrings(N_INPUTS);
for (size_t i = 0; i < N_INPUTS; ++i)
{
mixrings.at(i).resize(N_RING_MEMBERS);
for (size_t j = 0; j < N_RING_MEMBERS; ++j)
{
mixrings.at(i).at(j) = sources.at(i).outputs.at(j).second;
}
}
// serialize transaction to blob
const cryptonote::blobdata tx_blob = cryptonote::tx_to_blob(tx);
// de-serialize transaction from blob
cryptonote::transaction deserialized_tx;
ASSERT_TRUE(cryptonote::parse_and_validate_tx_from_blob(tx_blob, deserialized_tx));
// test non-input consensus rules
cryptonote::tx_verification_context tvc{};
ASSERT_TRUE(cryptonote::ver_non_input_consensus(deserialized_tx, tvc, HF_VERSION));
ASSERT_FALSE(tvc.m_verifivation_failed);
// test verify input rings [positive]
EXPECT_TRUE(cryptonote::ver_input_proofs_rings(deserialized_tx, mixrings));
// test verify input rings again (already expanded) [positive]
EXPECT_TRUE(cryptonote::ver_input_proofs_rings(deserialized_tx, mixrings));
// test verify input rings after modify to expansion [positive]
deserialized_tx.rct_signatures.mixRing.at(0).at(0) = {rct::pkGen(), rct::pkGen()};
EXPECT_TRUE(cryptonote::ver_input_proofs_rings(deserialized_tx, mixrings));
// test verify input rings after modify to dereferenced mixring [negative]
rct::ctkeyM modified_mixrings = mixrings;
modified_mixrings.at(0).at(0) = {rct::pkGen(), rct::pkGen()};
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
modified_mixrings = mixrings;
modified_mixrings.at(0).at(1) = {rct::pkGen(), rct::pkGen()};
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
// test verify input rings after add dereferenced mixring [negative]
modified_mixrings = mixrings;
modified_mixrings.emplace_back();
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
// test verify input rings after remove dereferenced mixring [negative]
modified_mixrings = mixrings;
modified_mixrings.pop_back();
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
// test verify input rings after add dereferenced decoy [negative]
modified_mixrings = mixrings;
modified_mixrings.at(0).push_back({rct::pkGen(), rct::pkGen()});
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
// test verify input rings after remove dereferenced decoy [negative]
{
modified_mixrings = mixrings;
rct::ctkeyV &mixring0 = modified_mixrings.at(0);
mixring0.erase(mixring0.begin());
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
}
{
modified_mixrings = mixrings;
rct::ctkeyV &mixring0 = modified_mixrings.at(0);
mixring0.erase(mixring0.begin() + 1);
EXPECT_FALSE(cryptonote::ver_input_proofs_rings(deserialized_tx, modified_mixrings));
}
}

View File

@@ -243,8 +243,6 @@ TEST(verRctNonSemanticsSimple, tx1_preconditions)
// If this unit test fails, something changed about transaction deserialization / expansion or
// something changed about RingCT signature verification.
cryptonote::rct_ver_cache_t rct_ver_cache;
cryptonote::transaction tx = expand_transaction_from_bin_file_and_pubkeys
(tx1_file_name, tx1_input_pubkeys);
const rct::rctSig& rs = tx.rct_signatures;
@@ -274,8 +272,8 @@ TEST(verRctNonSemanticsSimple, tx1_preconditions)
EXPECT_TRUE(rct::verRctSemanticsSimple(rs));
EXPECT_TRUE(rct::verRctNonSemanticsSimple(rs));
EXPECT_TRUE(rct::verRctSimple(rs));
EXPECT_TRUE(cryptonote::ver_rct_non_semantics_simple_cached(tx, tx1_input_pubkeys, rct_ver_cache, rct::RCTTypeBulletproofPlus));
EXPECT_TRUE(cryptonote::ver_rct_non_semantics_simple_cached(tx, tx1_input_pubkeys, rct_ver_cache, rct::RCTTypeBulletproofPlus));
EXPECT_TRUE(cryptonote::ver_input_proofs_rings(tx, tx1_input_pubkeys));
EXPECT_TRUE(cryptonote::ver_input_proofs_rings(tx, tx1_input_pubkeys));
}
#define SERIALIZABLE_SIG_CHANGES_SUBTEST(fieldmodifyclause) \