From c9cf0b78f8e5aac84b1190441ccae0a2b6b6aec3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 5 Nov 2021 17:51:54 +0000 Subject: [PATCH 1/5] wallet2: speedup large tx construction: cache public key validity 5.9 second -> 5.2 seconds on a test case --- src/wallet/wallet2.cpp | 54 ++++++++++++++++++++++++------------------ src/wallet/wallet2.h | 10 ++++---- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f4a5a5855..21834643a 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -7818,7 +7818,7 @@ bool wallet2::is_keys_file_locked() const return m_keys_file_locker->locked(); } -bool wallet2::tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const +bool wallet2::tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked, std::unordered_set &valid_public_keys_cache) const { if (!unlocked) // don't add locked outs return false; @@ -7829,16 +7829,18 @@ bool wallet2::tx_add_fake_output(std::vector valid_public_keys_cache; for(size_t idx: selected_transfers) { // Create new index @@ -7933,7 +7936,7 @@ void wallet2::light_wallet_get_outs(std::vector, size_t> outs_unique(const std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, bool rct) +void wallet2::get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, bool rct, std::unordered_set &valid_public_keys_cache) { std::vector rct_offsets; for (size_t attempts = 3; attempts > 0; --attempts) { - get_outs(outs, selected_transfers, fake_outputs_count, rct_offsets); + get_outs(outs, selected_transfers, fake_outputs_count, rct_offsets, valid_public_keys_cache); if (!rct) return; @@ -7997,7 +8000,7 @@ void wallet2::get_outs(std::vector> THROW_WALLET_EXCEPTION(error::wallet_internal_error, tr("Transaction sanity check failed")); } -void wallet2::get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, std::vector &rct_offsets) +void wallet2::get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, std::vector &rct_offsets, std::unordered_set &valid_public_keys_cache) { LOG_PRINT_L2("fake_outputs_count: " << fake_outputs_count); outs.clear(); @@ -8524,7 +8527,7 @@ void wallet2::get_outs(std::vector> if (req.outputs[i].index == out) { LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key << " (from existing ring)"); - tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked); + tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked, valid_public_keys_cache); found = true; break; } @@ -8549,7 +8552,7 @@ void wallet2::get_outs(std::vector> { size_t i = base + order[o]; LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key); - tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked); + tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked, valid_public_keys_cache); } if (outs.back().size() < fake_outputs_count + 1) { @@ -8593,7 +8596,7 @@ void wallet2::get_outs(std::vector> template void wallet2::transfer_selected(const std::vector& dsts, const std::vector& selected_transfers, size_t fake_outputs_count, - std::vector> &outs, + std::vector> &outs, std::unordered_set &valid_public_keys_cache, uint64_t unlock_time, uint64_t fee, const std::vector& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool use_view_tags) { @@ -8631,7 +8634,7 @@ void wallet2::transfer_selected(const std::vector dsts, const std::vector& selected_transfers, size_t fake_outputs_count, - std::vector> &outs, + std::vector> &outs, std::unordered_set &valid_public_keys_cache, uint64_t unlock_time, uint64_t fee, const std::vector& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config, bool use_view_tags) { using namespace cryptonote; @@ -8849,7 +8852,7 @@ void wallet2::transfer_selected_rct(std::vector wallet2::create_transactions_2(std::vector valid_public_keys_cache; const uint64_t base_fee = get_base_fee(priority); const uint64_t fee_quantization_mask = get_fee_quantization_mask(); @@ -10112,10 +10116,10 @@ std::vector wallet2::create_transactions_2(std::vector wallet2::create_transactions_2(std::vector test_ptx.fee) { if (use_rct) - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); else - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_quantization_mask); @@ -10206,6 +10210,7 @@ skip_tx: tx.selected_transfers, /* const std::list selected_transfers */ fake_outs_count, /* CONST size_t fake_outputs_count, */ tx.outs, /* MOD std::vector> &outs, */ + valid_public_keys_cache, unlock_time, /* CONST uint64_t unlock_time, */ tx.needed_fee, /* CONST uint64_t fee, */ extra, /* const std::vector& extra, */ @@ -10218,6 +10223,7 @@ skip_tx: tx.selected_transfers, fake_outs_count, tx.outs, + valid_public_keys_cache, unlock_time, tx.needed_fee, extra, @@ -10335,6 +10341,7 @@ std::vector wallet2::create_transactions_all(uint64_t below THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; const uint64_t fractional_threshold = (base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); + std::unordered_set valid_public_keys_cache; THROW_WALLET_EXCEPTION_IF(unlocked_balance(subaddr_account, false) == 0, error::wallet_internal_error, "No unlocked balance in the specified account"); @@ -10416,6 +10423,7 @@ std::vector wallet2::create_transactions_from(const crypton hw::device &hwdev = m_account.get_device(); boost::unique_lock hwdev_lock (hwdev); hw::reset_mode rst(hwdev); + std::unordered_set valid_public_keys_cache; uint64_t accumulated_fee, accumulated_outputs, accumulated_change; struct TX { @@ -10518,10 +10526,10 @@ std::vector wallet2::create_transactions_from(const crypton LOG_PRINT_L2("Trying to create a tx now, with " << tx.dsts.size() << " destinations and " << tx.selected_transfers.size() << " outputs"); if (use_rct) - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); else - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_quantization_mask); @@ -10555,10 +10563,10 @@ std::vector wallet2::create_transactions_from(const crypton dt.amount = dt_amount + dt_residue; } if (use_rct) - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); else - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_quantization_mask); @@ -10594,10 +10602,10 @@ std::vector wallet2::create_transactions_from(const crypton cryptonote::transaction test_tx; pending_tx test_ptx; if (use_rct) { - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, valid_public_keys_cache, unlock_time, tx.needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); } else { - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, valid_public_keys_cache, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); } auto txBlob = t_serializable_object_to_blob(test_ptx.tx); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 660e6a14b..41ef5b3b8 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1001,10 +1001,10 @@ private: uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL, uint64_t *time_to_unlock = NULL); template void transfer_selected(const std::vector& dsts, const std::vector& selected_transfers, size_t fake_outputs_count, - std::vector> &outs, + std::vector> &outs, std::unordered_set &valid_public_keys_cache, uint64_t unlock_time, uint64_t fee, const std::vector& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, const bool use_view_tags); void transfer_selected_rct(std::vector dsts, const std::vector& selected_transfers, size_t fake_outputs_count, - std::vector> &outs, + std::vector> &outs, std::unordered_set &valid_public_keys_cache, uint64_t unlock_time, uint64_t fee, const std::vector& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config, const bool use_view_tags); void commit_tx(pending_tx& ptx_vector); @@ -1665,9 +1665,9 @@ private: void set_unspent(size_t idx); bool is_spent(const transfer_details &td, bool strict = true) const; bool is_spent(size_t idx, bool strict = true) const; - void get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, bool rct); - void get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, std::vector &rct_offsets); - bool tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; + void get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, bool rct, std::unordered_set &valid_public_keys_cache); + void get_outs(std::vector> &outs, const std::vector &selected_transfers, size_t fake_outputs_count, std::vector &rct_offsets, std::unordered_set &valid_public_keys_cache); + bool tx_add_fake_output(std::vector> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked, std::unordered_set &valid_public_keys_cache) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector &unused_transfers_indices, const std::vector &unused_dust_indices) const; std::vector get_only_rct(const std::vector &unused_dust_indices, const std::vector &unused_transfers_indices) const; void scan_output(const cryptonote::transaction &tx, bool miner_tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map &tx_money_got_in_outs, std::vector &outs, bool pool); From 2b9517889774c9420e6cb336ec62c5842420cee4 Mon Sep 17 00:00:00 2001 From: Crypto City Date: Fri, 5 Nov 2021 17:47:37 +0000 Subject: [PATCH 2/5] wallet2: speedup large tx construction: batch ringdb updates 5.2 seconds -> 4.1 seconds on a test case --- src/wallet/ringdb.cpp | 17 ++++++++++++++--- src/wallet/ringdb.h | 1 + src/wallet/wallet2.cpp | 16 ++++++++++++++-- src/wallet/wallet2.h | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index bfbbbaeb7..d5d06f16b 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -387,20 +387,24 @@ bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative) +bool ringdb::set_rings(const crypto::chacha_key &chacha_key, const std::vector>> &rings, bool relative) { MDB_txn *txn; int dbr; bool tx_active = false; - dbr = resize_env(env, filename.c_str(), outs.size() * 64); + size_t n_outs = 0; + for (const auto &e: rings) + n_outs += e.second.size(); + dbr = resize_env(env, filename.c_str(), n_outs * 64); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - store_relative_ring(txn, dbi_rings, key_image, relative ? outs : cryptonote::absolute_output_offsets_to_relative(outs), chacha_key); + for (const auto &e: rings) + store_relative_ring(txn, dbi_rings, e.first, relative ? e.second : cryptonote::absolute_output_offsets_to_relative(e.second), chacha_key); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn setting ring to database: " + std::string(mdb_strerror(dbr))); @@ -408,6 +412,13 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } +bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative) +{ + std::vector>> rings; + rings.push_back(std::make_pair(key_image, outs)); + return set_rings(chacha_key, rings, relative); +} + bool ringdb::blackball_worker(const std::vector> &outputs, int op) { MDB_txn *txn; diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index e9941bf94..7b02f4605 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -50,6 +50,7 @@ namespace tools bool remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative); + bool set_rings(const crypto::chacha_key &chacha_key, const std::vector>> &rings, bool relative); bool blackball(const std::pair &output); bool blackball(const std::vector> &outputs); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 21834643a..e77e3bafd 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -7644,6 +7644,15 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector>> &rings, bool relative) +{ + if (!m_ringdb) + return false; + + try { return m_ringdb->set_rings(get_ringdb_key(), rings, relative); } + catch (const std::exception &e) { return false; } +} + bool wallet2::unset_ring(const std::vector &key_images) { if (!m_ringdb) @@ -8580,6 +8589,8 @@ void wallet2::get_outs(std::vector> } // save those outs in the ringdb for reuse + std::vector>> rings; + rings.reserve(selected_transfers.size()); for (size_t i = 0; i < selected_transfers.size(); ++i) { const size_t idx = selected_transfers[i]; @@ -8589,9 +8600,10 @@ void wallet2::get_outs(std::vector> ring.reserve(outs[i].size()); for (const auto &e: outs[i]) ring.push_back(std::get<0>(e)); - if (!set_ring(td.m_key_image, ring, false)) - MERROR("Failed to set ring for " << td.m_key_image); + rings.push_back(std::make_pair(td.m_key_image, std::move(ring))); } + if (!set_rings(rings, false)) + MERROR("Failed to set rings"); } template diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 41ef5b3b8..082b1d4e9 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1559,6 +1559,7 @@ private: bool get_ring(const crypto::key_image &key_image, std::vector &outs); bool get_rings(const crypto::hash &txid, std::vector>> &outs); bool set_ring(const crypto::key_image &key_image, const std::vector &outs, bool relative); + bool set_rings(const std::vector>> &rings, bool relative); bool unset_ring(const std::vector &key_images); bool unset_ring(const crypto::hash &txid); bool find_and_save_rings(bool force = true); From 353cd5355e4935ccff8e9e6c83385f81d3ce67e8 Mon Sep 17 00:00:00 2001 From: Crypto City Date: Fri, 5 Nov 2021 18:10:22 +0000 Subject: [PATCH 3/5] wallet2: speedup large tx construction: no pointless clsag generation 4.1 seconds -> 3.3 seconds on a test case --- src/ringct/rctSigs.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 3e85f60ce..bd67778ec 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -79,6 +79,7 @@ namespace return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I}; } + rct::BulletproofPlus make_dummy_bulletproof_plus(const std::vector &outamounts, rct::keyV &C, rct::keyV &masks) { const size_t n_outs = outamounts.size(); @@ -109,6 +110,13 @@ namespace return rct::BulletproofPlus{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I)}; } + + rct::clsag make_dummy_clsag(size_t ring_size) + { + const rct::key I = rct::identity(); + const size_t n_scalars = ring_size; + return rct::clsag{rct::keyV(n_scalars, I), I, I, I}; + } } namespace rct { @@ -1323,7 +1331,10 @@ namespace rct { { if (is_rct_clsag(rv.type)) { - rv.p.CLSAGs[i] = proveRctCLSAGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], kLRki ? &(*kLRki)[i]: NULL, msout ? &msout->c[i] : NULL, msout ? &msout->mu_p[i] : NULL, index[i], hwdev); + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + rv.p.CLSAGs[i] = make_dummy_clsag(rv.mixRing[i].size()); + else + rv.p.CLSAGs[i] = proveRctCLSAGSimple(full_message, rv.mixRing[i], inSk[i], a[i], pseudoOuts[i], kLRki ? &(*kLRki)[i]: NULL, msout ? &msout->c[i] : NULL, msout ? &msout->mu_p[i] : NULL, index[i], hwdev); } else { From 38cdf01c643df677a0c65b8932d62c3f6eb4e62b Mon Sep 17 00:00:00 2001 From: Crypto City Date: Fri, 5 Nov 2021 19:18:10 +0000 Subject: [PATCH 4/5] wallet2: speedup large tx construction: batch ringdb lookups 3.3 seconds -> 2.8 seconds on a test case --- src/wallet/ringdb.cpp | 22 +++++++++++++++++++++- src/wallet/ringdb.h | 1 + src/wallet/wallet2.cpp | 39 +++++++++++++++++++++++++++++++++++---- src/wallet/wallet2.h | 1 + 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index d5d06f16b..7e4f12f5b 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -344,12 +344,15 @@ bool ringdb::remove_rings(const crypto::chacha_key &chacha_key, const cryptonote return remove_rings(chacha_key, key_images); } -bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector &outs) +bool ringdb::get_rings(const crypto::chacha_key &chacha_key, const std::vector &key_images, std::vector> &all_outs) { MDB_txn *txn; int dbr; bool tx_active = false; + all_outs.clear(); + all_outs.reserve(key_images.size()); + dbr = resize_env(env, filename.c_str(), 0); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); @@ -357,6 +360,10 @@ bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_im epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; + for (size_t i = 0; i < key_images.size(); ++i) + { + const crypto::key_image &key_image = key_images[i]; + MDB_val key, data; std::string key_ciphertext = encrypt(key_image, chacha_key, 0); key.mv_data = (void*)key_ciphertext.data(); @@ -367,6 +374,7 @@ bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return false; THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size"); + std::vector outs; bool try_v0 = false; std::string data_plaintext = decrypt(std::string((const char*)data.mv_data, data.mv_size), key_image, chacha_key, 1); try { outs = decompress_ring(data_plaintext, V1TAG); if (outs.empty()) try_v0 = true; } @@ -380,6 +388,9 @@ bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_im MDEBUG("Relative: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); outs = cryptonote::relative_output_offsets_to_absolute(outs); MDEBUG("Absolute: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + all_outs.push_back(std::move(outs)); + + } dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn getting ring from database: " + std::string(mdb_strerror(dbr))); @@ -387,6 +398,15 @@ bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } +bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector &outs) +{ + std::vector> all_outs; + if (!get_rings(chacha_key, std::vector(1, key_image), all_outs)) + return false; + outs = std::move(all_outs.front()); + return true; +} + bool ringdb::set_rings(const crypto::chacha_key &chacha_key, const std::vector>> &rings, bool relative) { MDB_txn *txn; diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 7b02f4605..bdecdba37 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,6 +49,7 @@ namespace tools bool remove_rings(const crypto::chacha_key &chacha_key, const std::vector &key_images); bool remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector &outs); + bool get_rings(const crypto::chacha_key &chacha_key, const std::vector &key_images, std::vector> &all_outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative); bool set_rings(const crypto::chacha_key &chacha_key, const std::vector>> &rings, bool relative); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index e77e3bafd..3df67aac1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -7606,6 +7606,14 @@ bool wallet2::get_ring(const crypto::chacha_key &key, const crypto::key_image &k catch (const std::exception &e) { return false; } } +bool wallet2::get_rings(const crypto::chacha_key &key, const std::vector &key_images, std::vector> &outs) +{ + if (!m_ringdb) + return false; + try { return m_ringdb->get_rings(key, key_images, outs); } + catch (const std::exception &e) { return false; } +} + bool wallet2::get_rings(const crypto::hash &txid, std::vector>> &outs) { for (auto i: m_confirmed_txs) @@ -8126,6 +8134,25 @@ void wallet2::get_outs(std::vector> } } + std::vector ring_key_images; + ring_key_images.reserve(selected_transfers.size()); + std::unordered_map> existing_rings; + for(size_t idx: selected_transfers) + { + const transfer_details &td = m_transfers[idx]; + if (td.m_key_image_known && !td.m_key_image_partial) + ring_key_images.push_back(td.m_key_image); + } + if (!ring_key_images.empty()) + { + std::vector> all_outs; + if (get_rings(get_ringdb_key(), ring_key_images, all_outs)) + { + for (size_t i = 0; i < ring_key_images.size(); ++i) + existing_rings[ring_key_images[i]] = std::move(all_outs[i]); + } + } + // we ask for more, to have spares if some outputs are still locked size_t base_requested_outputs_count = (size_t)((fake_outputs_count + 1) * 1.5 + 1); LOG_PRINT_L2("base_requested_outputs_count: " << base_requested_outputs_count); @@ -8248,9 +8275,12 @@ void wallet2::get_outs(std::vector> // if we have a known ring, use it if (td.m_key_image_known && !td.m_key_image_partial) { - std::vector ring; - if (get_ring(get_ringdb_key(), td.m_key_image, ring)) + + const auto it = existing_rings.find(td.m_key_image); + const bool has_ring = it != existing_rings.end(); + if (has_ring) { + const std::vector &ring = it->second; MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, "An output in this transaction was previously spent on another chain with ring size " + @@ -8520,9 +8550,10 @@ void wallet2::get_outs(std::vector> // then pick outs from an existing ring, if any if (td.m_key_image_known && !td.m_key_image_partial) { - std::vector ring; - if (get_ring(get_ringdb_key(), td.m_key_image, ring)) + const auto it = existing_rings.find(td.m_key_image); + if (it != existing_rings.end()) { + const std::vector &ring = it->second; for (uint64_t out: ring) { if (out < num_outs) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 082b1d4e9..e9e5bc95e 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1558,6 +1558,7 @@ private: const std::string get_ring_database() const { return m_ring_database; } bool get_ring(const crypto::key_image &key_image, std::vector &outs); bool get_rings(const crypto::hash &txid, std::vector>> &outs); + bool get_rings(const crypto::chacha_key &key, const std::vector &key_images, std::vector> &outs); bool set_ring(const crypto::key_image &key_image, const std::vector &outs, bool relative); bool set_rings(const std::vector>> &rings, bool relative); bool unset_ring(const std::vector &key_images); From df2e68082513ceb07cbfc037a011565776532695 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 5 Nov 2021 19:57:49 +0000 Subject: [PATCH 5/5] wallet2: speedup large tx construction: reserve vector memory 2.8 seconds -> 2.6 seconds on a test case --- src/wallet/wallet2.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 3df67aac1..6def414de 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -8061,6 +8061,7 @@ void wallet2::get_outs(std::vector> cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); // request histogram for all outputs, except 0 if we have the rct distribution + req_t.amounts.reserve(selected_transfers.size()); for(size_t idx: selected_transfers) if (!m_transfers[idx].is_rct() || !has_rct_distribution) req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); @@ -8088,6 +8089,7 @@ void wallet2::get_outs(std::vector> { cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response resp_t = AUTO_VAL_INIT(resp_t); + req_t.amounts.reserve(req_t.amounts.size() + selected_transfers.size()); for(size_t idx: selected_transfers) req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); std::sort(req_t.amounts.begin(), req_t.amounts.end()); @@ -8166,6 +8168,8 @@ void wallet2::get_outs(std::vector> gamma.reset(new gamma_picker(rct_offsets)); size_t num_selected_transfers = 0; + req.outputs.reserve(selected_transfers.size() * (base_requested_outputs_count + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW)); + daemon_resp.outs.reserve(selected_transfers.size() * (base_requested_outputs_count + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW)); for(size_t idx: selected_transfers) { ++num_selected_transfers; @@ -8480,7 +8484,9 @@ void wallet2::get_outs(std::vector> COMMAND_RPC_GET_OUTPUTS_BIN::request chunk_req = AUTO_VAL_INIT(chunk_req); COMMAND_RPC_GET_OUTPUTS_BIN::response chunk_daemon_resp = AUTO_VAL_INIT(chunk_daemon_resp); chunk_req.get_txid = false; - for (size_t i = 0; i < std::min(req.outputs.size() - offset, chunk_size); ++i) + const size_t this_chunk_size = std::min(req.outputs.size() - offset, chunk_size); + chunk_req.outputs.reserve(this_chunk_size); + for (size_t i = 0; i < this_chunk_size; ++i) chunk_req.outputs.push_back(req.outputs[offset + i]); const boost::lock_guard lock{m_daemon_rpc_mutex};