Skip to content

Commit 59afcc8

Browse files
glozowmurchandamus
glozow
and
murchandamus
committed
Implement Mini version of BlockAssembler to calculate mining scores
Rewrite the same algo instead of reusing BlockAssembler because we have a few extra requirements that would make the changes invasive and difficult to review: - Only operate on the relevant transactions rather than full mempool - Remove transactions that will be replaced so they can't bump their ancestors - Don't hold mempool lock outside of the constructor - Skip things like max block weight and IsFinalTx - Additionally calculate fees to bump remaining ancestor packages to target feerate Co-authored-by: Murch <murch@murch.one>
1 parent 56484f0 commit 59afcc8

File tree

3 files changed

+489
-0
lines changed

3 files changed

+489
-0
lines changed

‎src/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ BITCOIN_CORE_H = \
211211
node/mempool_args.h \
212212
node/mempool_persist_args.h \
213213
node/miner.h \
214+
node/mini_miner.h \
214215
node/minisketchwrapper.h \
215216
node/psbt.h \
216217
node/transaction.h \
@@ -396,6 +397,7 @@ libbitcoin_node_a_SOURCES = \
396397
node/mempool_args.cpp \
397398
node/mempool_persist_args.cpp \
398399
node/miner.cpp \
400+
node/mini_miner.cpp \
399401
node/minisketchwrapper.cpp \
400402
node/psbt.cpp \
401403
node/transaction.cpp \

‎src/node/mini_miner.cpp

Lines changed: 366 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,366 @@
1+
// Copyright (c) 2023 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#include <node/mini_miner.h>
6+
7+
#include <consensus/amount.h>
8+
#include <policy/feerate.h>
9+
#include <primitives/transaction.h>
10+
#include <timedata.h>
11+
#include <util/check.h>
12+
#include <util/moneystr.h>
13+
14+
#include <algorithm>
15+
#include <numeric>
16+
#include <utility>
17+
18+
namespace node {
19+
20+
MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints)
21+
{
22+
LOCK(mempool.cs);
23+
// Find which outpoints to calculate bump fees for.
24+
// Anything that's spent by the mempool is to-be-replaced
25+
// Anything otherwise unavailable just has a bump fee of 0
26+
for (const auto& outpoint : outpoints) {
27+
if (!mempool.exists(GenTxid::Txid(outpoint.hash))) {
28+
// This UTXO is either confirmed or not yet submitted to mempool.
29+
// If it's confirmed, no bump fee is required.
30+
// If it's not yet submitted, we have no information, so return 0.
31+
m_bump_fees.emplace(outpoint, 0);
32+
continue;
33+
}
34+
35+
// UXTO is created by transaction in mempool, add to map.
36+
// Note: This will either create a missing entry or add the outpoint to an existing entry
37+
m_requested_outpoints_by_txid[outpoint.hash].push_back(outpoint);
38+
39+
if (const auto ptx{mempool.GetConflictTx(outpoint)}) {
40+
// This outpoint is already being spent by another transaction in the mempool. We
41+
// assume that the caller wants to replace this transaction and its descendants. It
42+
// would be unusual for the transaction to have descendants as the wallet won’t normally
43+
// attempt to replace transactions with descendants. If the outpoint is from a mempool
44+
// transaction, we still need to calculate its ancestors bump fees (added to
45+
// m_requested_outpoints_by_txid below), but after removing the to-be-replaced entries.
46+
//
47+
// Note that the descendants of a transaction include the transaction itself. Also note,
48+
// that this is only calculating bump fees. RBF fee rules should be handled separately.
49+
CTxMemPool::setEntries descendants;
50+
mempool.CalculateDescendants(mempool.GetIter(ptx->GetHash()).value(), descendants);
51+
for (const auto& desc_txiter : descendants) {
52+
m_to_be_replaced.insert(desc_txiter->GetTx().GetHash());
53+
}
54+
}
55+
}
56+
57+
// No unconfirmed UTXOs, so nothing mempool-related needs to be calculated.
58+
if (m_requested_outpoints_by_txid.empty()) return;
59+
60+
// Calculate the cluster and construct the entry map.
61+
std::vector<uint256> txids_needed;
62+
txids_needed.reserve(m_requested_outpoints_by_txid.size());
63+
for (const auto& [txid, _]: m_requested_outpoints_by_txid) {
64+
txids_needed.push_back(txid);
65+
}
66+
const auto cluster = mempool.GatherClusters(txids_needed);
67+
if (cluster.empty()) {
68+
// An empty cluster means that at least one of the transactions is missing from the mempool
69+
// (should not be possible given processing above) or DoS limit was hit.
70+
m_ready_to_calculate = false;
71+
return;
72+
}
73+
74+
// Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced.
75+
for (const auto& txiter : cluster) {
76+
if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) {
77+
auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter));
78+
m_entries.push_back(mapiter);
79+
} else {
80+
auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash());
81+
if (outpoints_it != m_requested_outpoints_by_txid.end()) {
82+
// This UTXO is the output of a to-be-replaced transaction. Bump fee is 0; spending
83+
// this UTXO is impossible as it will no longer exist after the replacement.
84+
for (const auto& outpoint : outpoints_it->second) {
85+
m_bump_fees.emplace(outpoint, 0);
86+
}
87+
m_requested_outpoints_by_txid.erase(outpoints_it);
88+
}
89+
}
90+
}
91+
92+
// Build the m_descendant_set_by_txid cache.
93+
for (const auto& txiter : cluster) {
94+
const auto& txid = txiter->GetTx().GetHash();
95+
// Cache descendants for future use. Unlike the real mempool, a descendant MiniMinerMempoolEntry
96+
// will not exist without its ancestor MiniMinerMempoolEntry, so these sets won't be invalidated.
97+
std::vector<MockEntryMap::iterator> cached_descendants;
98+
const bool remove{m_to_be_replaced.count(txid) > 0};
99+
CTxMemPool::setEntries descendants;
100+
mempool.CalculateDescendants(txiter, descendants);
101+
Assume(descendants.count(txiter) > 0);
102+
for (const auto& desc_txiter : descendants) {
103+
const auto txid_desc = desc_txiter->GetTx().GetHash();
104+
const bool remove_desc{m_to_be_replaced.count(txid_desc) > 0};
105+
auto desc_it{m_entries_by_txid.find(txid_desc)};
106+
Assume((desc_it == m_entries_by_txid.end()) == remove_desc);
107+
if (remove) Assume(remove_desc);
108+
// It's possible that remove=false but remove_desc=true.
109+
if (!remove && !remove_desc) {
110+
cached_descendants.push_back(desc_it);
111+
}
112+
}
113+
if (remove) {
114+
Assume(cached_descendants.empty());
115+
} else {
116+
m_descendant_set_by_txid.emplace(txid, cached_descendants);
117+
}
118+
}
119+
120+
// Release the mempool lock; we now have all the information we need for a subset of the entries
121+
// we care about. We will solely operate on the MiniMinerMempoolEntry map from now on.
122+
Assume(m_in_block.empty());
123+
Assume(m_requested_outpoints_by_txid.size() <= outpoints.size());
124+
SanityCheck();
125+
}
126+
127+
// Compare by min(ancestor feerate, individual feerate), then iterator
128+
//
129+
// Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate
130+
// parents do not incentive inclusion of their children. Therefore the mining algorithm only considers
131+
// transactions for inclusion on basis of the minimum of their own feerate or their ancestor feerate.
132+
struct AncestorFeerateComparator
133+
{
134+
template<typename I>
135+
bool operator()(const I& a, const I& b) const {
136+
auto min_feerate = [](const MiniMinerMempoolEntry& e) -> CFeeRate {
137+
const CAmount ancestor_fee{e.GetModFeesWithAncestors()};
138+
const int64_t ancestor_size{e.GetSizeWithAncestors()};
139+
const CAmount tx_fee{e.GetModifiedFee()};
140+
const int64_t tx_size{e.GetTxSize()};
141+
// Comparing ancestor feerate with individual feerate:
142+
// ancestor_fee / ancestor_size <= tx_fee / tx_size
143+
// Avoid division and possible loss of precision by
144+
// multiplying both sides by the sizes:
145+
return ancestor_fee * tx_size < tx_fee * ancestor_size ?
146+
CFeeRate(ancestor_fee, ancestor_size) :
147+
CFeeRate(tx_fee, tx_size);
148+
};
149+
CFeeRate a_feerate{min_feerate(a->second)};
150+
CFeeRate b_feerate{min_feerate(b->second)};
151+
if (a_feerate != b_feerate) {
152+
return a_feerate > b_feerate;
153+
}
154+
// Use txid as tiebreaker for stable sorting
155+
return a->first < b->first;
156+
}
157+
};
158+
159+
void MiniMiner::DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors)
160+
{
161+
Assume(ancestors.size() >= 1);
162+
// "Mine" all transactions in this ancestor set.
163+
for (auto& anc : ancestors) {
164+
Assume(m_in_block.count(anc->first) == 0);
165+
m_in_block.insert(anc->first);
166+
m_total_fees += anc->second.GetModifiedFee();
167+
m_total_vsize += anc->second.GetTxSize();
168+
auto it = m_descendant_set_by_txid.find(anc->first);
169+
// Each entry’s descendant set includes itself
170+
Assume(it != m_descendant_set_by_txid.end());
171+
for (auto& descendant : it->second) {
172+
// If these fail, we must be double-deducting.
173+
Assume(descendant->second.GetModFeesWithAncestors() >= anc->second.GetModifiedFee());
174+
Assume(descendant->second.vsize_with_ancestors >= anc->second.GetTxSize());
175+
descendant->second.fee_with_ancestors -= anc->second.GetModifiedFee();
176+
descendant->second.vsize_with_ancestors -= anc->second.GetTxSize();
177+
}
178+
}
179+
// Delete these entries.
180+
for (const auto& anc : ancestors) {
181+
m_descendant_set_by_txid.erase(anc->first);
182+
// The above loop should have deducted each ancestor's size and fees from each of their
183+
// respective descendants exactly once.
184+
Assume(anc->second.GetModFeesWithAncestors() == 0);
185+
Assume(anc->second.GetSizeWithAncestors() == 0);
186+
auto vec_it = std::find(m_entries.begin(), m_entries.end(), anc);
187+
Assume(vec_it != m_entries.end());
188+
m_entries.erase(vec_it);
189+
m_entries_by_txid.erase(anc);
190+
}
191+
}
192+
193+
void MiniMiner::SanityCheck() const
194+
{
195+
// m_entries, m_entries_by_txid, and m_descendant_set_by_txid all same size
196+
Assume(m_entries.size() == m_entries_by_txid.size());
197+
Assume(m_entries.size() == m_descendant_set_by_txid.size());
198+
// Cached ancestor values should be at least as large as the transaction's own fee and size
199+
Assume(std::all_of(m_entries.begin(), m_entries.end(), [](const auto& entry) {
200+
return entry->second.GetSizeWithAncestors() >= entry->second.GetTxSize() &&
201+
entry->second.GetModFeesWithAncestors() >= entry->second.GetModifiedFee();}));
202+
// None of the entries should be to-be-replaced transactions
203+
Assume(std::all_of(m_to_be_replaced.begin(), m_to_be_replaced.end(),
204+
[&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();}));
205+
}
206+
207+
void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate)
208+
{
209+
while (!m_entries_by_txid.empty()) {
210+
// Sort again, since transaction removal may change some m_entries' ancestor feerates.
211+
std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator());
212+
213+
// Pick highest ancestor feerate entry.
214+
auto best_iter = m_entries.begin();
215+
Assume(best_iter != m_entries.end());
216+
const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors();
217+
const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors();
218+
// Stop here. Everything that didn't "make it into the block" has bumpfee.
219+
if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) {
220+
break;
221+
}
222+
223+
// Calculate ancestors on the fly. This lookup should be fairly cheap, and ancestor sets
224+
// change at every iteration, so this is more efficient than maintaining a cache.
225+
std::set<MockEntryMap::iterator, IteratorComparator> ancestors;
226+
{
227+
std::set<MockEntryMap::iterator, IteratorComparator> to_process;
228+
to_process.insert(*best_iter);
229+
while (!to_process.empty()) {
230+
auto iter = to_process.begin();
231+
Assume(iter != to_process.end());
232+
ancestors.insert(*iter);
233+
for (const auto& input : (*iter)->second.GetTx().vin) {
234+
if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) {
235+
if (ancestors.count(parent_it) == 0) {
236+
to_process.insert(parent_it);
237+
}
238+
}
239+
}
240+
to_process.erase(iter);
241+
}
242+
}
243+
DeleteAncestorPackage(ancestors);
244+
SanityCheck();
245+
}
246+
Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize));
247+
// Do not try to continue building the block template with a different feerate.
248+
m_ready_to_calculate = false;
249+
}
250+
251+
std::map<COutPoint, CAmount> MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate)
252+
{
253+
if (!m_ready_to_calculate) return {};
254+
// Build a block template until the target feerate is hit.
255+
BuildMockTemplate(target_feerate);
256+
257+
// Each transaction that "made it into the block" has a bumpfee of 0, i.e. they are part of an
258+
// ancestor package with at least the target feerate and don't need to be bumped.
259+
for (const auto& txid : m_in_block) {
260+
// Not all of the block transactions were necessarily requested.
261+
auto it = m_requested_outpoints_by_txid.find(txid);
262+
if (it != m_requested_outpoints_by_txid.end()) {
263+
for (const auto& outpoint : it->second) {
264+
m_bump_fees.emplace(outpoint, 0);
265+
}
266+
m_requested_outpoints_by_txid.erase(it);
267+
}
268+
}
269+
270+
// A transactions and its ancestors will only be picked into a block when
271+
// both the ancestor set feerate and the individual feerate meet the target
272+
// feerate.
273+
//
274+
// We had to convince ourselves that after running the mini miner and
275+
// picking all eligible transactions into our MockBlockTemplate, there
276+
// could still be transactions remaining that have a lower individual
277+
// feerate than their ancestor feerate. So here is an example:
278+
//
279+
// ┌─────────────────┐
280+
// │ │
281+
// │ Grandparent │
282+
// │ 1700 vB │
283+
// │ 1700 sats │ Target feerate: 10 s/vB
284+
// │ 1 s/vB │ GP Ancestor Set Feerate (ASFR): 1 s/vB
285+
// │ │ P1_ASFR: 9.84 s/vB
286+
// └──────▲───▲──────┘ P2_ASFR: 2.47 s/vB
287+
// │ │ C_ASFR: 10.27 s/vB
288+
// ┌───────────────┐ │ │ ┌──────────────┐
289+
// │ ├────┘ └────┤ │ ⇒ C_FR < TFR < C_ASFR
290+
// │ Parent 1 │ │ Parent 2 │
291+
// │ 200 vB │ │ 200 vB │
292+
// │ 17000 sats │ │ 3000 sats │
293+
// │ 85 s/vB │ │ 15 s/vB │
294+
// │ │ │ │
295+
// └───────────▲───┘ └───▲──────────┘
296+
// │ │
297+
// │ ┌───────────┐ │
298+
// └────┤ ├────┘
299+
// │ Child │
300+
// │ 100 vB │
301+
// │ 900 sats │
302+
// │ 9 s/vB │
303+
// │ │
304+
// └───────────┘
305+
//
306+
// We therefore calculate both the bump fee that is necessary to elevate
307+
// the individual transaction to the target feerate:
308+
// target_feerate × tx_size - tx_fees
309+
// and the bump fee that is necessary to bump the entire ancestor set to
310+
// the target feerate:
311+
// target_feerate × ancestor_set_size - ancestor_set_fees
312+
// By picking the maximum from the two, we ensure that a transaction meets
313+
// both criteria.
314+
for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) {
315+
auto it = m_entries_by_txid.find(txid);
316+
Assume(it != m_entries_by_txid.end());
317+
if (it != m_entries_by_txid.end()) {
318+
Assume(target_feerate.GetFee(it->second.GetSizeWithAncestors()) > std::min(it->second.GetModifiedFee(), it->second.GetModFeesWithAncestors()));
319+
CAmount bump_fee_with_ancestors = target_feerate.GetFee(it->second.GetSizeWithAncestors()) - it->second.GetModFeesWithAncestors();
320+
CAmount bump_fee_individual = target_feerate.GetFee(it->second.GetTxSize()) - it->second.GetModifiedFee();
321+
const CAmount bump_fee{std::max(bump_fee_with_ancestors, bump_fee_individual)};
322+
Assume(bump_fee >= 0);
323+
for (const auto& outpoint : outpoints) {
324+
m_bump_fees.emplace(outpoint, bump_fee);
325+
}
326+
}
327+
}
328+
return m_bump_fees;
329+
}
330+
331+
std::optional<CAmount> MiniMiner::CalculateTotalBumpFees(const CFeeRate& target_feerate)
332+
{
333+
if (!m_ready_to_calculate) return std::nullopt;
334+
// Build a block template until the target feerate is hit.
335+
BuildMockTemplate(target_feerate);
336+
337+
// All remaining ancestors that are not part of m_in_block must be bumped, but no other relatives
338+
std::set<MockEntryMap::iterator, IteratorComparator> ancestors;
339+
std::set<MockEntryMap::iterator, IteratorComparator> to_process;
340+
for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) {
341+
// Skip any ancestors that already have a miner score higher than the target feerate
342+
// (already "made it" into the block)
343+
if (m_in_block.count(txid)) continue;
344+
auto iter = m_entries_by_txid.find(txid);
345+
if (iter == m_entries_by_txid.end()) continue;
346+
to_process.insert(iter);
347+
ancestors.insert(iter);
348+
}
349+
while (!to_process.empty()) {
350+
auto iter = to_process.begin();
351+
const CTransaction& tx = (*iter)->second.GetTx();
352+
for (const auto& input : tx.vin) {
353+
if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) {
354+
to_process.insert(parent_it);
355+
ancestors.insert(parent_it);
356+
}
357+
}
358+
to_process.erase(iter);
359+
}
360+
const auto ancestor_package_size = std::accumulate(ancestors.cbegin(), ancestors.cend(), int64_t{0},
361+
[](int64_t sum, const auto it) {return sum + it->second.GetTxSize();});
362+
const auto ancestor_package_fee = std::accumulate(ancestors.cbegin(), ancestors.cend(), CAmount{0},
363+
[](CAmount sum, const auto it) {return sum + it->second.GetModifiedFee();});
364+
return target_feerate.GetFee(ancestor_package_size) - ancestor_package_fee;
365+
}
366+
} // namespace node

0 commit comments

Comments
 (0)