Skip to content

Commit 7cdefb9

Browse files
laanwjlaanwj
committed
Merge pull request bitcoin#6410
5098c47 Implement accurate memory accounting for mempool (Pieter Wuille)
2 parents 5121c68 + 5098c47 commit 7cdefb9

File tree

8 files changed

+110
-2
lines changed

8 files changed

+110
-2
lines changed

‎src/memusage.h

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <set>
1212
#include <vector>
1313

14+
#include <boost/foreach.hpp>
1415
#include <boost/unordered_set.hpp>
1516
#include <boost/unordered_map.hpp>
1617

@@ -20,12 +21,28 @@ namespace memusage
2021
/** Compute the total memory used by allocating alloc bytes. */
2122
static size_t MallocUsage(size_t alloc);
2223

24+
/** Dynamic memory usage for built-in types is zero. */
25+
static inline size_t DynamicUsage(const int8_t& v) { return 0; }
26+
static inline size_t DynamicUsage(const uint8_t& v) { return 0; }
27+
static inline size_t DynamicUsage(const int16_t& v) { return 0; }
28+
static inline size_t DynamicUsage(const uint16_t& v) { return 0; }
29+
static inline size_t DynamicUsage(const int32_t& v) { return 0; }
30+
static inline size_t DynamicUsage(const uint32_t& v) { return 0; }
31+
static inline size_t DynamicUsage(const int64_t& v) { return 0; }
32+
static inline size_t DynamicUsage(const uint64_t& v) { return 0; }
33+
static inline size_t DynamicUsage(const float& v) { return 0; }
34+
static inline size_t DynamicUsage(const double& v) { return 0; }
35+
template<typename X> static inline size_t DynamicUsage(X * const &v) { return 0; }
36+
template<typename X> static inline size_t DynamicUsage(const X * const &v) { return 0; }
37+
template<typename X, typename Y> static inline size_t DynamicUsage(std::pair<X, Y> &p) { return 0; }
38+
2339
/** Compute the memory used for dynamically allocated but owned data structures.
2440
* For generic data types, this is *not* recursive. DynamicUsage(vector<vector<int> >)
2541
* will compute the memory used for the vector<int>'s, but not for the ints inside.
2642
* This is for efficiency reasons, as these functions are intended to be fast. If
2743
* application data structures require more accurate inner accounting, they should
28-
* do the recursion themselves, or use more efficient caching + updating on modification.
44+
* use RecursiveDynamicUsage, iterate themselves, or use more efficient caching +
45+
* updating on modification.
2946
*/
3047
template<typename X> static size_t DynamicUsage(const std::vector<X>& v);
3148
template<typename X> static size_t DynamicUsage(const std::set<X>& s);
@@ -34,6 +51,12 @@ template<typename X, typename Y> static size_t DynamicUsage(const boost::unorder
3451
template<typename X, typename Y, typename Z> static size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& s);
3552
template<typename X> static size_t DynamicUsage(const X& x);
3653

54+
template<typename X> static size_t RecursiveDynamicUsage(const std::vector<X>& v);
55+
template<typename X> static size_t RecursiveDynamicUsage(const std::set<X>& v);
56+
template<typename X, typename Y> static size_t RecursiveDynamicUsage(const std::map<X, Y>& v);
57+
template<typename X, typename Y> static size_t RecursiveDynamicUsage(const std::pair<X, Y>& v);
58+
template<typename X> static size_t RecursiveDynamicUsage(const X& v);
59+
3760
static inline size_t MallocUsage(size_t alloc)
3861
{
3962
// Measured on libc6 2.19 on Linux.
@@ -65,18 +88,54 @@ static inline size_t DynamicUsage(const std::vector<X>& v)
6588
return MallocUsage(v.capacity() * sizeof(X));
6689
}
6790

91+
template<typename X>
92+
static inline size_t RecursiveDynamicUsage(const std::vector<X>& v)
93+
{
94+
size_t usage = DynamicUsage(v);
95+
BOOST_FOREACH(const X& x, v) {
96+
usage += RecursiveDynamicUsage(x);
97+
}
98+
return usage;
99+
}
100+
68101
template<typename X>
69102
static inline size_t DynamicUsage(const std::set<X>& s)
70103
{
71104
return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
72105
}
73106

107+
template<typename X>
108+
static inline size_t RecursiveDynamicUsage(const std::set<X>& v)
109+
{
110+
size_t usage = DynamicUsage(v);
111+
BOOST_FOREACH(const X& x, v) {
112+
usage += RecursiveDynamicUsage(x);
113+
}
114+
return usage;
115+
}
116+
74117
template<typename X, typename Y>
75118
static inline size_t DynamicUsage(const std::map<X, Y>& m)
76119
{
77120
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
78121
}
79122

123+
template<typename X, typename Y>
124+
static inline size_t RecursiveDynamicUsage(const std::map<X, Y>& v)
125+
{
126+
size_t usage = DynamicUsage(v);
127+
for (typename std::map<X, Y>::const_iterator it = v.begin(); it != v.end(); it++) {
128+
usage += RecursiveDynamicUsage(*it);
129+
}
130+
return usage;
131+
}
132+
133+
template<typename X, typename Y>
134+
static inline size_t RecursiveDynamicUsage(const std::pair<X, Y>& v)
135+
{
136+
return RecursiveDynamicUsage(v.first) + RecursiveDynamicUsage(v.second);
137+
}
138+
80139
// Boost data structures
81140

82141
template<typename X>
@@ -106,6 +165,12 @@ static inline size_t DynamicUsage(const X& x)
106165
return x.DynamicMemoryUsage();
107166
}
108167

168+
template<typename X>
169+
static inline size_t RecursiveDynamicUsage(const X& x)
170+
{
171+
return DynamicUsage(x);
172+
}
173+
109174
}
110175

111176
#endif

‎src/primitives/transaction.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ void CTransaction::UpdateHash() const
7272
*const_cast<uint256*>(&hash) = SerializeHash(*this);
7373
}
7474

75+
size_t CTransaction::DynamicMemoryUsage() const
76+
{
77+
return memusage::RecursiveDynamicUsage(vin) + memusage::RecursiveDynamicUsage(vout);
78+
}
79+
7580
CTransaction::CTransaction() : nVersion(CTransaction::CURRENT_VERSION), vin(), vout(), nLockTime(0) { }
7681

7782
CTransaction::CTransaction(const CMutableTransaction &tx) : nVersion(tx.nVersion), vin(tx.vin), vout(tx.vout), nLockTime(tx.nLockTime) {

‎src/primitives/transaction.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#define BITCOIN_PRIMITIVES_TRANSACTION_H
88

99
#include "amount.h"
10+
#include "memusage.h"
1011
#include "script/script.h"
1112
#include "serialize.h"
1213
#include "uint256.h"
@@ -48,6 +49,8 @@ class COutPoint
4849
}
4950

5051
std::string ToString() const;
52+
53+
size_t DynamicMemoryUsage() const { return 0; }
5154
};
5255

5356
/** An input of a transaction. It contains the location of the previous
@@ -96,6 +99,8 @@ class CTxIn
9699
}
97100

98101
std::string ToString() const;
102+
103+
size_t DynamicMemoryUsage() const { return scriptSig.DynamicMemoryUsage(); }
99104
};
100105

101106
/** An output of a transaction. It contains the public key that the next input
@@ -166,6 +171,8 @@ class CTxOut
166171
}
167172

168173
std::string ToString() const;
174+
175+
size_t DynamicMemoryUsage() const { return scriptPubKey.DynamicMemoryUsage(); }
169176
};
170177

171178
struct CMutableTransaction;
@@ -249,6 +256,8 @@ class CTransaction
249256
}
250257

251258
std::string ToString() const;
259+
260+
size_t DynamicMemoryUsage() const;
252261
};
253262

254263
/** A mutable version of CTransaction. */

‎src/rpcblockchain.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -758,6 +758,7 @@ UniValue getmempoolinfo(const UniValue& params, bool fHelp)
758758
"{\n"
759759
" \"size\": xxxxx (numeric) Current tx count\n"
760760
" \"bytes\": xxxxx (numeric) Sum of all tx sizes\n"
761+
" \"usage\": xxxxx (numeric) Total memory usage for the mempool\n"
761762
"}\n"
762763
"\nExamples:\n"
763764
+ HelpExampleCli("getmempoolinfo", "")
@@ -767,6 +768,7 @@ UniValue getmempoolinfo(const UniValue& params, bool fHelp)
767768
UniValue ret(UniValue::VOBJ);
768769
ret.push_back(Pair("size", (int64_t) mempool.size()));
769770
ret.push_back(Pair("bytes", (int64_t) mempool.GetTotalTxSize()));
771+
ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage()));
770772

771773
return ret;
772774
}

‎src/script/script.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,3 +260,8 @@ std::string CScript::ToString() const
260260
}
261261
return str;
262262
}
263+
264+
size_t CScript::DynamicMemoryUsage() const
265+
{
266+
return memusage::DynamicUsage(*(static_cast<const std::vector<unsigned char>*>(this)));
267+
}

‎src/script/script.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#ifndef BITCOIN_SCRIPT_SCRIPT_H
77
#define BITCOIN_SCRIPT_SCRIPT_H
88

9+
#include "memusage.h"
910
#include "crypto/common.h"
1011

1112
#include <assert.h>
@@ -607,6 +608,8 @@ class CScript : public std::vector<unsigned char>
607608
// The default std::vector::clear() does not release memory.
608609
std::vector<unsigned char>().swap(*this);
609610
}
611+
612+
size_t DynamicMemoryUsage() const;
610613
};
611614

612615
class CReserveScript

‎src/txmempool.cpp

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
using namespace std;
1919

2020
CTxMemPoolEntry::CTxMemPoolEntry():
21-
nFee(0), nTxSize(0), nModSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false)
21+
nFee(0), nTxSize(0), nModSize(0), nUsageSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false)
2222
{
2323
nHeight = MEMPOOL_HEIGHT;
2424
}
@@ -31,6 +31,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee,
3131
{
3232
nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
3333
nModSize = tx.CalculateModifiedSize(nTxSize);
34+
nUsageSize = tx.DynamicMemoryUsage();
3435
}
3536

3637
CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other)
@@ -101,6 +102,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry,
101102
mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i);
102103
nTransactionsUpdated++;
103104
totalTxSize += entry.GetTxSize();
105+
cachedInnerUsage += entry.DynamicMemoryUsage();
104106
minerPolicyEstimator->processTransaction(entry, fCurrentEstimate);
105107

106108
return true;
@@ -146,6 +148,7 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem
146148

147149
removed.push_back(tx);
148150
totalTxSize -= mapTx[hash].GetTxSize();
151+
cachedInnerUsage -= mapTx[hash].DynamicMemoryUsage();
149152
mapTx.erase(hash);
150153
nTransactionsUpdated++;
151154
minerPolicyEstimator->removeTx(hash);
@@ -226,6 +229,7 @@ void CTxMemPool::clear()
226229
mapTx.clear();
227230
mapNextTx.clear();
228231
totalTxSize = 0;
232+
cachedInnerUsage = 0;
229233
++nTransactionsUpdated;
230234
}
231235

@@ -237,6 +241,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
237241
LogPrint("mempool", "Checking mempool with %u transactions and %u inputs\n", (unsigned int)mapTx.size(), (unsigned int)mapNextTx.size());
238242

239243
uint64_t checkTotal = 0;
244+
uint64_t innerUsage = 0;
240245

241246
CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins));
242247

@@ -245,6 +250,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
245250
for (std::map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) {
246251
unsigned int i = 0;
247252
checkTotal += it->second.GetTxSize();
253+
innerUsage += it->second.DynamicMemoryUsage();
248254
const CTransaction& tx = it->second.GetTx();
249255
bool fDependsWait = false;
250256
BOOST_FOREACH(const CTxIn &txin, tx.vin) {
@@ -299,6 +305,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const
299305
}
300306

301307
assert(totalTxSize == checkTotal);
308+
assert(innerUsage == cachedInnerUsage);
302309
}
303310

304311
void CTxMemPool::queryHashes(vector<uint256>& vtxid)
@@ -419,3 +426,8 @@ bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const {
419426
bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const {
420427
return mempool.exists(txid) || base->HaveCoins(txid);
421428
}
429+
430+
size_t CTxMemPool::DynamicMemoryUsage() const {
431+
LOCK(cs);
432+
return memusage::DynamicUsage(mapTx) + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage;
433+
}

‎src/txmempool.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class CTxMemPoolEntry
4040
CAmount nFee; //! Cached to avoid expensive parent-transaction lookups
4141
size_t nTxSize; //! ... and avoid recomputing tx size
4242
size_t nModSize; //! ... and modified size for priority
43+
size_t nUsageSize; //! ... and total memory usage
4344
int64_t nTime; //! Local time when entering the mempool
4445
double dPriority; //! Priority when entering the mempool
4546
unsigned int nHeight; //! Chain height when entering the mempool
@@ -58,6 +59,7 @@ class CTxMemPoolEntry
5859
int64_t GetTime() const { return nTime; }
5960
unsigned int GetHeight() const { return nHeight; }
6061
bool WasClearAtEntry() const { return hadNoDependencies; }
62+
size_t DynamicMemoryUsage() const { return nUsageSize; }
6163
};
6264

6365
class CBlockPolicyEstimator;
@@ -73,6 +75,7 @@ class CInPoint
7375
CInPoint(const CTransaction* ptxIn, uint32_t nIn) { ptx = ptxIn; n = nIn; }
7476
void SetNull() { ptx = NULL; n = (uint32_t) -1; }
7577
bool IsNull() const { return (ptx == NULL && n == (uint32_t) -1); }
78+
size_t DynamicMemoryUsage() const { return 0; }
7679
};
7780

7881
/**
@@ -93,6 +96,7 @@ class CTxMemPool
9396
CBlockPolicyEstimator* minerPolicyEstimator;
9497

9598
uint64_t totalTxSize; //! sum of all mempool tx' byte sizes
99+
uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves)
96100

97101
public:
98102
mutable CCriticalSection cs;
@@ -139,6 +143,7 @@ class CTxMemPool
139143
LOCK(cs);
140144
return mapTx.size();
141145
}
146+
142147
uint64_t GetTotalTxSize()
143148
{
144149
LOCK(cs);
@@ -162,6 +167,8 @@ class CTxMemPool
162167
/** Write/Read estimates to disk */
163168
bool WriteFeeEstimates(CAutoFile& fileout) const;
164169
bool ReadFeeEstimates(CAutoFile& filein);
170+
171+
size_t DynamicMemoryUsage() const;
165172
};
166173

167174
/**

0 commit comments

Comments
 (0)