Skip to content

Commit 2d46f1b

Browse files
committed
Merge #15118: Refactor block file logic
04cca33 Style cleanup. (Jim Posen) 4c01e4e flatfile: Unit tests for FlatFileSeq methods. (Jim Posen) 65a489e scripted-diff: Rename CBlockDiskPos to FlatFilePos. (Jim Posen) d6d8a78 Move CDiskBlockPos from chain to flatfile. (Jim Posen) e038093 validation: Refactor file flush logic into FlatFileSeq. (Jim Posen) 992404b validation: Refactor block file pre-allocation into FlatFileSeq. (Jim Posen) e2d2abb validation: Refactor OpenDiskFile into method on FlatFileSeq. (Jim Posen) 9183d6e validation: Extract basic block file logic into FlatFileSeq class. (Jim Posen) 62e7add util: Move CheckDiskSpace to util. (Jim Posen) Pull request description: This cleans up and refactors block file helpers so that they may be used by the block filter indexer. Per [design discussion](bitcoin/bitcoin#14121 (comment)) about storing BIP 157 block filters, it has been suggested that they are stored in the same way as block and undo data. This refactor is sufficient to simplify file operations for this use case, though in the future perhaps more pruning-related logic ought to be moved into the new classes. The basic abstraction is a `FlatFileSeq` which manages access to a sequence of numbered files into which raw data is written. Tree-SHA512: b2108756777f2dad8964a1a2ef2764486e708a4a4a8cfac47b5de8bcb0625388964438eb096b10cfd9ea39212c299b5cb32fa943e768db2333cf49ea7def157e
2 parents 789b0bb + 04cca33 commit 2d46f1b

File tree

12 files changed

+417
-181
lines changed

12 files changed

+417
-181
lines changed

src/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ BITCOIN_CORE_H = \
128128
core_io.h \
129129
core_memusage.h \
130130
cuckoocache.h \
131+
flatfile.h \
131132
fs.h \
132133
httprpc.h \
133134
httpserver.h \
@@ -247,6 +248,7 @@ libbitcoin_server_a_SOURCES = \
247248
chain.cpp \
248249
checkpoints.cpp \
249250
consensus/tx_verify.cpp \
251+
flatfile.cpp \
250252
httprpc.cpp \
251253
httpserver.cpp \
252254
index/base.cpp \

src/Makefile.test.include

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ BITCOIN_TESTS =\
8383
test/cuckoocache_tests.cpp \
8484
test/denialofservice_tests.cpp \
8585
test/descriptor_tests.cpp \
86+
test/flatfile_tests.cpp \
8687
test/fs_tests.cpp \
8788
test/getarg_tests.cpp \
8889
test/hash_tests.cpp \

src/chain.h

Lines changed: 5 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
#include <arith_uint256.h>
1010
#include <consensus/params.h>
11+
#include <flatfile.h>
1112
#include <primitives/block.h>
1213
#include <tinyformat.h>
1314
#include <uint256.h>
@@ -90,46 +91,6 @@ class CBlockFileInfo
9091
}
9192
};
9293

93-
struct CDiskBlockPos
94-
{
95-
int nFile;
96-
unsigned int nPos;
97-
98-
ADD_SERIALIZE_METHODS;
99-
100-
template <typename Stream, typename Operation>
101-
inline void SerializationOp(Stream& s, Operation ser_action) {
102-
READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
103-
READWRITE(VARINT(nPos));
104-
}
105-
106-
CDiskBlockPos() {
107-
SetNull();
108-
}
109-
110-
CDiskBlockPos(int nFileIn, unsigned int nPosIn) {
111-
nFile = nFileIn;
112-
nPos = nPosIn;
113-
}
114-
115-
friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) {
116-
return (a.nFile == b.nFile && a.nPos == b.nPos);
117-
}
118-
119-
friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) {
120-
return !(a == b);
121-
}
122-
123-
void SetNull() { nFile = -1; nPos = 0; }
124-
bool IsNull() const { return (nFile == -1); }
125-
126-
std::string ToString() const
127-
{
128-
return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos);
129-
}
130-
131-
};
132-
13394
enum BlockStatus: uint32_t {
13495
//! Unused.
13596
BLOCK_VALID_UNKNOWN = 0,
@@ -266,17 +227,17 @@ class CBlockIndex
266227
nNonce = block.nNonce;
267228
}
268229

269-
CDiskBlockPos GetBlockPos() const {
270-
CDiskBlockPos ret;
230+
FlatFilePos GetBlockPos() const {
231+
FlatFilePos ret;
271232
if (nStatus & BLOCK_HAVE_DATA) {
272233
ret.nFile = nFile;
273234
ret.nPos = nDataPos;
274235
}
275236
return ret;
276237
}
277238

278-
CDiskBlockPos GetUndoPos() const {
279-
CDiskBlockPos ret;
239+
FlatFilePos GetUndoPos() const {
240+
FlatFilePos ret;
280241
if (nStatus & BLOCK_HAVE_UNDO) {
281242
ret.nFile = nFile;
282243
ret.nPos = nUndoPos;

src/flatfile.cpp

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
// Copyright (c) 2009-2010 Satoshi Nakamoto
2+
// Copyright (c) 2009-2019 The Bitcoin Core developers
3+
// Distributed under the MIT software license, see the accompanying
4+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
5+
6+
#include <stdexcept>
7+
8+
#include <flatfile.h>
9+
#include <logging.h>
10+
#include <tinyformat.h>
11+
#include <util/system.h>
12+
13+
FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
14+
m_dir(std::move(dir)),
15+
m_prefix(prefix),
16+
m_chunk_size(chunk_size)
17+
{
18+
if (chunk_size == 0) {
19+
throw std::invalid_argument("chunk_size must be positive");
20+
}
21+
}
22+
23+
std::string FlatFilePos::ToString() const
24+
{
25+
return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos);
26+
}
27+
28+
fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
29+
{
30+
return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
31+
}
32+
33+
FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)
34+
{
35+
if (pos.IsNull()) {
36+
return nullptr;
37+
}
38+
fs::path path = FileName(pos);
39+
fs::create_directories(path.parent_path());
40+
FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+");
41+
if (!file && !read_only)
42+
file = fsbridge::fopen(path, "wb+");
43+
if (!file) {
44+
LogPrintf("Unable to open file %s\n", path.string());
45+
return nullptr;
46+
}
47+
if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) {
48+
LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
49+
fclose(file);
50+
return nullptr;
51+
}
52+
return file;
53+
}
54+
55+
size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space)
56+
{
57+
out_of_space = false;
58+
59+
unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size;
60+
unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size;
61+
if (n_new_chunks > n_old_chunks) {
62+
size_t old_size = pos.nPos;
63+
size_t new_size = n_new_chunks * m_chunk_size;
64+
size_t inc_size = new_size - old_size;
65+
66+
if (CheckDiskSpace(m_dir, inc_size)) {
67+
FILE *file = Open(pos);
68+
if (file) {
69+
LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile);
70+
AllocateFileRange(file, pos.nPos, inc_size);
71+
fclose(file);
72+
return inc_size;
73+
}
74+
} else {
75+
out_of_space = true;
76+
}
77+
}
78+
return 0;
79+
}
80+
81+
bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize)
82+
{
83+
FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos
84+
if (!file) {
85+
return error("%s: failed to open file %d", __func__, pos.nFile);
86+
}
87+
if (finalize && !TruncateFile(file, pos.nPos)) {
88+
fclose(file);
89+
return error("%s: failed to truncate file %d", __func__, pos.nFile);
90+
}
91+
if (!FileCommit(file)) {
92+
fclose(file);
93+
return error("%s: failed to commit file %d", __func__, pos.nFile);
94+
}
95+
96+
fclose(file);
97+
return true;
98+
}

src/flatfile.h

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright (c) 2009-2010 Satoshi Nakamoto
2+
// Copyright (c) 2009-2019 The Bitcoin Core developers
3+
// Distributed under the MIT software license, see the accompanying
4+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
5+
6+
#ifndef BITCOIN_FLATFILE_H
7+
#define BITCOIN_FLATFILE_H
8+
9+
#include <string>
10+
11+
#include <fs.h>
12+
#include <serialize.h>
13+
14+
struct FlatFilePos
15+
{
16+
int nFile;
17+
unsigned int nPos;
18+
19+
ADD_SERIALIZE_METHODS;
20+
21+
template <typename Stream, typename Operation>
22+
inline void SerializationOp(Stream& s, Operation ser_action) {
23+
READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
24+
READWRITE(VARINT(nPos));
25+
}
26+
27+
FlatFilePos() : nFile(-1), nPos(0) {}
28+
29+
FlatFilePos(int nFileIn, unsigned int nPosIn) :
30+
nFile(nFileIn),
31+
nPos(nPosIn)
32+
{}
33+
34+
friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) {
35+
return (a.nFile == b.nFile && a.nPos == b.nPos);
36+
}
37+
38+
friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) {
39+
return !(a == b);
40+
}
41+
42+
void SetNull() { nFile = -1; nPos = 0; }
43+
bool IsNull() const { return (nFile == -1); }
44+
45+
std::string ToString() const;
46+
};
47+
48+
/**
49+
* FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates
50+
* access to and efficient management of these files.
51+
*/
52+
class FlatFileSeq
53+
{
54+
private:
55+
const fs::path m_dir;
56+
const char* const m_prefix;
57+
const size_t m_chunk_size;
58+
59+
public:
60+
/**
61+
* Constructor
62+
*
63+
* @param dir The base directory that all files live in.
64+
* @param prefix A short prefix given to all file names.
65+
* @param chunk_size Disk space is pre-allocated in multiples of this amount.
66+
*/
67+
FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size);
68+
69+
/** Get the name of the file at the given position. */
70+
fs::path FileName(const FlatFilePos& pos) const;
71+
72+
/** Open a handle to the file at the given position. */
73+
FILE* Open(const FlatFilePos& pos, bool read_only = false);
74+
75+
/**
76+
* Allocate additional space in a file after the given starting position. The amount allocated
77+
* will be the minimum multiple of the sequence chunk size greater than add_size.
78+
*
79+
* @param[in] pos The starting position that bytes will be allocated after.
80+
* @param[in] add_size The minimum number of bytes to be allocated.
81+
* @param[out] out_of_space Whether the allocation failed due to insufficient disk space.
82+
* @return The number of bytes successfully allocated.
83+
*/
84+
size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space);
85+
86+
/**
87+
* Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
88+
*
89+
* @param[in] pos The first unwritten position in the file to be flushed.
90+
* @param[in] finalize True if no more data will be written to this file.
91+
* @return true on success, false on failure.
92+
*/
93+
bool Flush(const FlatFilePos& pos, bool finalize = false);
94+
};
95+
96+
#endif // BITCOIN_FLATFILE_H

src/index/txindex.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,27 +16,27 @@ constexpr char DB_TXINDEX_BLOCK = 'T';
1616

1717
std::unique_ptr<TxIndex> g_txindex;
1818

19-
struct CDiskTxPos : public CDiskBlockPos
19+
struct CDiskTxPos : public FlatFilePos
2020
{
2121
unsigned int nTxOffset; // after header
2222

2323
ADD_SERIALIZE_METHODS;
2424

2525
template <typename Stream, typename Operation>
2626
inline void SerializationOp(Stream& s, Operation ser_action) {
27-
READWRITEAS(CDiskBlockPos, *this);
27+
READWRITEAS(FlatFilePos, *this);
2828
READWRITE(VARINT(nTxOffset));
2929
}
3030

31-
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
31+
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
3232
}
3333

3434
CDiskTxPos() {
3535
SetNull();
3636
}
3737

3838
void SetNull() {
39-
CDiskBlockPos::SetNull();
39+
FlatFilePos::SetNull();
4040
nTxOffset = 0;
4141
}
4242
};

src/init.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -668,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles)
668668
if (fReindex) {
669669
int nFile = 0;
670670
while (true) {
671-
CDiskBlockPos pos(nFile, 0);
672-
if (!fs::exists(GetBlockPosFilename(pos, "blk")))
671+
FlatFilePos pos(nFile, 0);
672+
if (!fs::exists(GetBlockPosFilename(pos)))
673673
break; // No block files left to reindex
674674
FILE *file = OpenBlockFile(pos, true);
675675
if (!file)
@@ -1669,11 +1669,11 @@ bool AppInitMain(InitInterfaces& interfaces)
16691669

16701670
// ********************************************************* Step 11: import blocks
16711671

1672-
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) {
1672+
if (!CheckDiskSpace(GetDataDir())) {
16731673
InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir()));
16741674
return false;
16751675
}
1676-
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) {
1676+
if (!CheckDiskSpace(GetBlocksDir())) {
16771677
InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir()));
16781678
return false;
16791679
}

0 commit comments

Comments
 (0)