Merge bitcoin/bitcoin#32113: fuzz: enable running fuzz test cases in Debug mode

3669ecd doc: Document fuzz build options (Anthony Towns)
c1d01f5 fuzz: enable running fuzz test cases in Debug mode (Anthony Towns)

Pull request description:

  When building with

      BUILD_FOR_FUZZING=OFF
      BUILD_FUZZ_BINARY=ON
      CMAKE_BUILD_TYPE=Debug

  allow the fuzz binary to execute given test cases (without actual fuzzing) to make it easier to reproduce fuzz test failures in a more normal debug build.

  In Debug builds, deterministic fuzz behaviour is controlled via a runtime variable, which is normally false, but set to true automatically in the fuzz binary, unless the FUZZ_NONDETERMINISM environment variable is set.

ACKs for top commit:
  maflcko:
    re-ACK 3669ecd 🏉
  marcofleon:
    re ACK 3669ecd
  ryanofsky:
    Code review ACK 3669ecd with just variable renamed and documentation added since last review

Tree-SHA512: 5da5736462f98437d0aa1bd01aeacb9d46a9cc446a748080291067f7a27854c89f560f3a6481b760b9a0ea15a8d3ad90cd329ee2a008e5e347a101ed2516449e

Author: ryanofsky

doc/fuzzing.md

@@ -150,6 +150,37 @@ If you find coverage increasing inputs when fuzzing you are highly encouraged to
 
 Every single pull request submitted against the Bitcoin Core repo is automatically tested against all inputs in the [`bitcoin-core/qa-assets`](https://github.com/bitcoin-core/qa-assets) repo. Contributing new coverage increasing inputs is an easy way to help make Bitcoin Core more robust.
 
+## Building and debugging fuzz tests
+
+There are 3 ways fuzz tests can be built:
+
+1. With `-DBUILD_FOR_FUZZING=ON` which forces on fuzz determinism (skipping
+   proof of work checks, disabling random number seeding, disabling clock time)
+   and causes `Assume()` checks to abort on failure.
+
+   This is the normal way to run fuzz tests and generate new inputs. Because
+   determinism is hardcoded on in this build, only the fuzz binary can be built
+   and all other binaries are disabled.
+
+2. With `-DBUILD_FUZZ_BINARY=ON -DCMAKE_BUILD_TYPE=Debug` which causes
+   `Assume()` checks to abort on failure, and enables fuzz determinism, but
+   makes it optional.
+
+   Determinism is turned on in the fuzz binary by default, but can be turned off
+   by setting the `FUZZ_NONDETERMINISM` environment variable to any value, which
+   may be useful for running fuzz tests with code that deterministic execution
+   would otherwise skip.
+
+   Since `BUILD_FUZZ_BINARY`, unlike `BUILD_FOR_FUZZING`, does not hardcode on
+   determinism, this allows non-fuzz binaries to coexist in the same build,
+   making it possible to reproduce fuzz test failures in a normal build.
+
+3. With `-DBUILD_FUZZ_BINARY=ON -DCMAKE_BUILD_TYPE=Release`. In this build, the
+   fuzz binary will build but refuse to run, because in release builds
+   determinism is forced off and `Assume()` checks do not abort, so running the
+   tests would not be useful. This build is only useful for ensuring fuzz tests
+   compile and link.
+
 ## macOS hints for libFuzzer
 
 The default Clang/LLVM version supplied by Apple on macOS does not include

src/pow.cpp

@@ -139,7 +139,7 @@ bool PermittedDifficultyTransition(const Consensus::Params& params, int64_t heig
 // the most significant bit of the last byte of the hash is set.
 bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params& params)
 {
-    if constexpr (G_FUZZING) return (hash.data()[31] & 0x80) == 0;
+    if (EnableFuzzDeterminism()) return (hash.data()[31] & 0x80) == 0;
     return CheckProofOfWorkImpl(hash, nBits, params);
 }
 

src/test/fuzz/fuzz.cpp

@@ -149,10 +149,18 @@ static void initialize()
         std::cerr << "No fuzz target compiled for " << g_fuzz_target << "." << std::endl;
         std::exit(EXIT_FAILURE);
     }
-    if constexpr (!G_FUZZING) {
-        std::cerr << "Must compile with -DBUILD_FOR_FUZZING=ON to execute a fuzz target." << std::endl;
+    if constexpr (!G_FUZZING_BUILD && !G_ABORT_ON_FAILED_ASSUME) {
+        std::cerr << "Must compile with -DBUILD_FOR_FUZZING=ON or in Debug mode to execute a fuzz target." << std::endl;
         std::exit(EXIT_FAILURE);
     }
+    if (!EnableFuzzDeterminism()) {
+        if (std::getenv("FUZZ_NONDETERMINISM")) {
+            std::cerr << "Warning: FUZZ_NONDETERMINISM env var set, results may be inconsistent with fuzz build" << std::endl;
+        } else {
+            g_enable_dynamic_fuzz_determinism = true;
+            assert(EnableFuzzDeterminism());
+        }
+    }
     Assert(!g_test_one_input);
     g_test_one_input = &it->second.test_one_input;
     it->second.opts.init();

src/test/util/random.cpp

@@ -25,7 +25,7 @@ void SeedRandomStateForTest(SeedRand seedtype)
     // no longer truly random. It should be enough to get the seed once for the
     // process.
     static const auto g_ctx_seed = []() -> std::optional<uint256> {
-        if constexpr (G_FUZZING) return {};
+        if (EnableFuzzDeterminism()) return {};
         // If RANDOM_CTX_SEED is set, use that as seed.
         if (const char* num{std::getenv(RANDOM_CTX_SEED)}) {
             if (auto num_parsed{uint256::FromUserHex(num)}) {
@@ -40,7 +40,7 @@ void SeedRandomStateForTest(SeedRand seedtype)
     }();
 
     g_seeded_g_prng_zero = seedtype == SeedRand::ZEROS;
-    if constexpr (G_FUZZING) {
+    if (EnableFuzzDeterminism()) {
         Assert(g_seeded_g_prng_zero); // Only SeedRandomStateForTest(SeedRand::ZEROS) is allowed in fuzz tests
         Assert(!g_used_g_prng);       // The global PRNG must not have been used before SeedRandomStateForTest(SeedRand::ZEROS)
     }

src/test/util/setup_common.cpp

@@ -112,7 +112,7 @@ static void ExitFailure(std::string_view str_err)
 BasicTestingSetup::BasicTestingSetup(const ChainType chainType, TestOpts opts)
     : m_args{}
 {
-    if constexpr (!G_FUZZING) {
+    if (!EnableFuzzDeterminism()) {
         SeedRandomForTest(SeedRand::FIXED_SEED);
     }
     m_node.shutdown_signal = &m_interrupt;
@@ -203,7 +203,7 @@ BasicTestingSetup::~BasicTestingSetup()
 {
     m_node.ecc_context.reset();
     m_node.kernel.reset();
-    if constexpr (!G_FUZZING) {
+    if (!EnableFuzzDeterminism()) {
         SetMockTime(0s); // Reset mocktime for following tests
     }
     LogInstance().DisconnectTestLogger();
@@ -229,8 +229,9 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
         m_node.scheduler->m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { m_node.scheduler->serviceQueue(); });
         m_node.validation_signals =
             // Use synchronous task runner while fuzzing to avoid non-determinism
-            G_FUZZING ? std::make_unique<ValidationSignals>(std::make_unique<util::ImmediateTaskRunner>()) :
-                        std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(*m_node.scheduler));
+            EnableFuzzDeterminism() ?
+                std::make_unique<ValidationSignals>(std::make_unique<util::ImmediateTaskRunner>()) :
+                std::make_unique<ValidationSignals>(std::make_unique<SerialTaskRunner>(*m_node.scheduler));
         {
             // Ensure deterministic coverage by waiting for m_service_thread to be running
             std::promise<void> promise;
@@ -255,7 +256,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
             .notifications = *m_node.notifications,
             .signals = m_node.validation_signals.get(),
             // Use no worker threads while fuzzing to avoid non-determinism
-            .worker_threads_num = G_FUZZING ? 0 : 2,
+            .worker_threads_num = EnableFuzzDeterminism() ? 0 : 2,
         };
         if (opts.min_validation_cache) {
             chainman_opts.script_execution_cache_bytes = 0;

src/util/check.cpp

@@ -33,3 +33,5 @@ void assertion_fail(std::string_view file, int line, std::string_view func, std:
     fwrite(str.data(), 1, str.size(), stderr);
     std::abort();
 }
+
+std::atomic<bool> g_enable_dynamic_fuzz_determinism{false};

src/util/check.h

@@ -7,19 +7,44 @@
 
 #include <attributes.h>
 
+#include <atomic>
 #include <cassert> // IWYU pragma: export
 #include <stdexcept>
 #include <string>
 #include <string_view>
 #include <utility>
 
-constexpr bool G_FUZZING{
+constexpr bool G_FUZZING_BUILD{
 #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
     true
 #else
     false
 #endif
 };
+constexpr bool G_ABORT_ON_FAILED_ASSUME{
+#ifdef ABORT_ON_FAILED_ASSUME
+    true
+#else
+    false
+#endif
+};
+
+extern std::atomic<bool> g_enable_dynamic_fuzz_determinism;
+
+inline bool EnableFuzzDeterminism()
+{
+    if constexpr (G_FUZZING_BUILD) {
+        return true;
+    } else if constexpr (!G_ABORT_ON_FAILED_ASSUME) {
+        // Running fuzz tests is always disabled if Assume() doesn't abort
+        // (ie, non-fuzz non-debug builds), as otherwise tests which
+        // should fail due to a failing Assume may still pass. As such,
+        // we also statically disable fuzz determinism in that case.
+        return false;
+    } else {
+        return g_enable_dynamic_fuzz_determinism;
+    }
+}
 
 std::string StrFormatInternalBug(std::string_view msg, std::string_view file, int line, std::string_view func);
 
@@ -50,11 +75,7 @@ void assertion_fail(std::string_view file, int line, std::string_view func, std:
 template <bool IS_ASSERT, typename T>
 constexpr T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion)
 {
-    if (IS_ASSERT || std::is_constant_evaluated() || G_FUZZING
-#ifdef ABORT_ON_FAILED_ASSUME
-        || true
-#endif
-    ) {
+    if (IS_ASSERT || std::is_constant_evaluated() || G_FUZZING_BUILD || G_ABORT_ON_FAILED_ASSUME) {
         if (!val) {
             assertion_fail(file, line, func, assertion);
         }