Merge bitcoin/bitcoin#29130: wallet: Add createwalletdescriptor and gethdkeys RPCs for adding new automatically generated descriptors

746b6d8 test: Add test for createwalletdescriptor (Ava Chow)
2402b63 wallet: Test upgrade of pre-taproot wallet to have tr() descriptors (Ava Chow)
460ae1b wallet, rpc: Add createwalletdescriptor RPC (Ava Chow)
8e1a475 wallet: Be able to retrieve single key from descriptors (Ava Chow)
85b1fb1 wallet: Add GetActiveHDPubKeys to retrieve xpubs from active descriptors (Ava Chow)
73926f2 wallet, descspkm: Refactor wallet descriptor generation to standalone func (Andrew Chow)
54e74f4 wallet: Refactor function for single DescSPKM setup (Andrew Chow)
3b09d0e tests: Test for gethdkeys (Ava Chow)
5febe28 wallet, rpc: Add gethdkeys RPC (Ava Chow)
66632e5 wallet: Add IsActiveScriptPubKeyMan (Ava Chow)
fa6a259 desc spkm: Add functions to retrieve specific private keys (Ava Chow)
fe67841 descriptor: Be able to get the pubkeys involved in a descriptor (Ava Chow)
ef67458 key: Add constructor for CExtKey that takes CExtPubKey and CKey (Ava Chow)

Pull request description:

  This PR adds a `createwalletdescriptor` RPC which allows users to add new automatically generated descriptors to their wallet, e.g. to upgrade a 0.21.x wallet to contain a taproot descriptor. This RPC takes 3 arguments: the output type to create a descriptor for, whether the descriptor will be internal or external, and the HD key to use if the user wishes to use a specific key. The HD key is an optional parameter. If it is not specified, the wallet will use the key shared by the active descriptors, if they are all single key. For most users in the expected upgrade scenario, this should be sufficient. In more advanced cases, the user must specify the HD key to use.

  Currently, specified HD keys must already exist in the wallet. To make it easier for the user to know, `gethdkeys` is also added to list out the HD keys in use by all of the descriptors in the wallet. This will include all HD keys, whether we have the private key, for it, which descriptors use it and their activeness, and optionally the extended private key. In this way, users with more complex wallets will be still be able to get HD keys from their wallet for use in other scenarios, and if they want to use `createwalletdescriptor`, they can easily get the keys that they can specify to it.

  See also bitcoin/bitcoin#26728 (comment)

ACKs for top commit:
  Sjors:
    re-utACK 746b6d8
  furszy:
    ACK 746b6d8
  ryanofsky:
    Code review ACK 746b6d8, and this looks ready to merge. There were various suggested changes since last review where main change seems to be switching `gethdkeys` output to use normalized descriptors (removing hardened path components).

Tree-SHA512: f2849101e6fbf1f59cb031eaaaee97af5b1ae92aaab54c5716940d210f08ab4fc952df2725b636596cd5747b8f5beb1a7a533425bc10d09da02659473516fbda

Author: ryanofsky

src/key.h

@@ -223,6 +223,12 @@ struct CExtKey {
             a.key == b.key;
     }
 
+    CExtKey() = default;
+    CExtKey(const CExtPubKey& xpub, const CKey& key_in) : nDepth(xpub.nDepth), nChild(xpub.nChild), chaincode(xpub.chaincode), key(key_in)
+    {
+        std::copy(xpub.vchFingerprint, xpub.vchFingerprint + sizeof(xpub.vchFingerprint), vchFingerprint);
+    }
+
     void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const;
     void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]);
     [[nodiscard]] bool Derive(CExtKey& out, unsigned int nChild) const;

src/rpc/client.cpp

@@ -277,6 +277,11 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "logging", 1, "exclude" },
     { "disconnectnode", 1, "nodeid" },
     { "upgradewallet", 0, "version" },
+    { "gethdkeys", 0, "active_only" },
+    { "gethdkeys", 0, "options" },
+    { "gethdkeys", 0, "private" },
+    { "createwalletdescriptor", 1, "options" },
+    { "createwalletdescriptor", 1, "internal" },
     // Echo with conversion (For testing only)
     { "echojson", 0, "arg0" },
     { "echojson", 1, "arg1" },

src/script/descriptor.cpp

@@ -212,6 +212,11 @@ struct PubkeyProvider
 
     /** Derive a private key, if private data is available in arg. */
     virtual bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const = 0;
+
+    /** Return the non-extended public key for this PubkeyProvider, if it has one. */
+    virtual std::optional<CPubKey> GetRootPubKey() const = 0;
+    /** Return the extended public key for this PubkeyProvider, if it has one. */
+    virtual std::optional<CExtPubKey> GetRootExtPubKey() const = 0;
 };
 
 class OriginPubkeyProvider final : public PubkeyProvider
@@ -265,6 +270,14 @@ class OriginPubkeyProvider final : public PubkeyProvider
     {
         return m_provider->GetPrivKey(pos, arg, key);
     }
+    std::optional<CPubKey> GetRootPubKey() const override
+    {
+        return m_provider->GetRootPubKey();
+    }
+    std::optional<CExtPubKey> GetRootExtPubKey() const override
+    {
+        return m_provider->GetRootExtPubKey();
+    }
 };
 
 /** An object representing a parsed constant public key in a descriptor. */
@@ -310,6 +323,14 @@ class ConstPubkeyProvider final : public PubkeyProvider
     {
         return arg.GetKey(m_pubkey.GetID(), key);
     }
+    std::optional<CPubKey> GetRootPubKey() const override
+    {
+        return m_pubkey;
+    }
+    std::optional<CExtPubKey> GetRootExtPubKey() const override
+    {
+        return std::nullopt;
+    }
 };
 
 enum class DeriveType {
@@ -525,6 +546,14 @@ class BIP32PubkeyProvider final : public PubkeyProvider
         key = extkey.key;
         return true;
     }
+    std::optional<CPubKey> GetRootPubKey() const override
+    {
+        return std::nullopt;
+    }
+    std::optional<CExtPubKey> GetRootExtPubKey() const override
+    {
+        return m_root_extkey;
+    }
 };
 
 /** Base class for all Descriptor implementations. */
@@ -720,6 +749,19 @@ class DescriptorImpl : public Descriptor
     std::optional<int64_t> MaxSatisfactionWeight(bool) const override { return {}; }
 
     std::optional<int64_t> MaxSatisfactionElems() const override { return {}; }
+
+    void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const override
+    {
+        for (const auto& p : m_pubkey_args) {
+            std::optional<CPubKey> pub = p->GetRootPubKey();
+            if (pub) pubkeys.insert(*pub);
+            std::optional<CExtPubKey> ext_pub = p->GetRootExtPubKey();
+            if (ext_pub) ext_pubs.insert(*ext_pub);
+        }
+        for (const auto& arg : m_subdescriptor_args) {
+            arg->GetPubKeys(pubkeys, ext_pubs);
+        }
+    }
 };
 
 /** A parsed addr(A) descriptor. */

src/script/descriptor.h

@@ -158,6 +158,13 @@ struct Descriptor {
 
     /** Get the maximum size number of stack elements for satisfying this descriptor. */
     virtual std::optional<int64_t> MaxSatisfactionElems() const = 0;
+
+    /** Return all (extended) public keys for this descriptor, including any from subdescriptors.
+     *
+     * @param[out] pubkeys Any public keys
+     * @param[out] ext_pubs Any extended public keys
+     */
+    virtual void GetPubKeys(std::set<CPubKey>& pubkeys, std::set<CExtPubKey>& ext_pubs) const = 0;
 };
 
 /** Parse a `descriptor` string. Included private keys are put in `out`.

src/wallet/rpc/wallet.cpp

@@ -817,6 +817,217 @@ static RPCHelpMan migratewallet()
     };
 }
 
+RPCHelpMan gethdkeys()
+{
+    return RPCHelpMan{
+        "gethdkeys",
+        "\nList all BIP 32 HD keys in the wallet and which descriptors use them.\n",
+        {
+            {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", {
+                {"active_only", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show the keys for only active descriptors"},
+                {"private", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show private keys"}
+            }},
+        },
+        RPCResult{RPCResult::Type::ARR, "", "", {
+            {
+                {RPCResult::Type::OBJ, "", "", {
+                    {RPCResult::Type::STR, "xpub", "The extended public key"},
+                    {RPCResult::Type::BOOL, "has_private", "Whether the wallet has the private key for this xpub"},
+                    {RPCResult::Type::STR, "xprv", /*optional=*/true, "The extended private key if \"private\" is true"},
+                    {RPCResult::Type::ARR, "descriptors", "Array of descriptor objects that use this HD key",
+                    {
+                        {RPCResult::Type::OBJ, "", "", {
+                            {RPCResult::Type::STR, "desc", "Descriptor string representation"},
+                            {RPCResult::Type::BOOL, "active", "Whether this descriptor is currently used to generate new addresses"},
+                        }},
+                    }},
+                }},
+            }
+        }},
+        RPCExamples{
+            HelpExampleCli("gethdkeys", "") + HelpExampleRpc("gethdkeys", "")
+            + HelpExampleCliNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}}) + HelpExampleRpcNamed("gethdkeys", {{"active_only", "true"}, {"private", "true"}})
+        },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+        {
+            const std::shared_ptr<const CWallet> wallet = GetWalletForJSONRPCRequest(request);
+            if (!wallet) return UniValue::VNULL;
+
+            if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) {
+                throw JSONRPCError(RPC_WALLET_ERROR, "gethdkeys is not available for non-descriptor wallets");
+            }
+
+            LOCK(wallet->cs_wallet);
+
+            UniValue options{request.params[0].isNull() ? UniValue::VOBJ : request.params[0]};
+            const bool active_only{options.exists("active_only") ? options["active_only"].get_bool() : false};
+            const bool priv{options.exists("private") ? options["private"].get_bool() : false};
+            if (priv) {
+                EnsureWalletIsUnlocked(*wallet);
+            }
+
+
+            std::set<ScriptPubKeyMan*> spkms;
+            if (active_only) {
+                spkms = wallet->GetActiveScriptPubKeyMans();
+            } else {
+                spkms = wallet->GetAllScriptPubKeyMans();
+            }
+
+            std::map<CExtPubKey, std::set<std::tuple<std::string, bool, bool>>> wallet_xpubs;
+            std::map<CExtPubKey, CExtKey> wallet_xprvs;
+            for (auto* spkm : spkms) {
+                auto* desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(spkm)};
+                CHECK_NONFATAL(desc_spkm);
+                LOCK(desc_spkm->cs_desc_man);
+                WalletDescriptor w_desc = desc_spkm->GetWalletDescriptor();
+
+                // Retrieve the pubkeys from the descriptor
+                std::set<CPubKey> desc_pubkeys;
+                std::set<CExtPubKey> desc_xpubs;
+                w_desc.descriptor->GetPubKeys(desc_pubkeys, desc_xpubs);
+                for (const CExtPubKey& xpub : desc_xpubs) {
+                    std::string desc_str;
+                    bool ok = desc_spkm->GetDescriptorString(desc_str, false);
+                    CHECK_NONFATAL(ok);
+                    wallet_xpubs[xpub].emplace(desc_str, wallet->IsActiveScriptPubKeyMan(*spkm), desc_spkm->HasPrivKey(xpub.pubkey.GetID()));
+                    if (std::optional<CKey> key = priv ? desc_spkm->GetKey(xpub.pubkey.GetID()) : std::nullopt) {
+                        wallet_xprvs[xpub] = CExtKey(xpub, *key);
+                    }
+                }
+            }
+
+            UniValue response(UniValue::VARR);
+            for (const auto& [xpub, descs] : wallet_xpubs) {
+                bool has_xprv = false;
+                UniValue descriptors(UniValue::VARR);
+                for (const auto& [desc, active, has_priv] : descs) {
+                    UniValue d(UniValue::VOBJ);
+                    d.pushKV("desc", desc);
+                    d.pushKV("active", active);
+                    has_xprv |= has_priv;
+
+                    descriptors.push_back(std::move(d));
+                }
+                UniValue xpub_info(UniValue::VOBJ);
+                xpub_info.pushKV("xpub", EncodeExtPubKey(xpub));
+                xpub_info.pushKV("has_private", has_xprv);
+                if (priv) {
+                    xpub_info.pushKV("xprv", EncodeExtKey(wallet_xprvs.at(xpub)));
+                }
+                xpub_info.pushKV("descriptors", std::move(descriptors));
+
+                response.push_back(std::move(xpub_info));
+            }
+
+            return response;
+        },
+    };
+}
+
+static RPCHelpMan createwalletdescriptor()
+{
+    return RPCHelpMan{"createwalletdescriptor",
+        "Creates the wallet's descriptor for the given address type. "
+        "The address type must be one that the wallet does not already have a descriptor for."
+        + HELP_REQUIRING_PASSPHRASE,
+        {
+            {"type", RPCArg::Type::STR, RPCArg::Optional::NO, "The address type the descriptor will produce. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."},
+            {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", {
+                {"internal", RPCArg::Type::BOOL, RPCArg::DefaultHint{"Both external and internal will be generated unless this parameter is specified"}, "Whether to only make one descriptor that is internal (if parameter is true) or external (if parameter is false)"},
+                {"hdkey", RPCArg::Type::STR, RPCArg::DefaultHint{"The HD key used by all other active descriptors"}, "The HD key that the wallet knows the private key of, listed using 'gethdkeys', to use for this descriptor's key"},
+            }},
+        },
+        RPCResult{
+            RPCResult::Type::OBJ, "", "",
+            {
+                {RPCResult::Type::ARR, "descs", "The public descriptors that were added to the wallet",
+                    {{RPCResult::Type::STR, "", ""}}
+                }
+            },
+        },
+        RPCExamples{
+            HelpExampleCli("createwalletdescriptor", "bech32m")
+            + HelpExampleRpc("createwalletdescriptor", "bech32m")
+        },
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+        {
+            std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request);
+            if (!pwallet) return UniValue::VNULL;
+
+            //  Make sure wallet is a descriptor wallet
+            if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) {
+                throw JSONRPCError(RPC_WALLET_ERROR, "createwalletdescriptor is not available for non-descriptor wallets");
+            }
+
+            std::optional<OutputType> output_type = ParseOutputType(request.params[0].get_str());
+            if (!output_type) {
+                throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str()));
+            }
+
+            UniValue options{request.params[1].isNull() ? UniValue::VOBJ : request.params[1]};
+            UniValue internal_only{options["internal"]};
+            UniValue hdkey{options["hdkey"]};
+
+            std::vector<bool> internals;
+            if (internal_only.isNull()) {
+                internals.push_back(false);
+                internals.push_back(true);
+            } else {
+                internals.push_back(internal_only.get_bool());
+            }
+
+            LOCK(pwallet->cs_wallet);
+            EnsureWalletIsUnlocked(*pwallet);
+
+            CExtPubKey xpub;
+            if (hdkey.isNull()) {
+                std::set<CExtPubKey> active_xpubs = pwallet->GetActiveHDPubKeys();
+                if (active_xpubs.size() != 1) {
+                    throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to determine which HD key to use from active descriptors. Please specify with 'hdkey'");
+                }
+                xpub = *active_xpubs.begin();
+            } else {
+                xpub = DecodeExtPubKey(hdkey.get_str());
+                if (!xpub.pubkey.IsValid()) {
+                    throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to parse HD key. Please provide a valid xpub");
+                }
+            }
+
+            std::optional<CKey> key = pwallet->GetKey(xpub.pubkey.GetID());
+            if (!key) {
+                throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Private key for %s is not known", EncodeExtPubKey(xpub)));
+            }
+            CExtKey active_hdkey(xpub, *key);
+
+            std::vector<std::reference_wrapper<DescriptorScriptPubKeyMan>> spkms;
+            WalletBatch batch{pwallet->GetDatabase()};
+            for (bool internal : internals) {
+                WalletDescriptor w_desc = GenerateWalletDescriptor(xpub, *output_type, internal);
+                uint256 w_id = DescriptorID(*w_desc.descriptor);
+                if (!pwallet->GetScriptPubKeyMan(w_id)) {
+                    spkms.emplace_back(pwallet->SetupDescriptorScriptPubKeyMan(batch, active_hdkey, *output_type, internal));
+                }
+            }
+            if (spkms.empty()) {
+                throw JSONRPCError(RPC_WALLET_ERROR, "Descriptor already exists");
+            }
+
+            // Fetch each descspkm from the wallet in order to get the descriptor strings
+            UniValue descs{UniValue::VARR};
+            for (const auto& spkm : spkms) {
+                std::string desc_str;
+                bool ok = spkm.get().GetDescriptorString(desc_str, false);
+                CHECK_NONFATAL(ok);
+                descs.push_back(desc_str);
+            }
+            UniValue out{UniValue::VOBJ};
+            out.pushKV("descs", std::move(descs));
+            return out;
+        }
+    };
+}
+
 // addresses
 RPCHelpMan getaddressinfo();
 RPCHelpMan getnewaddress();
@@ -900,13 +1111,15 @@ Span<const CRPCCommand> GetWalletRPCCommands()
         {"wallet", &bumpfee},
         {"wallet", &psbtbumpfee},
         {"wallet", &createwallet},
+        {"wallet", &createwalletdescriptor},
         {"wallet", &restorewallet},
         {"wallet", &dumpprivkey},
         {"wallet", &dumpwallet},
         {"wallet", &encryptwallet},
         {"wallet", &getaddressesbylabel},
         {"wallet", &getaddressinfo},
         {"wallet", &getbalance},
+        {"wallet", &gethdkeys},
         {"wallet", &getnewaddress},
         {"wallet", &getrawchangeaddress},
         {"wallet", &getreceivedbyaddress},