From fc9a0d3f1296c3f7aeb96e6c97f985fea165fef9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 23 Sep 2024 18:50:51 -0400 Subject: [PATCH 001/131] start by refactoring block index into a read-through cache --- go.mod | 58 ++++++- go.sum | 97 +++++++++++ lib/blockchain.go | 327 +++++++++++++++++++++++-------------- lib/db_utils.go | 7 + lib/peer.go | 7 +- lib/pos_blockchain.go | 64 ++++---- lib/pos_blockchain_test.go | 90 +++++----- lib/pos_consensus.go | 27 ++- lib/pos_consensus_test.go | 8 +- lib/postgres.go | 13 +- lib/server.go | 15 +- lib/state_change_syncer.go | 2 +- lib/txindex.go | 10 +- 13 files changed, 497 insertions(+), 228 deletions(-) diff --git a/go.mod b/go.mod index 31047fa1b..a64a34f67 100644 --- a/go.mod +++ b/go.mod @@ -10,8 +10,8 @@ require ( github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudflare/circl v1.1.0 github.com/davecgh/go-spew v1.1.1 - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 - github.com/decred/dcrd/lru v1.1.1 + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 + github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/dgraph-io/badger/v3 v3.2103.5 @@ -36,8 +36,8 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + golang.org/x/crypto v0.27.0 + golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 ) @@ -48,13 +48,46 @@ require ( ) require ( + decred.org/cspp/v2 v2.2.0 // indirect github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/dchest/siphash v1.2.3 // indirect + github.com/decred/base58 v1.0.5 // indirect + github.com/decred/dcrd v1.9.4 // indirect + github.com/decred/dcrd/addrmgr/v2 v2.0.4 // indirect + github.com/decred/dcrd/bech32 v1.1.4 // indirect + github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 // indirect + github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 // indirect + github.com/decred/dcrd/certgen v1.2.0 // indirect + github.com/decred/dcrd/chaincfg/chainhash v1.0.4 // indirect + github.com/decred/dcrd/chaincfg/v3 v3.2.1 // indirect + github.com/decred/dcrd/connmgr/v3 v3.1.2 // indirect + github.com/decred/dcrd/container/apbf v1.0.1 // indirect + github.com/decred/dcrd/container/lru v1.0.0 // indirect + github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect + github.com/decred/dcrd/crypto/rand v1.0.0 // indirect + github.com/decred/dcrd/crypto/ripemd160 v1.0.2 // indirect + github.com/decred/dcrd/database/v3 v3.0.2 // indirect + github.com/decred/dcrd/dcrec v1.0.1 // indirect + github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 // indirect + github.com/decred/dcrd/dcrjson/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrutil/v4 v4.0.2 // indirect + github.com/decred/dcrd/gcs/v4 v4.1.0 // indirect + github.com/decred/dcrd/math/uint256 v1.0.2 // indirect + github.com/decred/dcrd/mixing v0.4.1 // indirect + github.com/decred/dcrd/peer/v3 v3.1.2 // indirect + github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 // indirect + github.com/decred/dcrd/txscript/v4 v4.1.1 // indirect + github.com/decred/dcrd/wire v1.7.0 // indirect + github.com/decred/go-socks v1.1.0 // indirect + github.com/decred/slog v1.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -63,15 +96,20 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jessevdk/go-flags v1.6.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jrick/bitset v1.0.0 // indirect + github.com/jrick/logrotate v1.1.2 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.12.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.9 // indirect @@ -90,6 +128,7 @@ require ( github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect @@ -99,11 +138,11 @@ require ( github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gonum.org/v1/gonum v0.6.1 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -113,5 +152,6 @@ require ( gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.3.0 // indirect mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 3698c73f3..821ddcfdd 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +decred.org/cspp/v2 v2.2.0 h1:VSOUC1w0Wo+QOGS0r1XO6TLnO16X67KuvpDmRRYyr08= +decred.org/cspp/v2 v2.2.0/go.mod h1:9nO3bfvCheOPIFZw5f6sRQ42CjBFB5RKSaJ9Iq6G4MA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -39,6 +41,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= +github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -94,6 +98,8 @@ github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GK github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a h1:clYxJ3Os0EQUKDDVU8M0oipllX0EkuFNBfhVQuIfyF0= +github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a/go.mod h1:z/9Ck1EDixEbBbZ2KH2qNHekEmDLTOZ+FyoIPWWSVOI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -111,14 +117,78 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/decred/base58 v1.0.5 h1:hwcieUM3pfPnE/6p3J100zoRfGkQxBulZHo7GZfOqic= +github.com/decred/base58 v1.0.5/go.mod h1:s/8lukEHFA6bUQQb/v3rjUySJ2hu+RioCzLukAVkrfw= +github.com/decred/dcrd v1.9.4 h1:DgeZdf8TTtGuZGVs5DnAL0ghbhnrPYfENneqUqEaq8o= +github.com/decred/dcrd v1.9.4/go.mod h1:7I6LovGdFimw3LTPCtOGQ+8VV1HGnO+2L2KA4yi5NWA= +github.com/decred/dcrd/addrmgr/v2 v2.0.4 h1:3MWJiO2STogQwNRF3W4yjCzSJtaxqtw+UI3x2+bYeOg= +github.com/decred/dcrd/addrmgr/v2 v2.0.4/go.mod h1:661DIS/De2iLNLMwIKazUQfQypUqJ5om7PXNX0fEMms= +github.com/decred/dcrd/bech32 v1.1.4 h1:wFlLM7Oic0MlIhQZdCQhdIqVc4CNaQ0vNR9fgCoWfe0= +github.com/decred/dcrd/bech32 v1.1.4/go.mod h1:jliqHZmCbVfT06Lh1mQywEKFVidRclbBJIUmwdoKhu0= +github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 h1:KDm6myUPi8j2TTL7LZ+iT+R/pIbxd8qG89fjJNitzx0= +github.com/decred/dcrd/blockchain/stake/v5 v5.0.1/go.mod h1:y1tMD1TssTlPmKDYbSrF3Ujznj+STkXFfYPwoVfe+xA= +github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 h1:zeI9CHkLM9be4QOBmIAtoPfs6NCgJM1lpmRUYE61I8o= +github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1/go.mod h1:yXZz/EgWdGw5nqMEvyKj/iXZ9I2VSyO95xKj6mRUMIM= +github.com/decred/dcrd/certgen v1.2.0 h1:FF6XXV//5q38/c6QbGQdR35ZJz0GPIkejsZZU3oHuBQ= +github.com/decred/dcrd/certgen v1.2.0/go.mod h1:LRh6dF2WPQeDA6QQSZE+SfK7AL6FuFtCRDHZf8DyGzg= +github.com/decred/dcrd/chaincfg/chainhash v1.0.4 h1:zRCv6tdncLfLTKYqu7hrXvs7hW+8FO/NvwoFvGsrluU= +github.com/decred/dcrd/chaincfg/chainhash v1.0.4/go.mod h1:hA86XxlBWwHivMvxzXTSD0ZCG/LoYsFdWnCekkTMCqY= +github.com/decred/dcrd/chaincfg/v3 v3.2.1 h1:x9zKJaU24WAKbxAR1UyFKHlM3oJgP0H9LodokM4X5lM= +github.com/decred/dcrd/chaincfg/v3 v3.2.1/go.mod h1:SDCWDtY7BLj0leXc9FuoA1YjSVKyCIBVAyxwZn6+sXc= +github.com/decred/dcrd/connmgr/v3 v3.1.2 h1:+xNopie2L3YYwwkz51k0h/pASATOBzHtl2O8eodGg04= +github.com/decred/dcrd/connmgr/v3 v3.1.2/go.mod h1:tdbErFiNOuy/sHrX2mwaOk+r1HLs3EBz2EGxsocMPe4= +github.com/decred/dcrd/container/apbf v1.0.1 h1:oepQzRtLADudsrx0AmmowoU1kambozINTMXduH6Mge0= +github.com/decred/dcrd/container/apbf v1.0.1/go.mod h1:paQplssZMsRhwOUcP6LDNDypyb7lwsFHrUKUkAPFWtQ= +github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI= +github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/crypto/rand v1.0.0 h1:Ah9Asl36OZt09sGSMbJZuL1HfwGdlC38q/ZUeLDVKRg= +github.com/decred/dcrd/crypto/rand v1.0.0/go.mod h1:coa7BbxSTiKH6esi257plGfMFYuGL4MTbQlLYnOdzpE= +github.com/decred/dcrd/crypto/ripemd160 v1.0.2 h1:TvGTmUBHDU75OHro9ojPLK+Yv7gDl2hnUvRocRCjsys= +github.com/decred/dcrd/crypto/ripemd160 v1.0.2/go.mod h1:uGfjDyePSpa75cSQLzNdVmWlbQMBuiJkvXw/MNKRY4M= +github.com/decred/dcrd/database/v3 v3.0.2 h1:rgP7XNZemTs8ZC7bnTKO8JO79Woj5nq+yQYmB9ry7yM= +github.com/decred/dcrd/database/v3 v3.0.2/go.mod h1:3Ge1yoxEOsqd72V5LTA9g0B7mlY0MGbpxeE1fniIXsQ= +github.com/decred/dcrd/dcrec v1.0.1 h1:gDzlndw0zYxM5BlaV17d7ZJV6vhRe9njPBFeg4Db2UY= +github.com/decred/dcrd/dcrec v1.0.1/go.mod h1:CO+EJd8eHFb8WHa84C7ZBkXsNUIywaTHb+UAuI5uo6o= +github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 h1:l/lhv2aJCUignzls81+wvga0TFlyoZx8QxRMQgXpZik= +github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3/go.mod h1:AKpV6+wZ2MfPRJnTbQ6NPgWrKzbe9RCIlCF/FKzMtM8= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrjson/v4 v4.1.0 h1:WJVogRnYnNxB5hWoGHODvP4fNTG1JycTuHHKt/XucHk= +github.com/decred/dcrd/dcrjson/v4 v4.1.0/go.mod h1:2qVikafVF9/X3PngQVmqkbUbyAl32uik0k/kydgtqMc= +github.com/decred/dcrd/dcrutil/v4 v4.0.2 h1:eIl3E6gGln54qE8nk5o5lLtjh2/9C2Rz63OpD662h+8= +github.com/decred/dcrd/dcrutil/v4 v4.0.2/go.mod h1:iS3JB1ac3R3FgfpTF1kBD+SPNet8TmiW3Br+/Jc5MC8= +github.com/decred/dcrd/gcs/v4 v4.1.0 h1:tpW7JW53yJZlgNwl/n2NL1b8NxHaIPRUyNuLMkB/Hks= +github.com/decred/dcrd/gcs/v4 v4.1.0/go.mod h1:nPTbGM/I3Ihe5KFvUmxZEqQP/jDZQjQ63+WEi/f4lqU= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/decred/dcrd/lru v1.1.1 h1:kWFDaW0OWx6AD6Ki342c+JPmHbiVdE6rK81pT3fuo/Y= github.com/decred/dcrd/lru v1.1.1/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= +github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= +github.com/decred/dcrd/math/uint256 v1.0.2 h1:o8peafL5QmuXGTergI3YDpDU0eq5Z0pQi88B8ym4PRA= +github.com/decred/dcrd/math/uint256 v1.0.2/go.mod h1:7M/y9wJJvlyNG/f/X6mxxhxo9dgloZHFiOfbiscl75A= +github.com/decred/dcrd/mixing v0.4.1 h1:W8ZCzhmNyzG1xjJMA3L6FOElmp98Ttnk3dDUxD6irAE= +github.com/decred/dcrd/mixing v0.4.1/go.mod h1:ySvVwTZyVz5YvevA6YjPrB6pJEwTm7IkHohTfaiHh2c= +github.com/decred/dcrd/peer/v3 v3.1.2 h1:Qe7SpqDtfM0HARmDYwr4WjUu16X6HQ7ZWNnHqE1swiw= +github.com/decred/dcrd/peer/v3 v3.1.2/go.mod h1:M9FxNkHuEBtsRW5gwzIH4cJTWk5xSkxy9zG+TEL1N2Y= +github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 h1:l0DnCcILTNrpy8APF3FLN312ChpkQaAuW30aC/RgBaw= +github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0/go.mod h1:j+kkRPXPJB5S9VFOsx8SQLcU7PTFkPKRc1aCHN4ENzA= +github.com/decred/dcrd/txscript/v4 v4.1.1 h1:R4M2+jMujgQA91899SkL0cW66d6DC76Gx+1W1oEHjc0= +github.com/decred/dcrd/txscript/v4 v4.1.1/go.mod h1:7ybmJoI+b6dxvQ+0aXdZpkyrj0PbnylJCzFxD1g8+/A= +github.com/decred/dcrd/wire v1.7.0 h1:5JHiDjEQeS4XUl4PfnTZYLwAD/E/+LwBmPRec/fP76o= +github.com/decred/dcrd/wire v1.7.0/go.mod h1:lAqrzV0SU4kyV6INLEJgDtUjJaTaVKrbF4LHtaYl+zU= +github.com/decred/go-socks v1.1.0 h1:dnENcc0KIqQo3HSXdgboXAHgqsCIutkqq6ntQjYtm2U= +github.com/decred/go-socks v1.1.0/go.mod h1:sDhHqkZH0X4JjSa02oYOGhcGHYp12FsY1jQ/meV8md0= +github.com/decred/slog v1.2.0 h1:soHAxV52B54Di3WtKLfPum9OFfWqwtf/ygf9njdfnPM= +github.com/decred/slog v1.2.0/go.mod h1:kVXlGnt6DHy2fV5OjSeuvCJ0OmlmTF6LFpEPMu/fOY0= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -210,6 +280,8 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -242,6 +314,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -285,11 +359,17 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jrick/bitset v1.0.0 h1:Ws0PXV3PwXqWK2n7Vz6idCdrV/9OrBXgHEJi27ZB9Dw= +github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= +github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -306,6 +386,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -477,6 +559,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -546,6 +630,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -609,6 +695,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -622,6 +710,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -664,6 +754,8 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -671,6 +763,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= @@ -704,6 +798,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -796,6 +891,8 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= +lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= diff --git a/lib/blockchain.go b/lib/blockchain.go index e4796ec54..a03b55e39 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -18,6 +18,8 @@ import ( "github.com/decred/dcrd/lru" + lru2 "github.com/decred/dcrd/container/lru" + "github.com/deso-protocol/core/collections" "github.com/google/uuid" @@ -527,6 +529,115 @@ type CheckpointBlockInfoAndError struct { Error error } +type BlockIndex struct { + db *badger.DB + snapshot *Snapshot + blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + tipNode *BlockNode +} + +func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { + return &BlockIndex{ + db: db, + snapshot: snapshot, + blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + tipNode: tipNode, + } +} + +func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockHash, *BlockNode] { + newMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + for key, val := range input { + newMap.Put(key, val) + } + return newMap +} + +func (bi *BlockIndex) setTipNode(tipNode *BlockNode) { + bi.tipNode = tipNode +} + +func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { + bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) +} + +func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { + val, exists := bi.blockIndexByHash.Get(*blockHash) + if exists { + return val, true, nil + } + // NOTE: this can be sped up significantly by mapping blockHash to blockNode or blockHash to blockHeight + // in badger, so we can avoid deserializing the full block. + fullBlock, err := GetBlock(blockHash, bi.db, bi.snapshot) + if err != nil { + return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem reading block from db") + } + nodeHeader := fullBlock.Header + fullBlockHash, err := fullBlock.Hash() + if err != nil { + return nil, false, errors.Wrapf(err, "_initChain: Problem hashing block") + } + blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(nodeHeader.Height), fullBlockHash, false) + if blockNode == nil { + return nil, false, nil + } + bi.addNewBlockNodeToBlockIndex(blockNode) + return blockNode, true, nil +} + +func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + val, exists := bi.blockIndexByHash.Get(*blockHash) + if exists { + return val, true + } + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) + } + bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false + } + bi.addNewBlockNodeToBlockIndex(bn) + return bn, true +} + +func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) + } + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) + _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) + blockNodes := []*BlockNode{} + for _, val := range valsFound { + blockNode, err := DeserializeBlockNode(val) + if err != nil { + glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) + continue + } + bi.addNewBlockNodeToBlockIndex(blockNode) + blockNodes = append(blockNodes, blockNode) + } + return blockNodes +} + +type BestChain struct { + db *badger.DB + IsHeaderChain bool + Chain []*BlockNode + ChainMap map[BlockHash]*BlockNode + tipNode *BlockNode +} + +func NewBestChain(db *badger.DB, isHeaderChain bool, tipNode *BlockNode) *BestChain { + return &BestChain{ + db: db, + IsHeaderChain: isHeaderChain, + Chain: []*BlockNode{}, + ChainMap: make(map[BlockHash]*BlockNode), + tipNode: tipNode, + } +} + type Blockchain struct { db *badger.DB postgres *Postgres @@ -556,10 +667,10 @@ type Blockchain struct { // // An in-memory index of the "tree" of blocks we are currently aware of. // This index includes forks and side-chains. - blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] - // blockIndexByHeight is an in-memory map of block height to block nodes. This is - // used to quickly find the safe blocks from which the chain can be extended for PoS - blockIndexByHeight map[uint64]map[BlockHash]*BlockNode + //blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] + //// blockIndexByHeight is an in-memory map of block height to block nodes. This is + //// used to quickly find the safe blocks from which the chain can be extended for PoS + //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode // An in-memory slice of the blocks on the main chain only. The end of // this slice is the best known tip that we have at any given time. bestChain []*BlockNode @@ -568,6 +679,10 @@ type Blockchain struct { bestHeaderChain []*BlockNode bestHeaderChainMap map[BlockHash]*BlockNode + blockIndex *BlockIndex + //bestChain *BestChain + //bestHeaderChain *BestChain + // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep // up to MaxOrphansInMemory of them in order to prevent memory exhaustion. @@ -705,52 +820,33 @@ func getCheckpointBlockInfoFromProviderHelper(provider string) *CheckpointBlockI } func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bc.blockIndexByHash.Set(*blockNode.Hash, blockNode) - if _, exists := bc.blockIndexByHeight[uint64(blockNode.Height)]; !exists { - bc.blockIndexByHeight[uint64(blockNode.Height)] = make(map[BlockHash]*BlockNode) - } - bc.blockIndexByHeight[uint64(blockNode.Height)][*blockNode.Hash] = blockNode + bc.blockIndex.blockIndexByHash.Put(*blockNode.Hash, blockNode) } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode], - _blockIndexByHeight map[uint64]map[BlockHash]*BlockNode, + _blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { - newBlockIndexByHash := collections.NewConcurrentMap[BlockHash, *BlockNode]() - newBlockIndexByHeight := make(map[uint64]map[BlockHash]*BlockNode) - bc.blockIndexByHash.Iterate(func(kk BlockHash, vv *BlockNode) { - newBlockIndexByHash.Set(kk, vv) - blockHeight := uint64(vv.Height) - if _, exists := newBlockIndexByHeight[blockHeight]; !exists { - newBlockIndexByHeight[blockHeight] = make(map[BlockHash]*BlockNode) - } - newBlockIndexByHeight[blockHeight][kk] = vv - }) - return newBlockIndexByHash, newBlockIndexByHeight -} - -func (bc *Blockchain) constructBlockIndexByHeight() map[uint64]map[BlockHash]*BlockNode { - newBlockIndex := make(map[uint64]map[BlockHash]*BlockNode) - bc.blockIndexByHash.Iterate(func(_ BlockHash, blockNode *BlockNode) { - blockHeight := uint64(blockNode.Height) - if _, exists := newBlockIndex[blockHeight]; !exists { - newBlockIndex[blockHeight] = make(map[BlockHash]*BlockNode) - } - newBlockIndex[blockHeight][*blockNode.Hash] = blockNode - }) - return newBlockIndex + newBlockIndexByHash := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + for _, key := range bc.blockIndex.blockIndexByHash.Keys() { + val, _ := bc.blockIndex.blockIndexByHash.Get(key) + newBlockIndexByHash.Put(key, val) + } + return newBlockIndexByHash } +// TODO: read through to DB. func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { - return collections.MapValues(bc.blockIndexByHeight[blockHeight]) + return bc.blockIndex.GetBlockNodesByHeight(blockHeight) } func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { - blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] - if !hasNestedMapAtHeight { - return false - } - return len(blocksAtHeight) > 0 + //blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] + //if !hasNestedMapAtHeight { + // return false + //} + //return len(blocksAtHeight) > 0 + blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) + return len(blockNodes) > 0 } func (bc *Blockchain) CopyBestChain() ([]*BlockNode, map[BlockHash]*BlockNode) { @@ -840,26 +936,48 @@ func (bc *Blockchain) _initChain() error { // to previous blocks we've read in and error if they don't. This works because // reading blocks in height order as we do here ensures that we'll always // add a block's parents, if they exist, before adding the block itself. + //var err error + //if bc.postgres != nil { + // bc.blockIndexByHash, err = bc.postgres.GetBlockIndex() + //} else { + // bc.blockIndexByHash, err = GetBlockIndex(bc.db, false /*bitcoinNodes*/, bc.params) + //} + //if err != nil { + // return errors.Wrapf(err, "_initChain: Problem reading block index from db") + //} + //bc.blockIndexByHeight = bc.constructBlockIndexByHeight() + + // For postgres, we still load the entire block index into memory. This is because var err error + var tipNode *BlockNode if bc.postgres != nil { - bc.blockIndexByHash, err = bc.postgres.GetBlockIndex() + bc.blockIndex.blockIndexByHash, err = bc.postgres.GetBlockIndex() + var exists bool + tipNode, exists = bc.blockIndex.blockIndexByHash.Get(*bestBlockHash) + if !exists { + return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) + } } else { - bc.blockIndexByHash, err = GetBlockIndex(bc.db, false /*bitcoinNodes*/, bc.params) - } - if err != nil { - return errors.Wrapf(err, "_initChain: Problem reading block index from db") + var tipNodeExists bool + // For badger, we only need the tip block to get started. + // Weird hack required for the genesis block. + if bestBlockHash.IsEqual(GenesisBlockHash) { + tipNode, tipNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(bestBlockHash, 0) + } else { + tipNode, tipNodeExists, err = bc.blockIndex.GetBlockNodeByHashOnly(bestBlockHash) + if err != nil { + return errors.Wrapf(err, "_initChain: Problem reading best block from db") + } + if !tipNodeExists { + return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) + } + } } - bc.blockIndexByHeight = bc.constructBlockIndexByHeight() + bc.blockIndex.setTipNode(tipNode) // At this point the blockIndexByHash should contain a full node tree with all // nodes pointing to valid parent nodes. { - // Find the tip node with the best node hash. - tipNode, exists := bc.blockIndexByHash.Get(*bestBlockHash) - if !exists { - return fmt.Errorf("_initChain(block): Best hash (%#v) not found in block index", bestBlockHash) - } - // Walk back from the best node to the genesis block and store them all // in bestChain. bc.bestChain, err = GetBestChain(tipNode) @@ -873,12 +991,6 @@ func (bc *Blockchain) _initChain() error { // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. { - // Find the tip node with the best node hash. - tipNode, exists := bc.blockIndexByHash.Get(*bestHeaderHash) - if !exists { - return fmt.Errorf("_initChain(header): Best hash (%#v) not found in block index", bestHeaderHash) - } - // Walk back from the best node to the genesis block and store them all // in bestChain. bc.bestHeaderChain, err = GetBestChain(tipNode) @@ -992,12 +1104,13 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), - blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - bestChainMap: make(map[BlockHash]*BlockNode), - + blockIndex: NewBlockIndex(db, snapshot, nil), + //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), + //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), + bestChainMap: make(map[BlockHash]*BlockNode), + //bestChain: NewBestChain(db, false), bestHeaderChainMap: make(map[BlockHash]*BlockNode), - + //bestHeaderChain: NewBestChain(db, true), blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), @@ -1065,14 +1178,16 @@ func fastLog2Floor(n uint32) uint8 { // // This function MUST be called with the chain state lock held (for reads). func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *collections.ConcurrentMap[BlockHash, *BlockNode], bestChainList []*BlockNode, + blockIndex *BlockIndex, bestChainList []*BlockNode, bestChainMap map[BlockHash]*BlockNode) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists := blockIndex.Get(*stopHash) + stopNode, stopNodeExists, stopNodeError := blockIndex.GetBlockNodeByHashOnly(stopHash) if len(locator) == 0 { - if !stopNodeExists { + if stopNodeError != nil || !stopNodeExists || stopNode == nil { + // TODO: what should we really do here? + glog.Errorf("locateInventory: Block %v is not known", stopHash) // No blocks with the stop hash were found so there is // nothing to do. return nil, 0 @@ -1104,7 +1219,8 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Calculate how many entries are needed. tip := bestChainList[len(bestChainList)-1] total := uint32((tip.Header.Height - startNode.Header.Height) + 1) - if stopNodeExists && stopNode.Header.Height >= startNode.Header.Height { + if stopNodeError != nil && stopNodeExists && stopNode != nil && + stopNode.Header.Height >= startNode.Header.Height { _, bestChainContainsStopNode := bestChainMap[*stopNode.Hash] if bestChainContainsStopNode { @@ -1126,7 +1242,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // // This function MUST be called with the ChainLock held (for reads). func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *collections.ConcurrentMap[BlockHash, *BlockNode], bestChainList []*BlockNode, + blockIndex *BlockIndex, bestChainList []*BlockNode, bestChainMap map[BlockHash]*BlockNode) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the @@ -1177,7 +1293,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndexByHash, bc.bestChain, bc.bestChainMap) + bc.blockIndex, bc.bestChain, bc.bestChainMap) return headers } @@ -1256,8 +1372,11 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { } func (bc *Blockchain) HeaderLocatorWithNodeHash(blockHash *BlockHash) ([]*BlockHash, error) { - node, exists := bc.blockIndexByHash.Get(*blockHash) - if !exists { + node, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + if err != nil { + return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Problem getting node for hash %v: %v", blockHash, err) + } + if !exists || node == nil { return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Node for hash %v is not in our blockIndexByHash", blockHash) } @@ -1336,9 +1455,9 @@ func (bc *Blockchain) GetBlockNodesToFetch( return blockNodesToFetch } -func (bc *Blockchain) HasHeader(headerHash *BlockHash) bool { - _, exists := bc.blockIndexByHash.Get(*headerHash) - return exists +func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(headerHash) + return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { @@ -1349,42 +1468,12 @@ func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { return bc.bestHeaderChain[blockHeight] } -func (bc *Blockchain) HasBlock(blockHash *BlockHash) bool { - node, nodeExists := bc.blockIndexByHash.Get(*blockHash) - if !nodeExists { - glog.V(2).Infof("Blockchain.HasBlock: Node with hash %v does not exist in node index", blockHash) - return false - } - - if (node.Status & StatusBlockProcessed) == 0 { - glog.V(2).Infof("Blockchain.HasBlock: Node %v does not have StatusBlockProcessed so we don't have the block", node) - return false - } - - // Node exists with StatusBlockProcess set means we have it. - return true -} - -func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) bool { +func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - _, exists := bc.blockIndexByHash.Get(*blockHash) - return exists -} - -// This needs to hold a lock on the blockchain because it read from an in-memory map that is -// not thread-safe. -func (bc *Blockchain) GetBlockHeaderFromIndex(blockHash *BlockHash) *MsgDeSoHeader { - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - - block, blockExists := bc.blockIndexByHash.Get(*blockHash) - if !blockExists { - return nil - } - - return block.Header + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + return exists, errors.Wrap(err, "Blockchain.HasBlockInBlockIndex: ") } // Don't need a lock because blocks don't get removed from the db after they're added @@ -1690,13 +1779,11 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, bestChainMap map[BlockHash]*BlockNode, - blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode], - blockIndexByHeight map[uint64]map[BlockHash]*BlockNode, + blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { bc.bestChain = bestChain bc.bestChainMap = bestChainMap - bc.blockIndexByHash = blockIndexByHash - bc.blockIndexByHeight = blockIndexByHeight + bc.blockIndex.blockIndexByHash = blockIndexByHash } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2025,7 +2112,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // index. If it does, then return an error. We should generally // expect that processHeaderPoW will only be called on headers we // haven't seen before. - _, nodeExists := bc.blockIndexByHash.Get(*headerHash) + _, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) if nodeExists { return false, false, HeaderErrorDuplicateHeader } @@ -2050,7 +2137,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if blockHeader.PrevBlockHash == nil { return false, false, HeaderErrorNilPrevHash } - parentNode, parentNodeExists := bc.blockIndexByHash.Get(*blockHeader.PrevBlockHash) + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists { // This block is an orphan if its parent doesn't exist and we don't // process unconnectedTxns. @@ -2167,9 +2254,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if bc.isSyncing() { bc.addNewBlockNodeToBlockIndex(newNode) } else { - newBlockIndexByHash, newBlockIndexByHeight := bc.CopyBlockIndexes() - bc.blockIndexByHash = newBlockIndexByHash - bc.blockIndexByHeight = newBlockIndexByHeight + newBlockIndexByHash := bc.CopyBlockIndexes() + bc.blockIndex.blockIndexByHash = newBlockIndexByHash bc.addNewBlockNodeToBlockIndex(newNode) } @@ -2312,7 +2398,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. - nodeToValidate, nodeExists := bc.blockIndexByHash.Get(*blockHash) + // TODO: validate that current height - 1 > 0 + nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. @@ -2333,7 +2420,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reset the pointers after having presumably added the header to the // block index. - nodeToValidate, nodeExists = bc.blockIndexByHash.Get(*blockHash) + // TODO: validate that current height - 1 > 0 + nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point @@ -2352,7 +2440,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - parentNode, parentNodeExists := bc.blockIndexByHash.Get(*blockHeader.PrevBlockHash) + // TODO: validate that current height - 1 > 0 + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { return false, true, nil } diff --git a/lib/db_utils.go b/lib/db_utils.go index c51609681..2bb61e0f7 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5188,6 +5188,13 @@ func _heightHashToNodeIndexPrefix(bitcoinNodes bool) []byte { return prefix } +func _heightHashToNodePrefixByHeight(height uint32, bitcoinNodes bool) []byte { + prefix := _heightHashToNodeIndexPrefix(bitcoinNodes) + heightBytes := make([]byte, 4) + binary.BigEndian.PutUint32(heightBytes[:], height) + return append(prefix, heightBytes[:]...) +} + func _heightHashToNodeIndexKey(height uint32, hash *BlockHash, bitcoinNodes bool) []byte { prefix := _heightHashToNodeIndexPrefix(bitcoinNodes) diff --git a/lib/peer.go b/lib/peer.go index 9fda1bc7b..aece3eeff 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -330,7 +330,12 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { } else if invVect.Type == InvTypeBlock { // For blocks, we check that the hash isn't known to us either in our // main header chain or in side chains. - if pp.srv.blockchain.HasHeader(¤tHash) { + exists, err := pp.srv.blockchain.HasHeader(¤tHash) + if exists { + continue + } + if err != nil { + glog.Errorf("Server._handleInv: Error checking if block exists: %v", err) continue } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index bede52036..ff433b196 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -110,7 +110,8 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // process headers. func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { // Fetch all potential children of this blockNode from the block index. - blockNodesAtNextHeight, exists := bc.blockIndexByHeight[blockNode.Header.Height+1] + blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) + exists := len(blockNodesAtNextHeight) > 0 if !exists { // No children of this blockNode exist in the block index. Exit early. return @@ -137,8 +138,7 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas _headerBlockNode *BlockNode, _isOrphan bool, _err error, ) { // Look up the header in the block index to check if it has already been validated and indexed. - blockNode, blockNodeExists := bc.blockIndexByHash.Get(*headerHash) - + blockNode, blockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, header.Height) // ------------------------------------ Base Cases ----------------------------------- // // The header is already validated. Exit early. @@ -157,7 +157,8 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } // The header is an orphan. No need to store it in the block index. Exit early. - parentBlockNode, parentBlockNodeExists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + // TODO: validate that height - 1 > 0 + parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !parentBlockNodeExists { return nil, true, nil } @@ -310,7 +311,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // We expect the utxoView for the parent block to be valid because we check that all ancestor blocks have // been validated. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash) + parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -401,7 +402,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // Now that we've processed this block, we check for any blocks that were previously // stored as orphans, which are children of this block. We can process them now. - blockNodesAtNextHeight := bc.blockIndexByHeight[uint64(blockNode.Height)+1] + blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(uint64(blockNode.Height) + 1) for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { if blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) && blockNodeAtNextHeight.IsStored() && @@ -569,7 +570,7 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success if err != nil { return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem hashing block") } - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) // If the blockNode doesn't exist, or the block is not committed, or it's already stored, then we're not dealing // with an archival block. Archival blocks must have an existing blockNode, be committed, and not be stored. if !exists || !blockNode.IsCommitted() || blockNode.IsStored() { @@ -687,7 +688,8 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi } // Base case - Check if the block is validated or validate failed. If so, we can return early. - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + // TODO: validate height doesn't overflow uint32 + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) if exists && (blockNode.IsValidateFailed() || blockNode.IsValidated()) { return blockNode, nil } @@ -708,7 +710,7 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // Run the validation for the parent and update the block index with the parent's status. We first // check if the parent has a cached status. If so, we use the cached status. Otherwise, we run // the full validation algorithm on it, then index it and use the result. - parentBlockNode, err := bc.validatePreviouslyIndexedBlockPoS(block.Header.PrevBlockHash, verifySignatures) + parentBlockNode, err := bc.validatePreviouslyIndexedBlockPoS(block.Header.PrevBlockHash, block.Header.Height-1, verifySignatures) if err != nil { return blockNode, errors.Wrapf(err, "validateAndIndexBlockPoS: Problem validating previously indexed block: ") } @@ -789,10 +791,11 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // cached block, and runs the validateAndIndexBlockPoS algorithm on it. It returns the resulting BlockNode. func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( blockHash *BlockHash, + blockHeight uint64, verifySignatures bool, ) (*BlockNode, error) { // Check if the block is already in the block index. If so, we check its current status first. - blockNode, exists := bc.blockIndexByHash.Get(*blockHash) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeight) if !exists { // We should never really hit this if the block has already been cached in the block index first. // We check here anyway to be safe. @@ -814,7 +817,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( return nil, errors.Wrapf(err, "validatePreviouslyIndexedBlockPoS: Problem fetching block from DB") } // Build utxoView for the block's parent. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash) + parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -890,7 +893,7 @@ func (bc *Blockchain) isValidBlockHeaderPoS(header *MsgDeSoHeader) error { // greater than its parent's timestamp. func (bc *Blockchain) isBlockTimestampValidRelativeToParentPoS(header *MsgDeSoHeader) error { // Validate that the timestamp is not less than its parent. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1050,7 +1053,7 @@ func (bc *Blockchain) hasValidBlockHeightPoS(header *MsgDeSoHeader) error { return RuleErrorPoSBlockBeforeCutoverHeight } // Validate that the block height is exactly one greater than its parent. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1065,7 +1068,7 @@ func (bc *Blockchain) hasValidBlockHeightPoS(header *MsgDeSoHeader) error { // hasValidBlockViewPoS validates the view for a given block header func (bc *Blockchain) hasValidBlockViewPoS(header *MsgDeSoHeader) error { // Validate that the view is greater than the latest uncommitted block. - parentBlockNode, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1094,7 +1097,7 @@ func (bc *Blockchain) hasValidBlockViewPoS(header *MsgDeSoHeader) error { func (bc *Blockchain) hasValidProposerRandomSeedSignaturePoS(header *MsgDeSoHeader) (bool, error) { // Validate that the leader proposed a valid random seed signature. - parentBlock, exists := bc.blockIndexByHash.Get(*header.PrevBlockHash) + parentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) if !exists { // Note: this should never happen as we only call this function after // we've validated that all ancestors exist in the block index. @@ -1319,11 +1322,13 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( return nil, nil, errors.New("getStoredLineageFromCommittedTip: No committed blocks found") } currentHash := header.PrevBlockHash.NewBlockHash() + currentHeight := header.Height - 1 ancestors := []*BlockNode{} prevHeight := header.Height prevView := header.GetView() for { - currentBlock, exists := bc.blockIndexByHash.Get(*currentHash) + // TODO: is currentHeight correct here? + currentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) if !exists { return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } @@ -1366,8 +1371,8 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (* if err != nil { return nil, errors.Wrapf(err, "getOrCreateBlockNodeFromBlockIndex: Problem hashing block %v", block) } - blockNode, _ := bc.blockIndexByHash.Get(*hash) - prevBlockNode, _ := bc.blockIndexByHash.Get(*block.Header.PrevBlockHash) + blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) if blockNode != nil { // If the block node already exists, we should set its parent if it doesn't have one already. if blockNode.Parent == nil { @@ -1676,7 +1681,7 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { } } for ii := 0; ii < len(uncommittedAncestors); ii++ { - if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, verifySignatures); err != nil { + if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, uint64(uncommittedAncestors[ii].Height), verifySignatures); err != nil { return errors.Wrapf(err, "runCommitRuleOnBestChain: Problem committing block %v", uncommittedAncestors[ii].Hash.String()) } @@ -1708,7 +1713,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // commitBlockPoS commits the block with the given hash. Specifically, this updates the // BlockStatus to include StatusBlockCommitted and flushes the view after connecting the block // to the DB and updates relevant badger indexes with info about the block. -func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, verifySignatures bool) error { +func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. blockNode, exists := bc.bestChainMap[*blockHash] if !exists { @@ -1720,7 +1725,7 @@ func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, verifySignatures bool return errors.Errorf("commitBlockPoS: Block %v is already committed", blockHash.String()) } // Connect a view up to block we are committing. - utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash) + utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) if err != nil { return errors.Wrapf(err, "commitBlockPoS: Problem initializing UtxoView: ") } @@ -1855,9 +1860,9 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er if currentParentHash == nil { return nil, errors.Errorf("GetUncommittedBlocks: Block %v has nil PrevBlockHash", currentBlock.Hash) } - currentBlock, _ = bc.blockIndexByHash.Get(*currentParentHash) + currentBlock, _ = bc.blockIndex.GetBlockNodeByHashAndHeight(currentParentHash, currentBlock.Header.Height-1) if currentBlock == nil { - return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in block index", currentBlock.Hash) + return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in block index", currentParentHash) } } return collections.Reverse(uncommittedBlockNodes), nil @@ -1892,7 +1897,8 @@ func (viewAndUtxoOps *BlockViewAndUtxoOps) Copy() *BlockViewAndUtxoOps { // GetUncommittedTipView builds a UtxoView to the uncommitted tip. func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { // Connect the uncommitted blocks to the tip so that we can validate subsequent blocks - blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*bc.BlockTip().Hash) + blockTip := bc.BlockTip() + blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedTipView: Problem getting UtxoView at block hash") } @@ -1917,12 +1923,12 @@ func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockV // all uncommitted ancestors of this block. Then it checks the block view cache to see if we have already // computed this view. If not, connecting the uncommitted ancestor blocks and saving to the cache. The // returned UtxoOps and FullBlock should NOT be modified. -func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash) ( +func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( *BlockViewAndUtxoOps, error) { // Always fetch the lineage from the committed tip to the block provided first to // ensure that a valid UtxoView is returned. uncommittedAncestors := []*BlockNode{} - currentBlock, _ := bc.blockIndexByHash.Get(blockHash) + currentBlock, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(&blockHash, blockHeight) if currentBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", blockHash) } @@ -1945,7 +1951,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash) ( if currentParentHash == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v has nil PrevBlockHash", currentBlock.Hash) } - currentBlock, _ = bc.blockIndexByHash.Get(*currentParentHash) + currentBlock, _ = bc.blockIndex.GetBlockNodeByHashAndHeight(currentParentHash, currentBlock.Header.Height-1) if currentBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", currentParentHash) } @@ -2108,8 +2114,8 @@ func (bc *Blockchain) GetProofOfStakeGenesisQuorumCertificate() (*QuorumCertific func (bc *Blockchain) GetFinalCommittedPoWBlock() (*BlockNode, error) { // Fetch the block node for the cutover block - blockNodes, blockNodesExist := bc.blockIndexByHeight[bc.params.GetFinalPoWBlockHeight()] - if !blockNodesExist { + blockNodes := bc.blockIndex.GetBlockNodesByHeight(bc.params.GetFinalPoWBlockHeight()) + if len(blockNodes) == 0 { return nil, errors.Errorf("Error fetching cutover block nodes before height %d", bc.params.GetFinalPoWBlockHeight()) } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 52ea4d696..2bd140003 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) bc.bestChain = []*BlockNode{genesisBlock} - bc.blockIndexByHash.Set(*genesisBlock.Hash, genesisBlock) + bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) randomBLSPrivateKey := _generateRandomBLSPrivateKey(t) @@ -301,7 +301,7 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndexByHash = collections.NewConcurrentMap[BlockHash, *BlockNode]() + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, nil) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -329,7 +329,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -377,17 +377,16 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { newHash, err := block.Hash() require.NoError(t, err) // Check the block index by hash - blockNodeFromIndex, exists := bc.blockIndexByHash.Get(*newHash) + blockNodeFromIndex, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uint64(blockNode.Height)) require.True(t, exists) require.True(t, blockNodeFromIndex.Hash.IsEqual(blockNode.Hash)) require.Equal(t, blockNodeFromIndex.Height, uint32(2)) require.True(t, blockNodeFromIndex.IsStored()) require.False(t, blockNodeFromIndex.IsValidated()) // Check the block index by height - byHeightBlockNodes, exists := bc.blockIndexByHeight[2] - require.True(t, exists) + byHeightBlockNodes := bc.blockIndex.GetBlockNodesByHeight(2) require.Len(t, byHeightBlockNodes, 1) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) // Check the DB for the block @@ -401,18 +400,17 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { // Okay now we update the status of the block to include validated. blockNode, err = bc.storeValidatedBlockInBlockIndex(block) require.NoError(t, err) - blockNodeFromIndex, exists = bc.blockIndexByHash.Get(*newHash) + blockNodeFromIndex, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uncommittedBlock.Header.Height) require.True(t, exists) require.True(t, blockNodeFromIndex.Hash.IsEqual(blockNode.Hash)) require.Equal(t, blockNodeFromIndex.Height, uint32(2)) require.True(t, blockNodeFromIndex.IsStored()) require.True(t, blockNodeFromIndex.IsValidated()) // Check the block index by height. - byHeightBlockNodes, exists = bc.blockIndexByHeight[2] - require.True(t, exists) + byHeightBlockNodes = bc.blockIndex.GetBlockNodesByHeight(2) require.Len(t, byHeightBlockNodes, 1) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) - require.True(t, byHeightBlockNodes[*newHash].IsValidated()) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].IsValidated()) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) @@ -428,18 +426,18 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { blockNode, err = bc.storeBlockInBlockIndex(block) require.NoError(t, err) // Make sure the blockIndexByHash is correct. - updatedBlockNode, exists := bc.blockIndexByHash.Get(*updatedBlockHash) + updatedBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(updatedBlockHash, uint64(blockNode.Height)) require.True(t, exists) require.True(t, updatedBlockNode.Hash.IsEqual(updatedBlockHash)) require.Equal(t, updatedBlockNode.Height, uint32(2)) require.True(t, updatedBlockNode.IsStored()) require.False(t, updatedBlockNode.IsValidated()) // Make sure the blockIndexByHeight is correct - byHeightBlockNodes, exists = bc.blockIndexByHeight[2] + byHeightBlockNodes = bc.blockIndex.GetBlockNodesByHeight(2) require.True(t, exists) require.Len(t, byHeightBlockNodes, 2) - require.True(t, byHeightBlockNodes[*newHash].Hash.IsEqual(newHash)) - require.True(t, byHeightBlockNodes[*updatedBlockHash].Hash.IsEqual(updatedBlockHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash) || byHeightBlockNodes[1].Hash.IsEqual(newHash)) + require.True(t, byHeightBlockNodes[0].Hash.IsEqual(updatedBlockHash) || byHeightBlockNodes[1].Hash.IsEqual(updatedBlockHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 2) @@ -477,7 +475,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { genesisNode, block2, } - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -810,7 +808,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain = []*BlockNode{genesisNode} - bc.blockIndexByHash = collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ @@ -847,7 +845,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain = append(bc.bestChain, block2) - bc.blockIndexByHash.Set(*hash2, block2) + bc.blockIndex.blockIndexByHash.Put(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) require.Equal(t, err, RuleErrorDoesNotExtendCommittedTip) @@ -1319,9 +1317,9 @@ func TestTryApplyNewTip(t *testing.T) { bc.addTipBlockToBestChain(bn1) bc.addTipBlockToBestChain(bn2) bc.addTipBlockToBestChain(bn3) - bc.blockIndexByHash.Set(*hash1, bn1) - bc.blockIndexByHash.Set(*hash2, bn2) - bc.blockIndexByHash.Set(*hash3, bn3) + bc.blockIndex.blockIndexByHash.Put(*hash1, bn1) + bc.blockIndex.blockIndexByHash.Put(*hash2, bn2) + bc.blockIndex.blockIndexByHash.Put(*hash3, bn3) // Simple reorg. Just replacing the uncommitted tip. newBlock := &MsgDeSoBlock{ @@ -1403,8 +1401,8 @@ func TestTryApplyNewTip(t *testing.T) { Height: 6, }, } - bc.blockIndexByHash.Set(*hash4, bn4) - bc.blockIndexByHash.Set(*hash5, bn5) + bc.blockIndex.blockIndexByHash.Put(*hash4, bn4) + bc.blockIndex.blockIndexByHash.Put(*hash5, bn5) // Set new block's parent to hash5 newBlockNode.Header.PrevBlockHash = hash5 @@ -1839,7 +1837,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { futureBlockHash, err = futureBlock.Hash() require.NoError(t, err) - futureBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*futureBlockHash) + futureBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(futureBlockHash, futureBlock.Header.Height) require.True(t, exists) require.False(t, futureBlockNode.IsCommitted()) require.True(t, futureBlockNode.IsStored()) @@ -1848,10 +1846,12 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { } var timeoutBlockHash *BlockHash + var timeoutBlockHeight uint64 { // Okay let's timeout view 15 var timeoutBlock *MsgDeSoBlock timeoutBlock = _generateRealBlock(testMeta, 15, 16, 381, blockHash3, true) + timeoutBlockHeight = timeoutBlock.Header.Height success, _, _, err := testMeta.chain.ProcessBlockPoS(timeoutBlock, 15, true) fmt.Println(err) require.True(t, success) @@ -1877,7 +1877,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { _, exists := testMeta.chain.bestChainMap[*timeoutBlockHash] require.False(t, exists) - timeoutBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*timeoutBlockHash) + timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) require.True(t, exists) require.False(t, timeoutBlockNode.IsCommitted()) } @@ -1905,7 +1905,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.Len(t, missingBlockHashes, 1) require.True(t, missingBlockHashes[0].IsEqual(dummyParentBlockHash)) require.NoError(t, err) - orphanBlockInIndex, orphanBlockExists := testMeta.chain.blockIndexByHash.Get(*orphanBlockHash) + orphanBlockInIndex, orphanBlockExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(orphanBlockHash, orphanBlock.Header.Height) require.True(t, orphanBlockExists) require.NotNil(t, orphanBlockInIndex) require.True(t, orphanBlockInIndex.IsStored()) @@ -1918,7 +1918,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.Len(t, missingBlockHashes, 0) require.NoError(t, err) - orphanBlockInIndex, orphanBlockExists = testMeta.chain.blockIndexByHash.Get(*orphanBlockHash) + orphanBlockInIndex, orphanBlockExists = testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(orphanBlockHash, orphanBlock.Header.Height) require.True(t, orphanBlockExists) require.NotNil(t, orphanBlockInIndex) require.True(t, orphanBlockInIndex.IsStored()) @@ -1944,7 +1944,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { require.True(t, missingBlockHashes[0].IsEqual(randomHash)) require.NoError(t, err) - malformedOrphanBlockInIndex, malformedOrphanBlockExists := testMeta.chain.blockIndexByHash.Get(*malformedOrphanBlockHash) + malformedOrphanBlockInIndex, malformedOrphanBlockExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(malformedOrphanBlockHash, malformedOrphanBlock.Header.Height) require.True(t, malformedOrphanBlockExists) require.True(t, malformedOrphanBlockInIndex.IsValidateFailed()) require.True(t, malformedOrphanBlockInIndex.IsStored()) @@ -2067,7 +2067,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2088,7 +2088,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.True(t, blockNode.IsValidateFailed()) @@ -2115,7 +2115,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.False(t, exists) } @@ -2167,7 +2167,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := realBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, realBlock.Header.Height) require.False(t, exists) } { @@ -2185,7 +2185,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2210,7 +2210,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.False(t, exists) } { @@ -2261,7 +2261,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // Get the block node from the block index. blockHash, err := nextEpochBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, nextEpochBlock.Header.Height) require.False(t, exists) } { @@ -2290,7 +2290,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // The block shouldn't be in the block index. blockHash, err := twoEpochsInFutureBlock.Hash() require.NoError(t, err) - _, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, twoEpochsInFutureBlock.Header.Height) require.False(t, exists) } { @@ -2307,7 +2307,7 @@ func TestProcessOrphanBlockPoS(t *testing.T) { // The block should be in the block index. blockHash, err := prevEpochBlock.Hash() require.NoError(t, err) - blockNode, exists := testMeta.chain.blockIndexByHash.Get(*blockHash) + blockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockHash, prevEpochBlock.Header.Height) require.True(t, exists) require.True(t, blockNode.IsStored()) require.False(t, blockNode.IsValidateFailed()) @@ -2375,7 +2375,7 @@ func TestHasValidProposerRandomSeedSignaturePoS(t *testing.T) { require.NoError(t, err) realBlockHash, err := realBlock.Hash() require.NoError(t, err) - realBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*realBlockHash) + realBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(realBlockHash, realBlock.Header.Height) require.True(t, exists) require.True(t, realBlockNode.IsStored()) require.False(t, realBlockNode.IsValidateFailed()) @@ -2461,13 +2461,13 @@ func _generateRealBlockWithFailingTxn(testMeta *TestMeta, blockHeight uint64, vi } // TODO: Get real seed signature. - prevBlock, exists := testMeta.chain.blockIndexByHash.Get(*prevBlockHash) + prevBlock, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, exists) // Always update the testMeta latestBlockView - latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash) + latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) require.NoError(testMeta.t, err) latestBlockView := latestBlockViewAndUtxoOps.UtxoView - latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndexByHash.Get(*prevBlockHash) + latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, latestBlockNodeExists) latestBlockHeight := latestBlockNode.Height testMeta.posMempool.UpdateLatestBlock(latestBlockView, uint64(latestBlockHeight)) @@ -2524,7 +2524,7 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) - _, exists := testMeta.chain.blockIndexByHash.Get(*newBlockHash) + _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) require.True(testMeta.t, exists) // Remove the transactions from this block from the mempool. // This prevents nonce reuse issues when trying to make failing blocks. @@ -2547,7 +2547,7 @@ func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, vie require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) require.True(testMeta.t, blockNode.IsValidated()) - newBlockNode, exists := testMeta.chain.blockIndexByHash.Get(*newBlockHash) + newBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) require.True(testMeta.t, exists) testMeta.chain.addTipBlockToBestChain(newBlockNode) // Update the latest block view @@ -2678,7 +2678,7 @@ func _getFullRealBlockTemplate( // Get leader voting private key. leaderVotingPrivateKey := testMeta.pubKeyToBLSKeyMap[leaderPublicKey] // Get hash of last block - chainTip, _ := testMeta.chain.blockIndexByHash.Get(*blockTemplate.Header.PrevBlockHash) + chainTip, _ := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(blockTemplate.Header.PrevBlockHash, blockTemplate.Header.Height-1) chainTipHash := chainTip.Hash // Get the vote signature payload // Hack to get view numbers working properly w/ PoW blocks. diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index d29421a13..a33983240 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -250,7 +250,10 @@ func (fc *FastHotStuffConsensus) handleBlockProposalEvent( // Fetch the parent block parentBlockHash := BlockHashFromConsensusInterface(event.QC.GetBlockHash()) - parentBlock, parentBlockExists := fc.blockchain.blockIndexByHash.Get(*parentBlockHash) + parentBlock, parentBlockExists, err := fc.blockchain.blockIndex.GetBlockNodeByHashOnly(parentBlockHash) + if err != nil { + return errors.Errorf("Error fetching parent block: %v", parentBlockHash) + } if !parentBlockExists { return errors.Errorf("Error fetching parent block: %v", parentBlockHash) } @@ -487,7 +490,8 @@ func (fc *FastHotStuffConsensus) HandleLocalTimeoutEvent(event *consensus.FastHo tipBlockHash := BlockHashFromConsensusInterface(event.TipBlockHash) // Fetch the HighQC from the Blockchain struct - tipBlockNode, tipBlockExists := fc.blockchain.blockIndexByHash.Get(*tipBlockHash) + // TODO: validate that TipHeight is a uint32 + tipBlockNode, tipBlockExists := fc.blockchain.blockIndex.GetBlockNodeByHashAndHeight(tipBlockHash, event.TipBlockHeight) if !tipBlockExists { return errors.Errorf("FastHotStuffConsensus.HandleLocalTimeoutEvent: Error fetching tip block: %v", tipBlockHash) } @@ -558,13 +562,17 @@ func (fc *FastHotStuffConsensus) HandleValidatorTimeout(pp *Peer, msg *MsgDeSoVa // If we don't have the highQC's block on hand, then we need to request it from the peer. We do // that first before storing the timeout message locally in the FastHotStuffEventLoop. This // prevents spamming of timeout messages by peers. - if !fc.blockchain.HasBlockInBlockIndex(msg.HighQC.BlockHash) { - err := errors.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Missing highQC's block: %v", msg.HighQC.BlockHash) + hasBlockInBlockIndex, err := fc.blockchain.HasBlockInBlockIndex(msg.HighQC.BlockHash) + if err != nil { + return nil, errors.Wrapf(err, "FastHotStuffConsensus.HandleValidatorTimeout: Error fetching block: ") + } + if !hasBlockInBlockIndex { + err = errors.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Missing highQC's block: %v", msg.HighQC.BlockHash) return []*BlockHash{msg.HighQC.BlockHash}, err } // Process the timeout message locally in the FastHotStuffEventLoop - if err := fc.fastHotStuffEventLoop.ProcessValidatorTimeout(msg); err != nil { + if err = fc.fastHotStuffEventLoop.ProcessValidatorTimeout(msg); err != nil { // If we can't process the timeout locally, then it must somehow be malformed, stale, // or a duplicate vote/timeout for the same view. glog.Errorf("FastHotStuffConsensus.HandleValidatorTimeout: Error processing timeout msg: %v", err) @@ -693,7 +701,7 @@ func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip(block *MsgDeSoBlock) ([ return nil, errors.Errorf("Error hashing tip block: %v", err) } - utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash) + utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) if err != nil { return nil, errors.Errorf("Error fetching UtxoView for tip block: %v", err) } @@ -733,13 +741,16 @@ func (fc *FastHotStuffConsensus) produceUnsignedBlockForBlockProposalEvent( parentBlockHash := BlockHashFromConsensusInterface(event.QC.GetBlockHash()) // Fetch the parent block - parentBlock, parentBlockExists := fc.blockchain.blockIndexByHash.Get(*parentBlockHash) + parentBlock, parentBlockExists, err := fc.blockchain.blockIndex.GetBlockNodeByHashOnly(parentBlockHash) + if err != nil { + return nil, errors.Errorf("Error fetching parent block: %v", parentBlockHash) + } if !parentBlockExists { return nil, errors.Errorf("Error fetching parent block: %v", parentBlockHash) } // Build a UtxoView at the parent block - parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash) + parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) if err != nil { // This should never happen as long as the parent block is a descendant of the committed tip. return nil, errors.Errorf("Error fetching UtxoView for parent block: %v", parentBlockHash) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index ac9ea0ed9..900afb71a 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -112,9 +112,11 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { params: &DeSoTestnetParams, blockchain: &Blockchain{ ChainLock: deadlock.RWMutex{}, - blockIndexByHash: collections.NewConcurrentMapFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, - }), + blockIndex: &BlockIndex{ + blockIndexByHash: newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + *blockHash: {Header: blockHeader}, + }), + }, params: &DeSoTestnetParams, }, fastHotStuffEventLoop: &consensus.MockFastHotStuffEventLoop{ diff --git a/lib/postgres.go b/lib/postgres.go index 4b43e47c0..0c5991756 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/deso-protocol/core/collections" + lru2 "github.com/decred/dcrd/container/lru" "net/url" "regexp" "strings" @@ -1305,16 +1305,16 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru2.Map[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap := collections.NewConcurrentMap[BlockHash, *BlockNode]() + blockMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { - blockMap.Set(*block.Hash, &BlockNode{ + blockMap.Put(*block.Hash, &BlockNode{ Hash: block.Hash, Height: uint32(block.Height), DifficultyTarget: block.DifficultyTarget, @@ -1333,7 +1333,8 @@ func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, } // Setup parent pointers - blockMap.Iterate(func(key BlockHash, blockNode *BlockNode) { + for _, key := range blockMap.Keys() { + blockNode, _ := blockMap.Get(key) // Genesis block has nil parent parentHash := blockNode.Header.PrevBlockHash if parentHash != nil { @@ -1343,7 +1344,7 @@ func (postgres *Postgres) GetBlockIndex() (*collections.ConcurrentMap[BlockHash, } blockNode.Parent = parent } - }) + } return blockMap, nil } diff --git a/lib/server.go b/lib/server.go index b8c3bb5c5..21756a665 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1086,7 +1086,13 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // have this issue. Hitting duplicates after we're done syncing is // fine and can happen in certain cases. headerHash, _ := headerReceived.Hash() - if srv.blockchain.HasHeader(headerHash) { + hasHeader, err := srv.blockchain.HasHeader(headerHash) + // TODO: what should we really do here? + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", headerHash, err) + return + } + if hasHeader { if srv.blockchain.isSyncing() { glog.Warningf("Server._handleHeaderBundle: Duplicate header %v received from peer %v "+ @@ -1309,7 +1315,12 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // we're either not aware of or that we don't think is the best chain. // Doing things this way makes it so that when we request blocks we // are 100% positive the peer has them. - if !srv.blockchain.HasHeader(msg.TipHash) { + hasHeader, err := srv.blockchain.HasHeader(msg.TipHash) + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", msg.TipHash, err) + return + } + if !hasHeader { glog.V(1).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ "blockchain so not requesting anything else from them. Our block "+ "tip %v, their tip %v:%d, peer: %v", diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 7498690b7..e825dfbb6 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -847,7 +847,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser // TODO: Have Z look at if we need to do some caching in the uncommitted blocks logic. // First connect the uncommitted blocks to the mempool view. for _, uncommittedBlock := range uncommittedBlocks { - utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash) + utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 9f8e15f47..5fe9cb515 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -225,7 +225,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // The only thing we can really do in this case is rebuild the entire index // from scratch. To do that, we return all the blocks in the index to detach // and all the blocks in the real chain to attach. - txindexTipNode, _ := txi.TXIndexChain.blockIndexByHash.Get(*txindexTipHash.Hash) + txindexTipNode, _ := txi.TXIndexChain.blockIndex.GetBlockNodeByHashAndHeight(txindexTipHash.Hash, uint64(txindexTipHash.Height)) // Get the committed tip. committedTip, _ := txi.CoreChain.GetCommittedTip() @@ -368,13 +368,13 @@ func (txi *TXIndex) Update() error { // Delete this block from the chain db so we don't get duplicate block errors. // Remove this block from our bestChain data structures. - newBlockIndexByHash, newBlockIndexByHeight := txi.TXIndexChain.CopyBlockIndexes() + newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] delete(newBestChainMap, *(blockToDetach.Hash)) - newBlockIndexByHash.Remove(*(blockToDetach.Hash)) + newBlockIndex.Delete(*(blockToDetach.Hash)) - txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndexByHash, newBlockIndexByHeight) + txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) // At this point the entries for the block should have been removed // from both our Txindex chain and our transaction index mappings. @@ -408,7 +408,7 @@ func (txi *TXIndex) Update() error { utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps - utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash) + utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { return fmt.Errorf("Update: Problem getting UtxoView at block hash %v: %v", blockToAttach.Header.PrevBlockHash, err) From f3e8e4c233e83ca9bb72aa0860a1c8845faa3c03 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 15:01:51 -0400 Subject: [PATCH 002/131] best chain and header chain with read through cache --- lib/block_view_lockups_test.go | 4 +- lib/block_view_test.go | 12 +- lib/blockchain.go | 368 ++++++++++++++++++++++----------- lib/db_utils.go | 87 +++++++- lib/pos_blockchain.go | 48 +++-- lib/pos_blockchain_test.go | 54 ++--- lib/server.go | 58 ++++-- lib/state_change_syncer.go | 6 +- lib/txindex.go | 4 +- 9 files changed, 443 insertions(+), 198 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 6d25cb1fa..faaa6a3e3 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2462,7 +2462,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) // Update the tip - testMeta.chain.bestChain = testMeta.chain.bestChain[:len(testMeta.chain.bestChain)-1] + testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] // Validate the state update utxoView = NewUtxoView( @@ -2517,7 +2517,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.bestChain = testMeta.chain.bestChain[:len(testMeta.chain.bestChain)-1] + testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 35f019271..2717aeb50 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -702,8 +702,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te require.NoError(err) // sanity-check that the last block hash is the same as the last header hash. require.Equal(true, bytes.Equal( - tm.chain.bestChain[len(tm.chain.bestChain)-1].Hash.ToBytes(), - tm.chain.bestHeaderChain[len(tm.chain.bestHeaderChain)-1].Hash.ToBytes())) + tm.chain.bestChain.GetTip().Hash.ToBytes(), + tm.chain.bestHeaderChain.GetTip().Hash.ToBytes())) // Last block shouldn't be nil, and the number of expectedTxns should be the same as in the testVectorBlock + 1, // because of the additional block reward. require.NotNil(lastBlock) @@ -791,10 +791,10 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.bestChain = tm.chain.bestChain[:len(tm.chain.bestChain)-1] - tm.chain.bestHeaderChain = tm.chain.bestHeaderChain[:len(tm.chain.bestHeaderChain)-1] - delete(tm.chain.bestChainMap, *lastBlockHash) - delete(tm.chain.bestHeaderChainMap, *lastBlockHash) + tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] + tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] + tm.chain.bestChain.ChainMap.Delete(*lastBlockHash) + tm.chain.bestHeaderChain.ChainMap.Delete(*lastBlockHash) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index a03b55e39..f1e35d305 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -536,12 +536,11 @@ type BlockIndex struct { tipNode *BlockNode } -func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { +func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { return &BlockIndex{ db: db, snapshot: snapshot, blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? - tipNode: tipNode, } } @@ -566,18 +565,14 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, if exists { return val, true, nil } - // NOTE: this can be sped up significantly by mapping blockHash to blockNode or blockHash to blockHeight - // in badger, so we can avoid deserializing the full block. - fullBlock, err := GetBlock(blockHash, bi.db, bi.snapshot) + height, err := GetHeightForHash(bi.db, bi.snapshot, blockHash) if err != nil { - return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem reading block from db") - } - nodeHeader := fullBlock.Header - fullBlockHash, err := fullBlock.Hash() - if err != nil { - return nil, false, errors.Wrapf(err, "_initChain: Problem hashing block") + if errors.Is(err, badger.ErrKeyNotFound) { + return nil, false, nil + } + return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(nodeHeader.Height), fullBlockHash, false) + blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil } @@ -622,20 +617,94 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { type BestChain struct { db *badger.DB + snapshot *Snapshot IsHeaderChain bool - Chain []*BlockNode - ChainMap map[BlockHash]*BlockNode - tipNode *BlockNode + Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. + ChainMap *lru2.Map[BlockHash, *BlockNode] } -func NewBestChain(db *badger.DB, isHeaderChain bool, tipNode *BlockNode) *BestChain { +func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool) *BestChain { return &BestChain{ db: db, + snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, - ChainMap: make(map[BlockHash]*BlockNode), - tipNode: tipNode, + ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + } +} + +func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { + bestChain.Chain = append(bestChain.Chain, tipNode) + bestChain.ChainMap.Put(*tipNode.Hash, tipNode) +} + +func (bestChain *BestChain) GetTip() *BlockNode { + if len(bestChain.Chain) == 0 { + return nil + } + return bestChain.Chain[len(bestChain.Chain)-1] +} + +func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { + if height > math.MaxUint32 { + return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) + } + + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) + _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) + if len(valsFound) == 0 { + return nil, false, nil + } + for _, val := range valsFound { + blockNode, err := DeserializeBlockNode(val) + if err != nil { + glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) + continue + } + if blockNode.IsCommitted() { + return blockNode, true, nil + } } + // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the + // best chain. + return nil, false, nil +} + +func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + val, exists := bestChain.ChainMap.Get(*blockHash) + if exists { + return val, true + } + if height > math.MaxUint32 { + glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) + } + bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false + } + bestChain.ChainMap.Put(*blockHash, bn) + return bn, true +} + +func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { + val, exists := bestChain.ChainMap.Get(*blockHash) + if exists { + return val, true, nil + } + + height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) + if err != nil { + if errors.Is(err, badger.ErrKeyNotFound) { + return nil, false, nil + } + return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") + } + bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) + if bn == nil { + return nil, false, nil + } + bestChain.ChainMap.Put(*blockHash, bn) + return bn, true, nil } type Blockchain struct { @@ -673,15 +742,15 @@ type Blockchain struct { //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode // An in-memory slice of the blocks on the main chain only. The end of // this slice is the best known tip that we have at any given time. - bestChain []*BlockNode - bestChainMap map[BlockHash]*BlockNode - - bestHeaderChain []*BlockNode - bestHeaderChainMap map[BlockHash]*BlockNode + //bestChain []*BlockNode + //bestChainMap map[BlockHash]*BlockNode + // + //bestHeaderChain []*BlockNode + //bestHeaderChainMap map[BlockHash]*BlockNode - blockIndex *BlockIndex - //bestChain *BestChain - //bestHeaderChain *BestChain + blockIndex *BlockIndex + bestChain *BestChain + bestHeaderChain *BestChain // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep @@ -849,23 +918,25 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, map[BlockHash]*BlockNode) { +func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := make(map[BlockHash]*BlockNode) - newBestChain = append(newBestChain, bc.bestChain...) - for kk, vv := range bc.bestChainMap { - newBestChainMap[kk] = vv + newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChain = append(newBestChain, bc.bestChain.Chain...) + for _, key := range bc.bestChain.ChainMap.Keys() { + val, _ := bc.bestChain.ChainMap.Get(key) + newBestChainMap.Put(key, val) } return newBestChain, newBestChainMap } -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, map[BlockHash]*BlockNode) { +func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := make(map[BlockHash]*BlockNode) - newBestChain = append(newBestChain, bc.bestHeaderChain...) - for kk, vv := range bc.bestHeaderChainMap { - newBestChainMap[kk] = vv + newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) + for _, key := range bc.bestHeaderChain.ChainMap.Keys() { + val, _ := bc.bestHeaderChain.ChainMap.Get(key) + newBestChainMap.Put(key, val) } return newBestChain, newBestChainMap @@ -873,8 +944,9 @@ func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, map[BlockHash]*BlockN // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { + // TODO: figure out how to iterate over best chain w/o having entire thing in memory. if bc.ChainState() == SyncStateFullyCurrent { - for _, blockNode := range bc.bestChain { + for _, blockNode := range bc.bestChain.Chain { if !blockNode.Status.IsFullyProcessed() { return false } @@ -971,21 +1043,26 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } + // Walk back the last 100 hours of blocks. + currBlockCounter := 1 + for currBlockCounter < 3600*100 && tipNode.Header.PrevBlockHash != nil { + bc.blockIndex.GetBlockNodeByHashAndHeight(tipNode.Header.PrevBlockHash, tipNode.Header.Height-1) + currBlockCounter++ + } } } - bc.blockIndex.setTipNode(tipNode) // At this point the blockIndexByHash should contain a full node tree with all // nodes pointing to valid parent nodes. { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestChain, err = GetBestChain(tipNode) + bc.bestChain.Chain, err = GetBestChain(tipNode) if err != nil { return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") } - for _, bestChainNode := range bc.bestChain { - bc.bestChainMap[*bestChainNode.Hash] = bestChainNode + for _, bestChainNode := range bc.bestChain.Chain { + bc.bestChain.ChainMap.Put(*bestChainNode.Hash, bestChainNode) } } @@ -993,12 +1070,12 @@ func (bc *Blockchain) _initChain() error { { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestHeaderChain, err = GetBestChain(tipNode) + bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) if err != nil { return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") } - for _, bestHeaderChainNode := range bc.bestHeaderChain { - bc.bestHeaderChainMap[*bestHeaderChainNode.Hash] = bestHeaderChainNode + for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { + bc.bestHeaderChain.ChainMap.Put(*bestHeaderChainNode.Hash, bestHeaderChainNode) } } @@ -1050,9 +1127,9 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, - bc.bestHeaderChainMap, + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, + bc.bestHeaderChain.ChainMap, blocksToDetach, blocksToAttach, ) @@ -1104,15 +1181,15 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot, nil), + blockIndex: NewBlockIndex(db, snapshot), //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - bestChainMap: make(map[BlockHash]*BlockNode), - //bestChain: NewBestChain(db, false), - bestHeaderChainMap: make(map[BlockHash]*BlockNode), - //bestHeaderChain: NewBestChain(db, true), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize - snapshotCache: NewSnapshotCache(), + //bestChainMap: make(map[BlockHash]*BlockNode), + bestChain: NewBestChain(db, snapshot, false), + //bestHeaderChainMap: make(map[BlockHash]*BlockNode), + bestHeaderChain: NewBestChain(db, snapshot, true), + blockViewCache: lru.NewKVCache(100), // TODO: parameterize + snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1178,8 +1255,7 @@ func fastLog2Floor(n uint32) uint8 { // // This function MUST be called with the chain state lock held (for reads). func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *BlockIndex, bestChainList []*BlockNode, - bestChainMap map[BlockHash]*BlockNode) (*BlockNode, uint32) { + blockIndex *BlockIndex, bestChain *BestChain) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. @@ -1198,9 +1274,22 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Find the most recent locator block hash in the main chain. In the // case none of the hashes in the locator are in the main chain, fall // back to the genesis block. - startNode := bestChainList[0] + startNode, startNodeExists, err := bestChain.GetBlockByHeight(0) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by height: %v", err) + return nil, 0 + } + if !startNodeExists { + glog.Errorf("locateInventory: Genesis block not found") + return nil, 0 + } for _, hash := range locator { - node, bestChainContainsNode := bestChainMap[*hash] + // TODO: replace w/ read-through cache call. + node, bestChainContainsNode, err := bestChain.GetBlockByHash(hash) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by hash: %v", err) + continue + } if bestChainContainsNode { startNode = node break @@ -1211,18 +1300,25 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // is no next block it means the most recently known block is the tip of // the best chain, so there is nothing more to do. nextNodeHeight := uint32(startNode.Header.Height) + 1 - if uint32(len(bestChainList)) <= nextNodeHeight { + startNode, startNodeExists, err = bestChain.GetBlockByHeight(uint64(nextNodeHeight)) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by height: %v", err) + return nil, 0 + } + if !startNodeExists { return nil, 0 } - startNode = bestChainList[nextNodeHeight] // Calculate how many entries are needed. - tip := bestChainList[len(bestChainList)-1] - total := uint32((tip.Header.Height - startNode.Header.Height) + 1) + total := (bestChain.GetTip().Height - startNode.Height) + 1 if stopNodeError != nil && stopNodeExists && stopNode != nil && stopNode.Header.Height >= startNode.Header.Height { - _, bestChainContainsStopNode := bestChainMap[*stopNode.Hash] + _, bestChainContainsStopNode, err := bestChain.GetBlockByHash(stopNode.Hash) + if err != nil { + glog.Errorf("locateInventory: Problem getting block by hash: %v", err) + return nil, 0 + } if bestChainContainsStopNode { total = uint32((stopNode.Header.Height - startNode.Header.Height) + 1) } @@ -1242,14 +1338,13 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // // This function MUST be called with the ChainLock held (for reads). func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *BlockIndex, bestChainList []*BlockNode, - bestChainMap map[BlockHash]*BlockNode) []*MsgDeSoHeader { + blockIndex *BlockIndex, bestChain *BestChain) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the // total number of nodes after it needed while respecting the stop hash // and max entries. node, total := locateInventory(locator, stopHash, maxHeaders, - blockIndex, bestChainList, bestChainMap) + blockIndex, bestChain) if total == 0 { return nil } @@ -1264,7 +1359,15 @@ func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, if uint32(len(headers)) == total { break } - node = bestChainList[node.Header.Height+1] + var nodeExists bool + node, nodeExists, err = bestChain.GetBlockByHeight(node.Header.Height + 1) + if err != nil { + glog.Errorf("locateHeaders: Problem getting block by height: %v", err) + break + } + if !nodeExists { + break + } } return headers } @@ -1293,7 +1396,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndex, bc.bestChain, bc.bestChainMap) + bc.blockIndex, bc.bestChain) return headers } @@ -1355,8 +1458,17 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { // ancestors must be too, so use a much faster O(1) lookup in // that case. Otherwise, fall back to walking backwards through // the nodes of the other chain to the correct ancestor. - if _, exists := bc.bestHeaderChainMap[*tip.Hash]; exists { - tip = bc.bestHeaderChain[height] + _, exists, err := bc.bestHeaderChain.GetBlockByHash(tip.Hash) + if err != nil { + glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) + exists = false + } + if exists { + var innerExists bool + tip, innerExists = bc.bestHeaderChain.GetBlockByHashAndHeight(tip.Hash, uint64(height)) + if !innerExists { + glog.Errorf("LatestLocator: Block %v not found in best header chain", tip.Hash) + } } else { tip = tip.Ancestor(uint32(height)) } @@ -1405,7 +1517,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChainMap[*bestBlockTip.Hash] + headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(bestBlockTip.Hash, uint64(bestBlockTip.Height)) if !blockTipExistsInBestHeaderChain { // If the hash of the tip of the best blockchain is not in the best header chain, then // this is a case where the header chain has forked off from the best block @@ -1424,7 +1536,17 @@ func (bc *Blockchain) GetBlockNodesToFetch( // an error and set it to the genesis block. glog.Errorf("GetBlockToFetch: headerNode was nil after iterating " + "backward through best header chain; using genesis block") - headerNodeStart = bc.bestHeaderChain[0] + var err error + var genesisBlockExists bool + headerNodeStart, genesisBlockExists, err = bc.bestHeaderChain.GetBlockByHeight(0) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting genesis block: %v", err) + return nil + } + if !genesisBlockExists { + glog.Errorf("GetBlockToFetch: Genesis block not found") + return nil + } } } @@ -1434,14 +1556,22 @@ func (bc *Blockchain) GetBlockNodesToFetch( currentHeight := headerNodeStart.Height + 1 blockNodesToFetch := []*BlockNode{} heightLimit := maxHeight - if heightLimit >= uint32(len(bc.bestHeaderChain)) { - heightLimit = uint32(len(bc.bestHeaderChain) - 1) + if heightLimit >= bc.bestHeaderChain.GetTip().Height { + heightLimit = bc.bestHeaderChain.GetTip().Height - 1 } for currentHeight <= heightLimit && len(blockNodesToFetch) < numBlocks { // Get the current hash and increment the height. - currentNode := bc.bestHeaderChain[currentHeight] + currentNode, currentNodeExists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(currentHeight)) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) + return nil + } + if !currentNodeExists { + glog.Errorf("GetBlockToFetch: Block at height %d not found", currentHeight) + return nil + } currentHeight++ if _, exists := blocksToIgnore[*currentNode.Hash]; exists { @@ -1460,12 +1590,12 @@ func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } -func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) *BlockNode { - if blockHeight >= uint32(len(bc.bestHeaderChain)) { - return nil +// TODO: delete me? +func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { + if blockHeight >= bc.bestChain.GetTip().Height { + return nil, false, nil } - - return bc.bestHeaderChain[blockHeight] + return bc.bestChain.GetBlockByHeight(uint64(blockHeight)) } func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { @@ -1487,14 +1617,14 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32) *MsgDeSoBlock { - numBlocks := uint32(len(bc.bestChain)) - - if height >= numBlocks { - return nil +func (bc *Blockchain) GetBlockAtHeight(height uint32) (*MsgDeSoBlock, error) { + bn, bnExists, err := bc.bestChain.GetBlockByHeight(uint64(height)) + if !bnExists || err != nil { + glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) + return nil, err } - return bc.GetBlock(bc.bestChain[height].Hash) + return bc.GetBlock(bn.Hash), nil } // GetBlockNodeWithHash looks for a block node in the bestChain list that matches the hash. @@ -1502,7 +1632,11 @@ func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { return nil } - return bc.bestChainMap[*hash] + bn, bnExists, err := bc.bestChain.GetBlockByHash(hash) + if !bnExists || err != nil { + return nil + } + return bn } // isTipMaxed compares the tip height to the MaxSyncBlockHeight height. @@ -1665,7 +1799,11 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight - for _, blockNode := range bc.bestChain { + // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, + // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? + // TODO: figure out how to iterate over best chain to checkArchivalMode + // when we only have a portion of best chain in memory. + for _, blockNode := range bc.bestChain.Chain { if uint64(blockNode.Height) > firstSnapshotHeight { return false } @@ -1715,13 +1853,7 @@ func (bc *Blockchain) isHyperSyncCondition() bool { // main chain for blocks, which is why separate functions are required for // each of them. func (bc *Blockchain) headerTip() *BlockNode { - if len(bc.bestHeaderChain) == 0 { - return nil - } - - // Note this should always work because we should have the genesis block - // in here. - return bc.bestHeaderChain[len(bc.bestHeaderChain)-1] + return bc.bestHeaderChain.GetTip() } func (bc *Blockchain) HeaderTip() *BlockNode { @@ -1753,15 +1885,7 @@ func (bc *Blockchain) Snapshot() *Snapshot { // invalidate and chop off the headers corresponding to those blocks and // their ancestors so the two generally stay in sync. func (bc *Blockchain) blockTip() *BlockNode { - var tip *BlockNode - - if len(bc.bestChain) == 0 { - return nil - } - - tip = bc.bestChain[len(bc.bestChain)-1] - - return tip + return bc.bestChain.GetTip() } func (bc *Blockchain) BlockTip() *BlockNode { @@ -1769,20 +1893,20 @@ func (bc *Blockchain) BlockTip() *BlockNode { } func (bc *Blockchain) BestChain() []*BlockNode { - return bc.bestChain + return bc.bestChain.Chain } func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { - bc.bestChain = bestChain + bc.bestChain.Chain = bestChain } func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, - bestChainMap map[BlockHash]*BlockNode, + bestChainMap *lru2.Map[BlockHash, *BlockNode], blockIndexByHash *lru2.Map[BlockHash, *BlockNode], ) { - bc.bestChain = bestChain - bc.bestChainMap = bestChainMap + bc.bestChain.Chain = bestChain + bc.bestChain.ChainMap = bestChainMap bc.blockIndex.blockIndexByHash = blockIndexByHash } @@ -2071,14 +2195,14 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockN return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap map[BlockHash]*BlockNode, detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap map[BlockHash]*BlockNode) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru2.Map[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 for blockOffset := 0; blockOffset < len(detachBlocks); blockOffset++ { blockIndex := tipIndex - blockOffset - delete(mainChainMap, *mainChainList[blockIndex].Hash) + mainChainMap.Delete(*mainChainList[blockIndex].Hash) } mainChainList = mainChainList[:len(mainChainList)-len(detachBlocks)] @@ -2087,7 +2211,7 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap map[BlockH // first, with the new tip at the end. for _, attachNode := range attachBlocks { mainChainList = append(mainChainList, attachNode) - mainChainMap[*attachNode.Hash] = attachNode + mainChainMap.Put(*attachNode.Hash, attachNode) } return mainChainList, mainChainMap @@ -2268,8 +2392,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B isMainChain = true _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, bc.bestHeaderChainMap, detachBlocks, attachBlocks) + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) // Note that we don't store the best header hash here and so this is an // in-memory-only adjustment. See the comment above on preventing attacks. @@ -2759,8 +2883,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now that we've set the best chain in the db, update our in-memory data // structure to reflect this. Do a quick check first to make sure it's consistent. - lastIndex := len(bc.bestChain) - 1 - bestChainHash := bc.bestChain[lastIndex].Hash + lastIndex := len(bc.bestChain.Chain) - 1 + bestChainHash := bc.bestChain.Chain[lastIndex].Hash if !bestChainHash.IsEqual(nodeToValidate.Header.PrevBlockHash) { return false, false, fmt.Errorf("ProcessBlock: Last block in bestChain "+ @@ -2771,13 +2895,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.bestChain = append(bc.bestChain, nodeToValidate) - bc.bestChainMap[*nodeToValidate.Hash] = nodeToValidate + bc.bestChain.PushNewTip(nodeToValidate) } else { newBestChain, newBestChainMap := bc.CopyBestChain() - newBestChain = append(newBestChain, nodeToValidate) - newBestChainMap[*nodeToValidate.Hash] = nodeToValidate - bc.bestChain, bc.bestChainMap = newBestChain, newBestChainMap + bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.bestChain.PushNewTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3041,7 +3163,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures newBestChain, newBestChainMap := bc.CopyBestChain() newBestChain, newBestChainMap = updateBestChainInMemory( newBestChain, newBestChainMap, detachBlocks, attachBlocks) - bc.bestChain, bc.bestChainMap = newBestChain, newBestChainMap + bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/db_utils.go b/lib/db_utils.go index 2bb61e0f7..ec9350f5a 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -604,7 +604,11 @@ type DBPrefixes struct { // When reading and writing data to this prefixes, please acquire the snapshotDbMutex in the snapshot. PrefixHypersyncSnapshotDBPrefix []byte `prefix_id:"[97]"` - // NEXT_TAG: 98 + // PrefixHashToHeight is used to store the height of a block given its hash. + // This helps us map a block hash to its height so we can look up the full info + // in PrefixHeightHashToNodeInfo. + PrefixHashToHeight []byte `prefix_id:"[98]"` + // NEXT_TAG: 99 } // DecodeStateKey decodes a state key into a DeSoEncoder type. This is useful for encoders which don't have a stored @@ -5206,6 +5210,12 @@ func _heightHashToNodeIndexKey(height uint32, hash *BlockHash, bitcoinNodes bool return key } +func _hashToHeightIndexKey(hash *BlockHash) []byte { + key := append([]byte{}, Prefixes.PrefixHashToHeight...) + key = append(key, hash[:]...) + return key +} + func GetHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, height uint32, hash *BlockHash, bitcoinNodes bool) *BlockNode { @@ -5245,9 +5255,44 @@ func PutHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, if err := DBSetWithTxn(txn, snap, key, serializedNode, eventManager); err != nil { return err } + + hashToHeightKey := _hashToHeightIndexKey(node.Hash) + if err = DBSetWithTxn(txn, snap, hashToHeightKey, UintToBuf(uint64(node.Height)), eventManager); err != nil { + return err + } + return nil } +func PutHashToHeightBatch(handle *badger.DB, snap *Snapshot, hashToHeight map[BlockHash]uint32, eventManager *EventManager) error { + return handle.Update(func(txn *badger.Txn) error { + for hash, height := range hashToHeight { + key := _hashToHeightIndexKey(&hash) + if err := DBSetWithTxn(txn, snap, key, UintToBuf(uint64(height)), eventManager); err != nil { + return errors.Wrap(err, "PutHashToHeightBatch: Problem setting hash to height") + } + } + return nil + }) +} + +func GetHeightForHash(db *badger.DB, snap *Snapshot, hash *BlockHash) (uint64, error) { + var height uint64 + err := db.View(func(txn *badger.Txn) error { + key := _hashToHeightIndexKey(hash) + heightBytes, err := DBGetWithTxn(txn, snap, key) + if err != nil { + return err + } + height = DecodeUint64(heightBytes) + return nil + }) + if err != nil { + return 0, err + } + return height, nil +} + func PutHeightHashToNodeInfoBatch(handle *badger.DB, snap *Snapshot, nodes []*BlockNode, bitcoinNodes bool, eventManager *EventManager) error { @@ -5551,9 +5596,47 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } +func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { + return handle.View(func(txn *badger.Txn) error { + prefix := _heightHashToNodeIndexPrefix(false) + opts := badger.DefaultIteratorOptions + opts.Prefix = prefix + // We don't need values for this migration. + opts.PrefetchValues = false + nodeIterator := txn.NewIterator(opts) + defer nodeIterator.Close() + hashToHeightMap := make(map[BlockHash]uint32) + for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { + item := nodeIterator.Item().Key() + + // Parse the key to get the height and hash. + height := binary.BigEndian.Uint32(item[1:5]) + hash := BlockHash{} + copy(hash[:], item[5:]) + hashToHeightMap[hash] = height + if len(hashToHeightMap) < 10000 { + continue + } + innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) + if innerErr != nil { + return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") + } + hashToHeightMap = make(map[BlockHash]uint32) + } + if len(hashToHeightMap) > 0 { + innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) + if innerErr != nil { + return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") + } + } + return nil + }) +} + func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} - for tipNode != nil { + maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. + for tipNode != nil && len(reversedBestChain) < maxBestChainInitLength { if (tipNode.Status&StatusBlockValidated) == 0 && (tipNode.Status&StatusBitcoinHeaderValidated) == 0 { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff433b196..6adb79f12 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -55,7 +55,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. - if _, isInBestHeaderChain := bc.bestHeaderChainMap[*headerHash]; isInBestHeaderChain { + if _, isInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(headerHash, header.Height); isInBestHeaderChain { return true, false, nil } @@ -91,9 +91,9 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode) - bc.bestHeaderChain, bc.bestHeaderChainMap = updateBestChainInMemory( - bc.bestHeaderChain, - bc.bestHeaderChainMap, + bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + bc.bestHeaderChain.Chain, + bc.bestHeaderChain.ChainMap, blocksToDetach, blocksToAttach, ) @@ -1639,8 +1639,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.bestChain = append(bc.bestChain, blockNode) - bc.bestChainMap[*blockNode.Hash] = blockNode + bc.bestChain.PushNewTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1649,9 +1648,9 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { // the bestChain slice and bestChainMap map. func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. - lastBlock := bc.bestChain[len(bc.bestChain)-1] - delete(bc.bestChainMap, *lastBlock.Hash) - bc.bestChain = bc.bestChain[:len(bc.bestChain)-1] + lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] + bc.bestChain.ChainMap.Delete(*lastBlock.Hash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] return lastBlock } @@ -1674,9 +1673,9 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { return errors.New("runCommitRuleOnBestChain: No committed blocks found") } uncommittedAncestors := []*BlockNode{} - for ii := idx + 1; ii < len(bc.bestChain); ii++ { - uncommittedAncestors = append(uncommittedAncestors, bc.bestChain[ii]) - if bc.bestChain[ii].Hash.IsEqual(blockToCommit) { + for ii := idx + 1; ii < len(bc.bestChain.Chain); ii++ { + uncommittedAncestors = append(uncommittedAncestors, bc.bestChain.Chain[ii]) + if bc.bestChain.Chain[ii].Hash.IsEqual(blockToCommit) { break } } @@ -1698,8 +1697,16 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen ) { // TODO: Is it sufficient that the current block's header points to the parent // or does it need to have something to do with the QC? - parent := bc.bestChainMap[*currentBlock.Header.PrevBlockHash] - grandParent := bc.bestChainMap[*parent.Header.PrevBlockHash] + parent, exists := bc.bestChain.GetBlockByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + glog.Errorf("canCommitGrandparent: Parent block %v not found in best chain map", currentBlock.Header.PrevBlockHash.String()) + return nil, false + } + grandParent, exists := bc.bestChain.GetBlockByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) + if !exists { + glog.Errorf("canCommitGrandparent: Grandparent block %v not found in best chain map", parent.Header.PrevBlockHash.String()) + return nil, false + } if grandParent.IsCommitted() { return nil, false } @@ -1715,7 +1722,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // to the DB and updates relevant badger indexes with info about the block. func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. - blockNode, exists := bc.bestChainMap[*blockHash] + blockNode, exists := bc.bestChain.GetBlockByHashAndHeight(blockHash, blockHeight) if !exists { return errors.Errorf("commitBlockPoS: Block %v not found in best chain map", blockHash.String()) } @@ -1844,7 +1851,10 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er } bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - tipBlock, exists := bc.bestChainMap[*tipHash] + tipBlock, exists, err := bc.bestChain.GetBlockByHash(tipHash) + if err != nil { + return nil, errors.Wrapf(err, "GetUncommittedBlocks: Problem getting block %v", tipHash.String()) + } if !exists { return nil, errors.Errorf("GetUncommittedBlocks: Block %v not found in best chain map", tipHash.String()) } @@ -2015,9 +2025,9 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc // GetCommittedTip returns the highest committed block and its index in the best chain. func (bc *Blockchain) GetCommittedTip() (*BlockNode, int) { - for ii := len(bc.bestChain) - 1; ii >= 0; ii-- { - if bc.bestChain[ii].IsCommitted() { - return bc.bestChain[ii], ii + for ii := len(bc.bestChain.Chain) - 1; ii >= 0; ii-- { + if bc.bestChain.Chain[ii].IsCommitted() { + return bc.bestChain.Chain[ii], ii } } return nil, -1 diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 2bd140003..64daa127c 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -251,7 +251,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain = []*BlockNode{genesisBlock} + bc.bestChain.Chain = []*BlockNode{genesisBlock} bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -301,7 +301,7 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, nil) + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -471,7 +471,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain = []*BlockNode{ + bc.bestChain.Chain = []*BlockNode{ genesisNode, block2, } @@ -807,7 +807,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain = []*BlockNode{genesisNode} + bc.bestChain.Chain = []*BlockNode{genesisNode} bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) @@ -844,7 +844,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain = append(bc.bestChain, block2) + bc.bestChain.Chain = append(bc.bestChain.Chain, block2) bc.blockIndex.blockIndexByHash.Put(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1235,7 +1235,7 @@ func TestShouldReorg(t *testing.T) { hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) hash3 := NewBlockHash(RandomBytes(32)) - bc.bestChain = []*BlockNode{ + bc.bestChain.Chain = []*BlockNode{ { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, @@ -1249,7 +1249,7 @@ func TestShouldReorg(t *testing.T) { newBlock := &BlockNode{ Header: &MsgDeSoHeader{ ProposedInView: 2, - PrevBlockHash: bc.bestChain[1].Hash, + PrevBlockHash: bc.bestChain.Chain[1].Hash, }, } @@ -1335,7 +1335,7 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - return collections.Any(bc.bestChain, func(bn *BlockNode) bool { + return collections.Any(bc.bestChain.Chain, func(bn *BlockNode) bool { return bn.Hash.IsEqual(hash) }) } @@ -1349,30 +1349,30 @@ func TestTryApplyNewTip(t *testing.T) { require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap := bc.bestChainMap[*hash3] + _, hash3ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash3) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) require.Len(t, connectedBlockHashes, 1) require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap := bc.bestChainMap[*newBlockHash] + _, newBlockExistsInBestChainMap := bc.bestChain.ChainMap.Get(*newBlockHash) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap := bc.bestChainMap[*hash2] + _, hash2ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash2) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) - _, hash1ExistsInBestChainMap := bc.bestChainMap[*hash1] + _, hash1ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash1) require.True(t, hash1ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.bestChain = bc.bestChain[:len(bc.bestChain)-1] - delete(bc.bestChainMap, *newBlockHash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] + bc.bestChain.ChainMap.Delete(*newBlockHash) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1420,19 +1420,19 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap = bc.bestChainMap[*hash3] + _, hash3ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash3) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap = bc.bestChainMap[*hash2] + _, hash2ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash2) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap := bc.bestChainMap[*hash4] + _, hash4ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash4) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap := bc.bestChainMap[*hash5] + _, hash5ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash5) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1441,10 +1441,10 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 2) // Reset the state of the best chain. - delete(bc.bestChainMap, *hash4) - delete(bc.bestChainMap, *hash5) - delete(bc.bestChainMap, *newBlockHash) - bc.bestChain = bc.bestChain[:len(bc.bestChain)-3] + bc.bestChain.ChainMap.Delete(*hash4) + bc.bestChain.ChainMap.Delete(*hash5) + bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) @@ -1514,8 +1514,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChainMap[*hash1] = bn1 - bc.bestChainMap[*hash2] = bn2 + bc.bestChain.ChainMap.Put(*hash1, bn1) + bc.bestChain.ChainMap.Put(*hash2, bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) @@ -1657,7 +1657,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists := testMeta.chain.bestChainMap[*committedHash] + blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*committedHash) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1682,7 +1682,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists := testMeta.chain.bestChainMap[*uncommittedBlockHash] + blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*uncommittedBlockHash) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) // TODO: Verify DB results?? Kinda silly to make sure everything is missing. @@ -1874,7 +1874,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists := testMeta.chain.bestChainMap[*timeoutBlockHash] + _, exists := testMeta.chain.bestChain.ChainMap.Get(*timeoutBlockHash) require.False(t, exists) timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) diff --git a/lib/server.go b/lib/server.go index 21756a665..debcd2b3e 100644 --- a/lib/server.go +++ b/lib/server.go @@ -861,7 +861,8 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Go through the block nodes in the blockchain and download the blocks if they're not stored. - for _, blockNode := range srv.blockchain.bestChain { + // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. + for _, blockNode := range srv.blockchain.bestChain.Chain { // We find the first block that's not stored and get ready to download blocks starting from this block onwards. if blockNode.Status&StatusBlockStored == 0 { maxBlocksInFlight := MaxBlocksInFlight @@ -876,8 +877,8 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { blockNodesToFetch := []*BlockNode{} // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. var heightLimit int - for heightLimit = len(srv.blockchain.bestChain) - 1; heightLimit >= 0; heightLimit-- { - if !srv.blockchain.bestChain[heightLimit].Status.IsFullyProcessed() { + for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { + if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { break } } @@ -888,7 +889,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode := srv.blockchain.bestChain[currentHeight] + currentNode := srv.blockchain.bestChain.Chain[currentHeight] currentHeight++ // If we've already requested this block then we don't request it again. @@ -1015,9 +1016,11 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b var hasSeenCheckpointBlockHash bool var checkpointBlockNode *BlockNode if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChainMap[*checkpointBlockInfo.Hash] + checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } else { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChainMap[*checkpointBlockInfo.Hash] + checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } // If we haven't seen the checkpoint block hash yet, we skip signature verification. if !hasSeenCheckpointBlockHash { @@ -1047,9 +1050,11 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { } hasSeenCheckPointBlockHash := false if isHeaders { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChainMap[*checkpointBlockInfo.Hash] + _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } else { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestChainMap[*checkpointBlockInfo.Hash] + _, hasSeenCheckPointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( + checkpointBlockInfo.Hash, checkpointBlockInfo.Height) } if !hasSeenCheckPointBlockHash { return fmt.Sprintf("", checkpointBlockInfo.String()) @@ -1234,11 +1239,21 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // expected height at which the snapshot should be taking place. We do this to make sure that the // snapshot we receive from the peer is up-to-date. // TODO: error handle if the hash doesn't exist for some reason. + expectedSnapshotHeightBlock, expectedSnapshotHeightblockExists, err := + srv.blockchain.bestHeaderChain.GetBlockByHeight(expectedSnapshotHeight) + if err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem getting expected snapshot height block, error (%v)", err) + return + } + if !expectedSnapshotHeightblockExists || expectedSnapshotHeightBlock == nil { + glog.Errorf("Server._handleHeaderBundle: Expected snapshot height block doesn't exist.") + return + } srv.HyperSyncProgress.SnapshotMetadata = &SnapshotEpochMetadata{ SnapshotBlockHeight: expectedSnapshotHeight, FirstSnapshotBlockHeight: expectedSnapshotHeight, CurrentEpochChecksumBytes: []byte{}, - CurrentEpochBlockHash: srv.blockchain.bestHeaderChain[expectedSnapshotHeight].Hash, + CurrentEpochBlockHash: expectedSnapshotHeightBlock.Hash, } srv.HyperSyncProgress.PrefixProgress = []*SyncPrefixProgress{} srv.HyperSyncProgress.Completed = false @@ -1646,9 +1661,17 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.snapshot.PrintChecksum("Finished hyper sync. Checksum is:") glog.Infof(CLog(Magenta, fmt.Sprintf("Metadata checksum: (%v)", srv.HyperSyncProgress.SnapshotMetadata.CurrentEpochChecksumBytes))) - + blockNode, exists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(msg.SnapshotMetadata.SnapshotBlockHeight) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error (%v)", err) + return + } + if !exists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + return + } glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - srv.blockchain.bestHeaderChain[msg.SnapshotMetadata.SnapshotBlockHeight], srv.blockchain.bestChain))) + blockNode, srv.blockchain.bestChain.Chain))) // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. @@ -1691,14 +1714,21 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. var blockNodeBatch []*BlockNode for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode := srv.blockchain.bestHeaderChain[ii] + currentNode, currentNodeExists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(ii) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + break + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + break + } // Do not set the StatusBlockStored flag, because we still need to download the past blocks. currentNode.Status |= StatusBlockProcessed currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.bestChainMap[*currentNode.Hash] = currentNode - srv.blockchain.bestChain = append(srv.blockchain.bestChain, currentNode) + srv.blockchain.bestChain.PushNewTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index e825dfbb6..0172e30f9 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -777,7 +777,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser return true, nil } - blockHeight := uint64(server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Height) + blockHeight := uint64(server.blockchain.bestChain.GetTip().Height) stateChangeSyncer.MempoolFlushId = originalCommittedFlushId @@ -804,7 +804,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser mempoolUtxoView.Snapshot = nil server.blockchain.ChainLock.RLock() - mempoolUtxoView.TipHash = server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Hash + mempoolUtxoView.TipHash = server.blockchain.bestChain.GetTip().Hash server.blockchain.ChainLock.RUnlock() // A new transaction is created so that we can simulate writes to the db without actually writing to the db. @@ -815,7 +815,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser defer txn.Discard() glog.V(2).Infof("Time since mempool sync start: %v", time.Since(startTime)) startTime = time.Now() - err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain[len(server.blockchain.bestChain)-1].Height)) + err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain.GetTip().Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 5fe9cb515..3c1a1f78b 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -371,8 +371,8 @@ func (txi *TXIndex) Update() error { newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] - delete(newBestChainMap, *(blockToDetach.Hash)) - newBlockIndex.Delete(*(blockToDetach.Hash)) + newBestChainMap.Delete(*blockToDetach.Hash) + newBlockIndex.Delete(*blockToDetach.Hash) txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) From a2e5b4bebcca69420591353ee8d5c321fefd4dcb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 15:05:32 -0400 Subject: [PATCH 003/131] check bestHeaderChain.ChainMap ONLY when processing pos headers --- lib/pos_blockchain.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 6adb79f12..1b8e1d082 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -55,7 +55,10 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. - if _, isInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(headerHash, header.Height); isInBestHeaderChain { + // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly + // updated. This is necessary because the block index may have been updated with the header but the + // bestHeaderChain.ChainMap may not have been updated yet. + if _, isInBestHeaderChain := bc.bestHeaderChain.ChainMap.Get(*headerHash); isInBestHeaderChain { return true, false, nil } From ff6ee241c08bf2a643c8fbfce21c59bcbc3d9e9a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Sep 2024 17:49:20 -0400 Subject: [PATCH 004/131] updates for locating headers and faster cache usage --- lib/blockchain.go | 63 +++++++++++++++++++++++++++++++++++++++++------ lib/constants.go | 2 ++ lib/server.go | 15 +++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index f1e35d305..d0ed35a09 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -61,7 +61,7 @@ const ( // have room for multiple forks each an entire history's length with this value). If // each node takes up 100 bytes of space this amounts to around 500MB, which also seems // like a reasonable size. - MaxBlockIndexNodes = 5000000 + MaxBlockIndexNodes = 50000000 // TODO: trim this down somehow... ) type BlockStatus uint32 @@ -533,7 +533,6 @@ type BlockIndex struct { db *badger.DB snapshot *Snapshot blockIndexByHash *lru2.Map[BlockHash, *BlockNode] - tipNode *BlockNode } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { @@ -552,10 +551,6 @@ func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockH return newMap } -func (bi *BlockIndex) setTipNode(tipNode *BlockNode) { - bi.tipNode = tipNode -} - func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) } @@ -572,6 +567,7 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } + // TODO: cache current height to exit early? blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil @@ -588,6 +584,7 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } + // TODO: cache current height to exit early? bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -600,6 +597,7 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } + // TODO: cache current height to exit early? prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) blockNodes := []*BlockNode{} @@ -650,6 +648,33 @@ func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, e return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) } + if height > uint64(bestChain.GetTip().Height) { + return nil, false, nil + } + + currTip := bestChain.GetTip() + if currTip != nil { + currNode := &BlockNode{} + *currNode = *currTip + if currNode.Height == uint32(height) { + return currNode, true, nil + } + for currNode != nil && !currNode.IsCommitted() { + if currNode.Height < uint32(height) { + break + } + if currNode.Height == uint32(height) { + return currNode, true, nil + } + var currNodeExists bool + currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( + currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) + if !currNodeExists { + break + } + } + } + prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) if len(valsFound) == 0 { @@ -678,6 +703,9 @@ func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } + if height > uint64(bestChain.GetTip().Height) { + return nil, false + } bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -699,10 +727,16 @@ func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bo } return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") } + if height > uint64(bestChain.GetTip().Height) { + return nil, false, nil + } bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false, nil } + if !bn.IsCommitted() { + return nil, false, nil + } bestChain.ChainMap.Put(*blockHash, bn) return bn, true, nil } @@ -1465,9 +1499,14 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { } if exists { var innerExists bool - tip, innerExists = bc.bestHeaderChain.GetBlockByHashAndHeight(tip.Hash, uint64(height)) + tip, innerExists, err = bc.bestHeaderChain.GetBlockByHeight(uint64(height)) + if err != nil { + glog.Errorf("LatestLocator: Problem getting block by height: %v", err) + break + } if !innerExists { - glog.Errorf("LatestLocator: Block %v not found in best header chain", tip.Hash) + glog.Errorf("LatestLocator: Block %v not found in best header chain", height) + break } } else { tip = tip.Ancestor(uint32(height)) @@ -1896,6 +1935,14 @@ func (bc *Blockchain) BestChain() []*BlockNode { return bc.bestChain.Chain } +func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash) (*BlockNode, bool, error) { + return bc.bestChain.GetBlockByHash(blockHash) +} + +func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64) (*BlockNode, bool, error) { + return bc.bestChain.GetBlockByHeight(height) +} + func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { bc.bestChain.Chain = bestChain } diff --git a/lib/constants.go b/lib/constants.go index e9001c8c3..5629a50a1 100644 --- a/lib/constants.go +++ b/lib/constants.go @@ -1955,3 +1955,5 @@ const DefaultMainnetCheckpointProvider = "https://node.deso.org" const DefaultTestnetCheckpointProvider = "https://test.deso.org" const RoutePathGetCommittedTipBlockInfo = "/api/v0/get-committed-tip-block-info" + +const BlockIndexMigrationFileName = "block_index_migration.txt" diff --git a/lib/server.go b/lib/server.go index debcd2b3e..35eafe4be 100644 --- a/lib/server.go +++ b/lib/server.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net" + "path/filepath" "reflect" "runtime" "strings" @@ -363,6 +364,16 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } } +func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { + glog.V(0).Info("Running block index migration") + blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) + hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) + if err == nil && hasRunMigration { + return nil + } + return RunBlockIndexMigration(db, nil, nil) +} + // NewServer initializes all of the internal data structures. Right now this basically // looks as follows: // - ConnectionManager starts and keeps track of peers. @@ -439,6 +450,10 @@ func NewServer( _shouldRestart bool, ) { + if err := RunBlockIndexMigrationOnce(_db, _dataDir); err != nil { + return nil, errors.Wrapf(err, "NewServer: Problem running block index migration"), true + } + var err error // Only initialize state change syncer if the directories are defined. From 71b1eaa9a78729c0d3a12e95619ca240ff7e82a2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 25 Sep 2024 17:50:00 -0400 Subject: [PATCH 005/131] more fixes --- lib/blockchain.go | 108 +++++++++++++++++++++++++++++------------- lib/db_utils.go | 2 +- lib/pos_blockchain.go | 2 +- lib/server.go | 24 +++++++--- lib/txindex.go | 2 +- 5 files changed, 94 insertions(+), 44 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index d0ed35a09..ab7f24b9d 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -467,6 +467,7 @@ func CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget + // TODO: this needs to be replaced with a call to GetBlockNodeByHeight firstNode := lastNode.Ancestor(firstNodeHeight) if firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ @@ -619,15 +620,17 @@ type BestChain struct { IsHeaderChain bool Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. ChainMap *lru2.Map[BlockHash, *BlockNode] + params *DeSoParams } -func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool) *BestChain { +func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { return &BestChain{ db: db, snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + params: params, } } @@ -648,31 +651,42 @@ func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, e return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) } - if height > uint64(bestChain.GetTip().Height) { + currTip := bestChain.GetTip() + if currTip != nil && height > uint64(currTip.Height) { return nil, false, nil } - currTip := bestChain.GetTip() if currTip != nil { - currNode := &BlockNode{} - *currNode = *currTip - if currNode.Height == uint32(height) { - return currNode, true, nil - } - for currNode != nil && !currNode.IsCommitted() { - if currNode.Height < uint32(height) { - break - } - if currNode.Height == uint32(height) { - return currNode, true, nil - } - var currNodeExists bool - currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( - currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) - if !currNodeExists { - break + currTipHeight := currTip.Height + delta := currTipHeight - uint32(height) + if delta < uint32(len(bestChain.Chain)) { + targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] + if uint64(targetNode.Height) == height { + return targetNode, true, nil } } + //targetIndex := (len(bestChain.Chain)-1) + //// We can probably do some binary search thing here instead. + //currNode := &BlockNode{} + //*currNode = *currTip + //if currNode.Height == uint32(height) { + // return currNode, true, nil + //} + //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. + //for currNode != nil && !currNode.IsStored() { + // if currNode.Height < uint32(height) { + // break + // } + // if currNode.Height == uint32(height) { + // return currNode, true, nil + // } + // var currNodeExists bool + // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( + // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) + // if !currNodeExists { + // break + // } + //} } prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) @@ -1160,7 +1174,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) + _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1219,9 +1233,9 @@ func NewBlockchain( //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), //bestChainMap: make(map[BlockHash]*BlockNode), - bestChain: NewBestChain(db, snapshot, false), + bestChain: NewBestChain(db, snapshot, false, params), //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - bestHeaderChain: NewBestChain(db, snapshot, true), + bestHeaderChain: NewBestChain(db, snapshot, true, params), blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), @@ -1509,6 +1523,7 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { break } } else { + // TODO: this needs to be replaced with a read-through cache call. tip = tip.Ancestor(uint32(height)) } @@ -2099,7 +2114,8 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { //} } -func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { +// node1 is the current tip and node2 is a new node. +func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, blockIndex *BlockIndex) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil @@ -2107,18 +2123,33 @@ func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { // Get the two nodes to be at the same height. if node1.Height > node2.Height { - node1 = node1.Ancestor(node2.Height) + node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1Parent, node2, blockIndex) } else if node1.Height < node2.Height { - node2 = node2.Ancestor(node1.Height) + node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1, node2Parent, blockIndex) } // Iterate the nodes backward until they're either the same or we // reach the end of the lists. We only need to check node1 for nil // since they're the same height and we are iterating both back // in tandem. - for node1 != nil && !node1.Hash.IsEqual(node2.Hash) { - node1 = node1.Parent - node2 = node2.Parent + if !node1.Hash.IsEqual(node2.Hash) { + node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + if !exists { + return nil + } + node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + if !exists { + return nil + } + return _FindCommonAncestor(node1Parent, node2Parent, blockIndex) } // By now either node1 == node2 and we found the common ancestor or @@ -2198,9 +2229,10 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { +func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) ( + _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { // Find the common ancestor of this block and the main header chain. - commonAncestor := _FindCommonAncestor(tip, newNode) + commonAncestor := _FindCommonAncestor(tip, newNode, blockIndex) // Log a warning if the reorg is going to be a big one. numBlocks := tip.Height - commonAncestor.Height @@ -2229,8 +2261,16 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode) (_commonAncestor *BlockN // attachNodes will have the new node as its first element and work back to // the node right after the common ancestor as its last element. attachBlocks := []*BlockNode{} - for currentBlock := newNode; *currentBlock.Hash != *commonAncestor.Hash; currentBlock = currentBlock.Parent { + currentBlock := &BlockNode{} + *currentBlock = *newNode + for *currentBlock.Hash != *commonAncestor.Hash { attachBlocks = append(attachBlocks, currentBlock) + var exists bool + currentBlock, exists = blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + // TODO: what should we do here? + glog.Fatal("GetReorgBlocks: Failed to find parent of block") + } } // Reverse attachBlocks so that the node right after the common ancestor // will be the first element and the node at the end of the list will be @@ -2438,7 +2478,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode) + _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3003,7 +3043,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate) + commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate, bc.blockIndex) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/db_utils.go b/lib/db_utils.go index ec9350f5a..e9520a44c 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5284,7 +5284,7 @@ func GetHeightForHash(db *badger.DB, snap *Snapshot, hash *BlockHash) (uint64, e if err != nil { return err } - height = DecodeUint64(heightBytes) + height, _ = Uvarint(heightBytes) return nil }) if err != nil { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1b8e1d082..a839ec850 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode) + _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode, bc.blockIndex) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/server.go b/lib/server.go index 35eafe4be..f19b91c3c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -365,13 +365,21 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { - glog.V(0).Info("Running block index migration") blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) + glog.V(0).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { + glog.V(0).Info("Block index migration has already been run") return nil } - return RunBlockIndexMigration(db, nil, nil) + glog.V(0).Info("Running block index migration") + if err = RunBlockIndexMigration(db, nil, nil); err != nil { + return errors.Wrapf(err, "Problem running block index migration") + } + if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { + return errors.Wrapf(err, "Problem saving block index migration file") + } + return nil } // NewServer initializes all of the internal data structures. Right now this basically @@ -1405,6 +1413,8 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) + glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (chain map: %v) - (chain: %v) ", + srv.blockchain.bestHeaderChain.ChainMap.Len(), len(srv.blockchain.bestHeaderChain.Chain)) } func (srv *Server) _handleGetBlocks(pp *Peer, msg *MsgDeSoGetBlocks) { @@ -1682,12 +1692,12 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { return } if !exists { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") - return + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist: (%v)", msg.SnapshotMetadata.SnapshotBlockHeight) + //return + } else { + glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", + blockNode, srv.blockchain.bestChain.Chain))) } - glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - blockNode, srv.blockchain.bestChain.Chain))) - // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. checksumBytes, err := srv.snapshot.Checksum.ToBytes() diff --git a/lib/txindex.go b/lib/txindex.go index 3c1a1f78b..36e8ac416 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -242,7 +242,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip) + commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip, txi.TXIndexChain.blockIndex) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From 17a56bd6ae64ad91b3c095a3de8c569f9ac2fa1e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 14:44:51 -0400 Subject: [PATCH 006/131] more fixes --- lib/blockchain.go | 94 +++++++++++++++++++++++++++++++------------ lib/pos_blockchain.go | 2 +- lib/server.go | 13 +----- lib/txindex.go | 5 ++- 4 files changed, 74 insertions(+), 40 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ab7f24b9d..004af0456 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1174,7 +1174,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, bc.blockIndex) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1644,6 +1644,14 @@ func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } +func (bc *Blockchain) HasHeaderByHashAndHeight(headerHash *BlockHash, height uint64) bool { + if height > uint64(bc.headerTip().Height) { + return false + } + _, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, height) + return exists +} + // TODO: delete me? func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { if blockHeight >= bc.bestChain.GetTip().Height { @@ -2114,42 +2122,62 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { //} } -// node1 is the current tip and node2 is a new node. -func _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, blockIndex *BlockIndex) *BlockNode { +// Note: we make some assumptions that we only care about ancestors in the best chain. +func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, isHeaderChain bool) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil } + chainToUse := &BestChain{} + if isHeaderChain { + chainToUse = bc.bestHeaderChain + } else { + chainToUse = bc.bestChain + } - // Get the two nodes to be at the same height. - if node1.Height > node2.Height { - node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) - if !exists { + committedTip, _ := bc.GetCommittedTip() + // If both nodes are at a height greater than the committed tip, then we know the + // committed tip is a common ancestor. + // TODO: this isn't necessarily the GREATEST common ancestor. It's just a common ancestor. + if node1.Height > committedTip.Height && node2.Height > committedTip.Height { + // If both nodes are at a height greater than the committed tip, then we know that + // we have valid parent pointers and can use the Ancestor function to get use to the right place. + if node1.Height > node2.Height { + node1 = node1.Ancestor(node2.Height) + } else if node2.Height > node1.Height { + node2 = node2.Ancestor(node1.Height) + } + } else { + var exists bool + var err error + // Get the two nodes to be at the same height. + if node1.Height > node2.Height { + node1, exists, err = chainToUse.GetBlockByHeight(uint64(node2.Height)) + } else if node1.Height < node2.Height { + node2, exists, err = chainToUse.GetBlockByHeight(uint64(node1.Height)) + } + if err != nil { return nil } - return _FindCommonAncestor(node1Parent, node2, blockIndex) - } else if node1.Height < node2.Height { - node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) if !exists { return nil } - return _FindCommonAncestor(node1, node2Parent, blockIndex) } // Iterate the nodes backward until they're either the same or we // reach the end of the lists. We only need to check node1 for nil // since they're the same height and we are iterating both back // in tandem. + var exists bool if !node1.Hash.IsEqual(node2.Hash) { - node1Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) + node1, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) if !exists { return nil } - node2Parent, exists := blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) + node2, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) if !exists { return nil } - return _FindCommonAncestor(node1Parent, node2Parent, blockIndex) } // By now either node1 == node2 and we found the common ancestor or @@ -2229,17 +2257,24 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) ( +func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeaderChain bool) ( _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { // Find the common ancestor of this block and the main header chain. - commonAncestor := _FindCommonAncestor(tip, newNode, blockIndex) + commonAncestor := bc._FindCommonAncestor(tip, newNode, isHeaderChain) + + if commonAncestor == nil { + glog.Fatalf("No common ancestor found between tip and new node: tip hash (%v), newNode hash (%v)", tip.Hash, newNode.Hash) + return + } // Log a warning if the reorg is going to be a big one. - numBlocks := tip.Height - commonAncestor.Height - if numBlocks > 10 { - glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ - "block (%v) at height (%d) to block (%v) at height of (%d)", - numBlocks, tip, tip.Height, newNode, newNode.Height) + if tip != nil && commonAncestor != nil { + numBlocks := tip.Height - commonAncestor.Height + if numBlocks > 10 { + glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ + "block (%v) at height (%d) to block (%v) at height of (%d)", + numBlocks, tip, tip.Height, newNode, newNode.Height) + } } // Get the blocks to detach. Start at the tip and work backwards to the @@ -2249,7 +2284,14 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) // detachBlocks will have the current tip as its first element and parents // of the tip thereafter. detachBlocks := []*BlockNode{} - for currentBlock := tip; *currentBlock.Hash != *commonAncestor.Hash; currentBlock = currentBlock.Parent { + currentBlock := &BlockNode{} + *currentBlock = *tip + for currentBlock != nil && *currentBlock.Hash != *commonAncestor.Hash { + var exists bool + currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + if !exists { + glog.Fatalf("GetReorgBlocks: Failed to find parent of block. Parent hash %v", currentBlock.Header.PrevBlockHash) + } detachBlocks = append(detachBlocks, currentBlock) } @@ -2261,12 +2303,12 @@ func GetReorgBlocks(tip *BlockNode, newNode *BlockNode, blockIndex *BlockIndex) // attachNodes will have the new node as its first element and work back to // the node right after the common ancestor as its last element. attachBlocks := []*BlockNode{} - currentBlock := &BlockNode{} + currentBlock = &BlockNode{} *currentBlock = *newNode for *currentBlock.Hash != *commonAncestor.Hash { attachBlocks = append(attachBlocks, currentBlock) var exists bool - currentBlock, exists = blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { // TODO: what should we do here? glog.Fatal("GetReorgBlocks: Failed to find parent of block") @@ -2478,7 +2520,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := GetReorgBlocks(headerTip, newNode, bc.blockIndex) + _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3043,7 +3085,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(currentTip, nodeToValidate, bc.blockIndex) + commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate, false) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index a839ec850..46a0ce78e 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := GetReorgBlocks(currentTip, blockNode, bc.blockIndex) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode, true) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/server.go b/lib/server.go index f19b91c3c..71f003c8b 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1114,12 +1114,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // have this issue. Hitting duplicates after we're done syncing is // fine and can happen in certain cases. headerHash, _ := headerReceived.Hash() - hasHeader, err := srv.blockchain.HasHeader(headerHash) - // TODO: what should we really do here? - if err != nil { - glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", headerHash, err) - return - } + hasHeader := srv.blockchain.HasHeaderByHashAndHeight(headerHash, headerReceived.Height) if hasHeader { if srv.blockchain.isSyncing() { @@ -1353,11 +1348,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // we're either not aware of or that we don't think is the best chain. // Doing things this way makes it so that when we request blocks we // are 100% positive the peer has them. - hasHeader, err := srv.blockchain.HasHeader(msg.TipHash) - if err != nil { - glog.Errorf("Server._handleHeaderBundle: Error checking if header %v exists: %v", msg.TipHash, err) - return - } + hasHeader := srv.blockchain.HasHeaderByHashAndHeight(msg.TipHash, uint64(msg.TipHeight)) if !hasHeader { glog.V(1).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ "blockchain so not requesting anything else from them. Our block "+ diff --git a/lib/txindex.go b/lib/txindex.go index 36e8ac416..4f2998d86 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -171,7 +171,8 @@ func (txi *TXIndex) Start() { txi.updateWaitGroup.Done() return default: - if txi.CoreChain.ChainState() == SyncStateFullyCurrent { + chainState := txi.CoreChain.ChainState() + if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && txi.CoreChain.headerTip().Height-txi.CoreChain.blockTip().Height < 10) { if !txi.CoreChain.IsFullyStored() { glog.V(1).Infof("TXIndex: Waiting, blockchain is not fully stored") break @@ -242,7 +243,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := GetReorgBlocks(&derefedTxindexTipNode, committedTip, txi.TXIndexChain.blockIndex) + commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip, false) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From d15392733a30711ebe78526a4f640dbac43f6e4e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 16:02:00 -0400 Subject: [PATCH 007/131] more fix, specifically around .Ancestor --- lib/block_producer.go | 4 +- lib/blockchain.go | 107 ++++++++++++++++++----------------------- lib/blockchain_test.go | 35 +++++++++++--- lib/pos_blockchain.go | 2 +- lib/txindex.go | 2 +- 5 files changed, 79 insertions(+), 71 deletions(-) diff --git a/lib/block_producer.go b/lib/block_producer.go index c54f6771e..a52318102 100644 --- a/lib/block_producer.go +++ b/lib/block_producer.go @@ -368,8 +368,8 @@ func (desoBlockProducer *DeSoBlockProducer) _getBlockTemplate(publicKey []byte) blockRet.Header.TransactionMerkleRoot = merkleRoot // Compute the next difficulty target given the current tip. - diffTarget, err := CalcNextDifficultyTarget( - lastNode, CurrentHeaderVersion, desoBlockProducer.params) + diffTarget, err := desoBlockProducer.chain.CalcNextDifficultyTarget( + lastNode, CurrentHeaderVersion) if err != nil { return nil, nil, nil, errors.Wrapf(err, "DeSoBlockProducer._getBlockTemplate: Problem computing next difficulty: ") } diff --git a/lib/blockchain.go b/lib/blockchain.go index 004af0456..4de5b6988 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -406,7 +406,7 @@ func NewBlockNode( } } -func (nn *BlockNode) Ancestor(height uint32) *BlockNode { +func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode { if height > nn.Height { return nil } @@ -414,6 +414,14 @@ func (nn *BlockNode) Ancestor(height uint32) *BlockNode { node := nn for ; node != nil && node.Height != height; node = node.Parent { // Keep iterating node until the condition no longer holds. + if node.Parent == nil { + var exists bool + node.Parent, exists = blockIndex.GetBlockNodeByHashAndHeight( + node.Header.PrevBlockHash, uint64(node.Height-1)) + if !exists { + return nil + } + } } return node @@ -424,24 +432,24 @@ func (nn *BlockNode) Ancestor(height uint32) *BlockNode { // height minus provided distance. // // This function is safe for concurrent access. -func (nn *BlockNode) RelativeAncestor(distance uint32) *BlockNode { - return nn.Ancestor(nn.Height - distance) +func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { + return nn.Ancestor(nn.Height-distance, blockIndex) } // CalcNextDifficultyTarget computes the difficulty target expected of the // next block. -func CalcNextDifficultyTarget( - lastNode *BlockNode, version uint32, params *DeSoParams) (*BlockHash, error) { +func (bc *Blockchain) CalcNextDifficultyTarget( + lastNode *BlockNode, version uint32) (*BlockHash, error) { // Compute the blocks in each difficulty cycle. - blocksPerRetarget := uint32(params.TimeBetweenDifficultyRetargets / params.TimeBetweenBlocks) + blocksPerRetarget := uint32(bc.params.TimeBetweenDifficultyRetargets / bc.params.TimeBetweenBlocks) // We effectively skip the first difficulty retarget by returning the default // difficulty value for the first cycle. Not doing this (or something like it) // would cause the genesis block's timestamp, which could be off by several days // to significantly skew the first cycle in a way that is mostly annoying for // testing but also suboptimal for the mainnet. - minDiffBytes, err := hex.DecodeString(params.MinDifficultyTargetHex) + minDiffBytes, err := hex.DecodeString(bc.params.MinDifficultyTargetHex) if err != nil { return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem computing min difficulty") } @@ -462,14 +470,19 @@ func CalcNextDifficultyTarget( } // If we get here it means we reached a difficulty retarget point. - targetSecs := int64(params.TimeBetweenDifficultyRetargets / time.Second) - minRetargetTimeSecs := targetSecs / params.MaxDifficultyRetargetFactor - maxRetargetTimeSecs := targetSecs * params.MaxDifficultyRetargetFactor + targetSecs := int64(bc.params.TimeBetweenDifficultyRetargets / time.Second) + minRetargetTimeSecs := targetSecs / bc.params.MaxDifficultyRetargetFactor + maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget // TODO: this needs to be replaced with a call to GetBlockNodeByHeight - firstNode := lastNode.Ancestor(firstNodeHeight) - if firstNode == nil { + firstNode, exists, err := bc.bestChain.GetBlockByHeight(uint64(firstNodeHeight)) + if err != nil { + return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ + "beginning of retarget interval at height %d during retarget from height %d", + firstNodeHeight, lastNode.Height) + } + if !exists || firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) @@ -993,7 +1006,9 @@ func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. - if bc.ChainState() == SyncStateFullyCurrent { + chainState := bc.ChainState() + if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && + bc.headerTip().Height-bc.blockTip().Height < 10) { for _, blockNode := range bc.bestChain.Chain { if !blockNode.Status.IsFullyProcessed() { return false @@ -1174,7 +1189,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode, true) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, @@ -1523,8 +1538,7 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { break } } else { - // TODO: this needs to be replaced with a read-through cache call. - tip = tip.Ancestor(uint32(height)) + tip = tip.Ancestor(uint32(height), bc.blockIndex) } // Once 11 entries have been included, start doubling the @@ -2123,45 +2137,18 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { } // Note: we make some assumptions that we only care about ancestors in the best chain. -func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, isHeaderChain bool) *BlockNode { +func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. return nil } - chainToUse := &BestChain{} - if isHeaderChain { - chainToUse = bc.bestHeaderChain - } else { - chainToUse = bc.bestChain - } - committedTip, _ := bc.GetCommittedTip() - // If both nodes are at a height greater than the committed tip, then we know the - // committed tip is a common ancestor. - // TODO: this isn't necessarily the GREATEST common ancestor. It's just a common ancestor. - if node1.Height > committedTip.Height && node2.Height > committedTip.Height { - // If both nodes are at a height greater than the committed tip, then we know that - // we have valid parent pointers and can use the Ancestor function to get use to the right place. - if node1.Height > node2.Height { - node1 = node1.Ancestor(node2.Height) - } else if node2.Height > node1.Height { - node2 = node2.Ancestor(node1.Height) - } - } else { - var exists bool - var err error - // Get the two nodes to be at the same height. - if node1.Height > node2.Height { - node1, exists, err = chainToUse.GetBlockByHeight(uint64(node2.Height)) - } else if node1.Height < node2.Height { - node2, exists, err = chainToUse.GetBlockByHeight(uint64(node1.Height)) - } - if err != nil { - return nil - } - if !exists { - return nil - } + // If both nodes are at a height greater than the committed tip, then we know that + // we have valid parent pointers and can use the Ancestor function to get use to the right place. + if node1.Height > node2.Height { + node1 = node1.Ancestor(node2.Height, bc.blockIndex) + } else if node2.Height > node1.Height { + node2 = node2.Ancestor(node1.Height, bc.blockIndex) } // Iterate the nodes backward until they're either the same or we @@ -2169,7 +2156,7 @@ func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode, is // since they're the same height and we are iterating both back // in tandem. var exists bool - if !node1.Hash.IsEqual(node2.Hash) { + for !node1.Hash.IsEqual(node2.Hash) { node1, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) if !exists { return nil @@ -2257,10 +2244,12 @@ func CheckTransactionSanity(txn *MsgDeSoTxn, blockHeight uint32, params *DeSoPar return nil } -func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeaderChain bool) ( +func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( _commonAncestor *BlockNode, _detachNodes []*BlockNode, _attachNodes []*BlockNode) { + // TODO: finding common ancestors is very expensive for txindex when txindex is very far + // behind. Currently, it requires loading the entire chain into memory. // Find the common ancestor of this block and the main header chain. - commonAncestor := bc._FindCommonAncestor(tip, newNode, isHeaderChain) + commonAncestor := bc._FindCommonAncestor(tip, newNode) if commonAncestor == nil { glog.Fatalf("No common ancestor found between tip and new node: tip hash (%v), newNode hash (%v)", tip.Hash, newNode.Hash) @@ -2268,7 +2257,7 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeade } // Log a warning if the reorg is going to be a big one. - if tip != nil && commonAncestor != nil { + if tip != nil { numBlocks := tip.Height - commonAncestor.Height if numBlocks > 10 { glog.Warningf("GetReorgBlocks: Proceeding with reorg of (%d) blocks from "+ @@ -2287,12 +2276,12 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode, isHeade currentBlock := &BlockNode{} *currentBlock = *tip for currentBlock != nil && *currentBlock.Hash != *commonAncestor.Hash { + detachBlocks = append(detachBlocks, currentBlock) var exists bool currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { glog.Fatalf("GetReorgBlocks: Failed to find parent of block. Parent hash %v", currentBlock.Header.PrevBlockHash) } - detachBlocks = append(detachBlocks, currentBlock) } // Get the blocks to attach. Start at the new node and work backwards to @@ -2452,8 +2441,8 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // the parent block. Note that if the parent block is in the block index // then it has necessarily had its difficulty validated, and so using it to // do this check makes sense. - diffTarget, err := CalcNextDifficultyTarget( - parentNode, blockHeader.Version, bc.params) + diffTarget, err := bc.CalcNextDifficultyTarget( + parentNode, blockHeader.Version) if err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem computing difficulty "+ @@ -2520,7 +2509,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode, true) + _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) @@ -3085,7 +3074,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Find the common ancestor of this block and the main chain. // TODO: Reorgs with postgres? - commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate, false) + commonAncestor, detachBlocks, attachBlocks := bc.GetReorgBlocks(currentTip, nodeToValidate) // Log a warning if the reorg is going to be a big one. numBlocks := currentTip.Height - commonAncestor.Height if numBlocks > 10 { diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index c54c056a1..18c136607 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1222,6 +1222,8 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1229,6 +1231,8 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1236,7 +1240,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1273,7 +1277,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 30; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1318,6 +1322,8 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1325,6 +1331,8 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1332,7 +1340,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1369,7 +1377,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 30; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1414,6 +1422,8 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1421,6 +1431,8 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1428,7 +1440,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1469,12 +1481,15 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, TimeBetweenBlocks: 2 * time.Second, MaxDifficultyRetargetFactor: 3, } + bc.params = fakeParams nodes := []*BlockNode{} diffsAsInts := []int64{} @@ -1483,7 +1498,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1524,6 +1539,8 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { _ = assert _ = require + bc, _, _ := NewTestBlockchain(t) + fakeParams := &DeSoParams{ MinDifficultyTargetHex: hex.EncodeToString(BigintToHash(big.NewInt(100000))[:]), TimeBetweenDifficultyRetargets: 6 * time.Second, @@ -1531,6 +1548,8 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { MaxDifficultyRetargetFactor: 2, } + bc.params = fakeParams + nodes := []*BlockNode{} diffsAsInts := []int64{} for ii := 0; ii < 13; ii++ { @@ -1538,7 +1557,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { if ii > 0 { lastNode = nodes[ii-1] } - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, @@ -1575,7 +1594,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { diffsAsInts = []int64{} for ii := 13; ii < 34; ii++ { lastNode := nodes[ii-1] - nextDiff, err := CalcNextDifficultyTarget(lastNode, HeaderVersion0, fakeParams) + nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 46a0ce78e..1d455be28 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -93,7 +93,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode, true) + _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, diff --git a/lib/txindex.go b/lib/txindex.go index 4f2998d86..b7643ef7a 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -243,7 +243,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // At this point, we know our txindex tip is in our block index so // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip, false) + commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip) return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks } From 47e1958abadb24cdb430ad72537275732312cfbb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 16:28:22 -0400 Subject: [PATCH 008/131] bump test timeout --- test.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.Dockerfile b/test.Dockerfile index 3eb8c7501..6262c5c55 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "30m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 5e5c0c18fec3fc074171df905ca81df3456414a3 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 19:41:20 -0400 Subject: [PATCH 009/131] load all blocks after committed tip in block index --- lib/blockchain.go | 64 +++++++++++++++++++++++++++++++------- lib/db_utils.go | 63 +++++++++++++++++++++++++++++++++++++ lib/pos_blockchain_test.go | 6 ++-- lib/pos_consensus_test.go | 15 ++++----- 4 files changed, 126 insertions(+), 22 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 4de5b6988..86a0d9001 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -544,29 +544,54 @@ type CheckpointBlockInfoAndError struct { } type BlockIndex struct { - db *badger.DB - snapshot *Snapshot - blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + db *badger.DB + snapshot *Snapshot + blockIndexByHash *lru2.Map[BlockHash, *BlockNode] + blockIndexByHeight *lru2.Map[uint64, []*BlockNode] + maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { return &BlockIndex{ - db: db, - snapshot: snapshot, - blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + db: db, + snapshot: snapshot, + blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + blockIndexByHeight: lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + maxHeightSeen: 0, } } -func newBlockIndexByHashFromMap(input map[BlockHash]*BlockNode) *lru2.Map[BlockHash, *BlockNode] { - newMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) +func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { + newHashToBlockNodeMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap := lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes) + maxHeight := uint64(0) for key, val := range input { - newMap.Put(key, val) + newHashToBlockNodeMap.Put(key, val) + blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) + if !exists { + blocksAtHeight = []*BlockNode{} + } + blocksAtHeight = append(blocksAtHeight, val) + newHeightToBlockNodeMap.Put(uint64(val.Height), append(blocksAtHeight, val)) + if uint64(val.Height) > maxHeight { + maxHeight = uint64(val.Height) + } } - return newMap + bi.blockIndexByHash = newHashToBlockNodeMap + bi.blockIndexByHeight = newHeightToBlockNodeMap + bi.maxHeightSeen = maxHeight } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) + blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) + if !exists { + blocksAtHeight = []*BlockNode{} + } + bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) + if uint64(blockNode.Height) > bi.maxHeightSeen { + bi.maxHeightSeen = uint64(blockNode.Height) + } } func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { @@ -581,7 +606,9 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - // TODO: cache current height to exit early? + if height > bi.maxHeightSeen { + return nil, false, nil + } blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil @@ -598,7 +625,9 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } - // TODO: cache current height to exit early? + if height > bi.maxHeightSeen { + return nil, false + } bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -611,6 +640,13 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } + if height > bi.maxHeightSeen { + return []*BlockNode{} + } + blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) + if exists { + return blockNodesAtHeight + } // TODO: cache current height to exit early? prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) @@ -1113,6 +1149,10 @@ func (bc *Blockchain) _initChain() error { currBlockCounter++ } } + if err = bc.blockIndex.LoadBlockIndexFromHeight(tipNode.Height, bc.params); err != nil { + return errors.Wrapf(err, "_initChain: Problem loading block index from db") + } + } // At this point the blockIndexByHash should contain a full node tree with all diff --git a/lib/db_utils.go b/lib/db_utils.go index e9520a44c..cc3e4faee 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5596,6 +5596,69 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } +func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams) error { + prefix := _heightHashToNodePrefixByHeight(height, false) + + return bi.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + nodeIterator := txn.NewIterator(opts) + defer nodeIterator.Close() + for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { + var blockNode *BlockNode + + // Don't bother checking the key. We assume that the key lines up + // with what we've stored in the value in terms of (height, block hash). + item := nodeIterator.Item() + err := item.Value(func(blockNodeBytes []byte) error { + // Deserialize the block node. + var err error + // TODO: There is room for optimization here by pre-allocating a + // contiguous list of block nodes and then populating that list + // rather than having each blockNode be a stand-alone allocation. + blockNode, err = DeserializeBlockNode(blockNodeBytes) + if err != nil { + return err + } + return nil + }) + if err != nil { + return err + } + + // If we got here it means we read a blockNode successfully. Store it + // into our node index. + bi.addNewBlockNodeToBlockIndex(blockNode) + + // Find the parent of this block, which should already have been read + // in and connect it. Skip the genesis block, which has height 0. Also + // skip the block if its PrevBlockHash is empty, which will be true for + // the BitcoinStartBlockNode. + // + // TODO: There is room for optimization here by keeping a reference to + // the last node we've iterated over and checking if that node is the + // parent. Doing this would avoid an expensive hashmap check to get + // the parent by its block hash. + if blockNode.Height == 0 || (*blockNode.Header.PrevBlockHash == BlockHash{}) { + continue + } + if parent, ok := bi.GetBlockNodeByHashAndHeight(blockNode.Header.PrevBlockHash, uint64(blockNode.Height)); ok { + // We found the parent node so connect it. + blockNode.Parent = parent + } else { + // If we're syncing a DeSo node and we hit a PoS block, we expect there to + // be orphan blocks in the block index. In this case, we don't throw an error. + if params.IsPoSBlockHeight(uint64(blockNode.Height)) { + continue + } + // In this case we didn't find the parent so error. There shouldn't + // be any unconnectedTxns in our block index. + return fmt.Errorf("GetBlockIndex: Could not find parent for blockNode: %+v", blockNode) + } + } + return nil + }) +} + func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { return handle.View(func(txn *badger.Txn) error { prefix := _heightHashToNodeIndexPrefix(false) diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 64daa127c..fa50800ce 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -329,7 +329,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -475,7 +475,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { genesisNode, block2, } - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -808,7 +808,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain.Chain = []*BlockNode{genesisNode} - bc.blockIndex.blockIndexByHash = newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 900afb71a..81e785c5b 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -102,6 +102,11 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { currentView := blockHeader.ValidatorsVoteQC.GetView() + 1 nextView := currentView + 1 + blockIndex := NewBlockIndex(nil, nil) + blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + *blockHash: {Header: blockHeader}, + }) + // Create a mock consensus fastHotStuffConsensus := FastHotStuffConsensus{ lock: sync.RWMutex{}, @@ -111,13 +116,9 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { }, params: &DeSoTestnetParams, blockchain: &Blockchain{ - ChainLock: deadlock.RWMutex{}, - blockIndex: &BlockIndex{ - blockIndexByHash: newBlockIndexByHashFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, - }), - }, - params: &DeSoTestnetParams, + ChainLock: deadlock.RWMutex{}, + blockIndex: blockIndex, + params: &DeSoTestnetParams, }, fastHotStuffEventLoop: &consensus.MockFastHotStuffEventLoop{ OnIsInitialized: alwaysReturnTrue, From 9a32c2631216532caad73953c51077c416967b44 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 26 Sep 2024 19:55:08 -0400 Subject: [PATCH 010/131] fix GetBlockNodeByHashAndHeight - no need for checking height > maxHeight --- lib/blockchain.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 86a0d9001..8ac3f39ff 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -606,9 +606,6 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } - if height > bi.maxHeightSeen { - return nil, false, nil - } blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil From 5359be82be1977571c321605b3daf845b9d325e6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 00:04:54 -0400 Subject: [PATCH 011/131] bump timeout to 60m --- test.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.Dockerfile b/test.Dockerfile index 6262c5c55..21f1dc087 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "30m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "60m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 89a845e1f3b89713afcacd0a5e5f88d921cd2204 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 11:33:56 -0400 Subject: [PATCH 012/131] try hashicorp lru cache --- go.mod | 40 +------------- go.sum | 104 ++----------------------------------- lib/block_view_test.go | 4 +- lib/blockchain.go | 85 +++++++++++++++++------------- lib/miner.go | 23 ++++++++ lib/pos_blockchain.go | 2 +- lib/pos_blockchain_test.go | 26 +++++----- lib/postgres.go | 8 +-- lib/txindex.go | 4 +- test.Dockerfile | 2 +- 10 files changed, 100 insertions(+), 198 deletions(-) diff --git a/go.mod b/go.mod index a64a34f67..ddf84244b 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-pg/pg/v10 v10.10.0 github.com/golang/glog v1.0.0 github.com/google/uuid v1.2.0 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/oleiade/lane v1.0.1 @@ -48,46 +49,14 @@ require ( ) require ( - decred.org/cspp/v2 v2.2.0 // indirect github.com/Microsoft/go-winio v0.4.16 // indirect - github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect - github.com/dchest/siphash v1.2.3 // indirect - github.com/decred/base58 v1.0.5 // indirect - github.com/decred/dcrd v1.9.4 // indirect - github.com/decred/dcrd/addrmgr/v2 v2.0.4 // indirect - github.com/decred/dcrd/bech32 v1.1.4 // indirect - github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 // indirect - github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 // indirect - github.com/decred/dcrd/certgen v1.2.0 // indirect - github.com/decred/dcrd/chaincfg/chainhash v1.0.4 // indirect - github.com/decred/dcrd/chaincfg/v3 v3.2.1 // indirect - github.com/decred/dcrd/connmgr/v3 v3.1.2 // indirect - github.com/decred/dcrd/container/apbf v1.0.1 // indirect - github.com/decred/dcrd/container/lru v1.0.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/decred/dcrd/crypto/rand v1.0.0 // indirect - github.com/decred/dcrd/crypto/ripemd160 v1.0.2 // indirect - github.com/decred/dcrd/database/v3 v3.0.2 // indirect - github.com/decred/dcrd/dcrec v1.0.1 // indirect - github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 // indirect - github.com/decred/dcrd/dcrjson/v4 v4.1.0 // indirect - github.com/decred/dcrd/dcrutil/v4 v4.0.2 // indirect - github.com/decred/dcrd/gcs/v4 v4.1.0 // indirect - github.com/decred/dcrd/math/uint256 v1.0.2 // indirect - github.com/decred/dcrd/mixing v0.4.1 // indirect - github.com/decred/dcrd/peer/v3 v3.1.2 // indirect - github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 // indirect - github.com/decred/dcrd/txscript/v4 v4.1.1 // indirect - github.com/decred/dcrd/wire v1.7.0 // indirect - github.com/decred/go-socks v1.1.0 // indirect - github.com/decred/slog v1.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -99,17 +68,12 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.8 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/jessevdk/go-flags v1.6.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jrick/bitset v1.0.0 // indirect - github.com/jrick/logrotate v1.1.2 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.12.3 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.9 // indirect @@ -128,7 +92,6 @@ require ( github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tinylib/msgp v1.1.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect @@ -152,6 +115,5 @@ require ( gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.3.0 // indirect mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 821ddcfdd..c159e2333 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -decred.org/cspp/v2 v2.2.0 h1:VSOUC1w0Wo+QOGS0r1XO6TLnO16X67KuvpDmRRYyr08= -decred.org/cspp/v2 v2.2.0/go.mod h1:9nO3bfvCheOPIFZw5f6sRQ42CjBFB5RKSaJ9Iq6G4MA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -41,8 +39,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= -github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -98,8 +94,6 @@ github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GK github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a h1:clYxJ3Os0EQUKDDVU8M0oipllX0EkuFNBfhVQuIfyF0= -github.com/companyzero/sntrup4591761 v0.0.0-20220309191932-9e0f3af2f07a/go.mod h1:z/9Ck1EDixEbBbZ2KH2qNHekEmDLTOZ+FyoIPWWSVOI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -117,78 +111,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= -github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/decred/base58 v1.0.5 h1:hwcieUM3pfPnE/6p3J100zoRfGkQxBulZHo7GZfOqic= -github.com/decred/base58 v1.0.5/go.mod h1:s/8lukEHFA6bUQQb/v3rjUySJ2hu+RioCzLukAVkrfw= -github.com/decred/dcrd v1.9.4 h1:DgeZdf8TTtGuZGVs5DnAL0ghbhnrPYfENneqUqEaq8o= -github.com/decred/dcrd v1.9.4/go.mod h1:7I6LovGdFimw3LTPCtOGQ+8VV1HGnO+2L2KA4yi5NWA= -github.com/decred/dcrd/addrmgr/v2 v2.0.4 h1:3MWJiO2STogQwNRF3W4yjCzSJtaxqtw+UI3x2+bYeOg= -github.com/decred/dcrd/addrmgr/v2 v2.0.4/go.mod h1:661DIS/De2iLNLMwIKazUQfQypUqJ5om7PXNX0fEMms= -github.com/decred/dcrd/bech32 v1.1.4 h1:wFlLM7Oic0MlIhQZdCQhdIqVc4CNaQ0vNR9fgCoWfe0= -github.com/decred/dcrd/bech32 v1.1.4/go.mod h1:jliqHZmCbVfT06Lh1mQywEKFVidRclbBJIUmwdoKhu0= -github.com/decred/dcrd/blockchain/stake/v5 v5.0.1 h1:KDm6myUPi8j2TTL7LZ+iT+R/pIbxd8qG89fjJNitzx0= -github.com/decred/dcrd/blockchain/stake/v5 v5.0.1/go.mod h1:y1tMD1TssTlPmKDYbSrF3Ujznj+STkXFfYPwoVfe+xA= -github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1 h1:zeI9CHkLM9be4QOBmIAtoPfs6NCgJM1lpmRUYE61I8o= -github.com/decred/dcrd/blockchain/standalone/v2 v2.2.1/go.mod h1:yXZz/EgWdGw5nqMEvyKj/iXZ9I2VSyO95xKj6mRUMIM= -github.com/decred/dcrd/certgen v1.2.0 h1:FF6XXV//5q38/c6QbGQdR35ZJz0GPIkejsZZU3oHuBQ= -github.com/decred/dcrd/certgen v1.2.0/go.mod h1:LRh6dF2WPQeDA6QQSZE+SfK7AL6FuFtCRDHZf8DyGzg= -github.com/decred/dcrd/chaincfg/chainhash v1.0.4 h1:zRCv6tdncLfLTKYqu7hrXvs7hW+8FO/NvwoFvGsrluU= -github.com/decred/dcrd/chaincfg/chainhash v1.0.4/go.mod h1:hA86XxlBWwHivMvxzXTSD0ZCG/LoYsFdWnCekkTMCqY= -github.com/decred/dcrd/chaincfg/v3 v3.2.1 h1:x9zKJaU24WAKbxAR1UyFKHlM3oJgP0H9LodokM4X5lM= -github.com/decred/dcrd/chaincfg/v3 v3.2.1/go.mod h1:SDCWDtY7BLj0leXc9FuoA1YjSVKyCIBVAyxwZn6+sXc= -github.com/decred/dcrd/connmgr/v3 v3.1.2 h1:+xNopie2L3YYwwkz51k0h/pASATOBzHtl2O8eodGg04= -github.com/decred/dcrd/connmgr/v3 v3.1.2/go.mod h1:tdbErFiNOuy/sHrX2mwaOk+r1HLs3EBz2EGxsocMPe4= -github.com/decred/dcrd/container/apbf v1.0.1 h1:oepQzRtLADudsrx0AmmowoU1kambozINTMXduH6Mge0= -github.com/decred/dcrd/container/apbf v1.0.1/go.mod h1:paQplssZMsRhwOUcP6LDNDypyb7lwsFHrUKUkAPFWtQ= -github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI= -github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/crypto/rand v1.0.0 h1:Ah9Asl36OZt09sGSMbJZuL1HfwGdlC38q/ZUeLDVKRg= -github.com/decred/dcrd/crypto/rand v1.0.0/go.mod h1:coa7BbxSTiKH6esi257plGfMFYuGL4MTbQlLYnOdzpE= -github.com/decred/dcrd/crypto/ripemd160 v1.0.2 h1:TvGTmUBHDU75OHro9ojPLK+Yv7gDl2hnUvRocRCjsys= -github.com/decred/dcrd/crypto/ripemd160 v1.0.2/go.mod h1:uGfjDyePSpa75cSQLzNdVmWlbQMBuiJkvXw/MNKRY4M= -github.com/decred/dcrd/database/v3 v3.0.2 h1:rgP7XNZemTs8ZC7bnTKO8JO79Woj5nq+yQYmB9ry7yM= -github.com/decred/dcrd/database/v3 v3.0.2/go.mod h1:3Ge1yoxEOsqd72V5LTA9g0B7mlY0MGbpxeE1fniIXsQ= -github.com/decred/dcrd/dcrec v1.0.1 h1:gDzlndw0zYxM5BlaV17d7ZJV6vhRe9njPBFeg4Db2UY= -github.com/decred/dcrd/dcrec v1.0.1/go.mod h1:CO+EJd8eHFb8WHa84C7ZBkXsNUIywaTHb+UAuI5uo6o= -github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3 h1:l/lhv2aJCUignzls81+wvga0TFlyoZx8QxRMQgXpZik= -github.com/decred/dcrd/dcrec/edwards/v2 v2.0.3/go.mod h1:AKpV6+wZ2MfPRJnTbQ6NPgWrKzbe9RCIlCF/FKzMtM8= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/decred/dcrd/dcrjson/v4 v4.1.0 h1:WJVogRnYnNxB5hWoGHODvP4fNTG1JycTuHHKt/XucHk= -github.com/decred/dcrd/dcrjson/v4 v4.1.0/go.mod h1:2qVikafVF9/X3PngQVmqkbUbyAl32uik0k/kydgtqMc= -github.com/decred/dcrd/dcrutil/v4 v4.0.2 h1:eIl3E6gGln54qE8nk5o5lLtjh2/9C2Rz63OpD662h+8= -github.com/decred/dcrd/dcrutil/v4 v4.0.2/go.mod h1:iS3JB1ac3R3FgfpTF1kBD+SPNet8TmiW3Br+/Jc5MC8= -github.com/decred/dcrd/gcs/v4 v4.1.0 h1:tpW7JW53yJZlgNwl/n2NL1b8NxHaIPRUyNuLMkB/Hks= -github.com/decred/dcrd/gcs/v4 v4.1.0/go.mod h1:nPTbGM/I3Ihe5KFvUmxZEqQP/jDZQjQ63+WEi/f4lqU= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/decred/dcrd/lru v1.1.1 h1:kWFDaW0OWx6AD6Ki342c+JPmHbiVdE6rK81pT3fuo/Y= -github.com/decred/dcrd/lru v1.1.1/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= -github.com/decred/dcrd/math/uint256 v1.0.2 h1:o8peafL5QmuXGTergI3YDpDU0eq5Z0pQi88B8ym4PRA= -github.com/decred/dcrd/math/uint256 v1.0.2/go.mod h1:7M/y9wJJvlyNG/f/X6mxxhxo9dgloZHFiOfbiscl75A= -github.com/decred/dcrd/mixing v0.4.1 h1:W8ZCzhmNyzG1xjJMA3L6FOElmp98Ttnk3dDUxD6irAE= -github.com/decred/dcrd/mixing v0.4.1/go.mod h1:ySvVwTZyVz5YvevA6YjPrB6pJEwTm7IkHohTfaiHh2c= -github.com/decred/dcrd/peer/v3 v3.1.2 h1:Qe7SpqDtfM0HARmDYwr4WjUu16X6HQ7ZWNnHqE1swiw= -github.com/decred/dcrd/peer/v3 v3.1.2/go.mod h1:M9FxNkHuEBtsRW5gwzIH4cJTWk5xSkxy9zG+TEL1N2Y= -github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0 h1:l0DnCcILTNrpy8APF3FLN312ChpkQaAuW30aC/RgBaw= -github.com/decred/dcrd/rpc/jsonrpc/types/v4 v4.3.0/go.mod h1:j+kkRPXPJB5S9VFOsx8SQLcU7PTFkPKRc1aCHN4ENzA= -github.com/decred/dcrd/txscript/v4 v4.1.1 h1:R4M2+jMujgQA91899SkL0cW66d6DC76Gx+1W1oEHjc0= -github.com/decred/dcrd/txscript/v4 v4.1.1/go.mod h1:7ybmJoI+b6dxvQ+0aXdZpkyrj0PbnylJCzFxD1g8+/A= -github.com/decred/dcrd/wire v1.7.0 h1:5JHiDjEQeS4XUl4PfnTZYLwAD/E/+LwBmPRec/fP76o= -github.com/decred/dcrd/wire v1.7.0/go.mod h1:lAqrzV0SU4kyV6INLEJgDtUjJaTaVKrbF4LHtaYl+zU= -github.com/decred/go-socks v1.1.0 h1:dnENcc0KIqQo3HSXdgboXAHgqsCIutkqq6ntQjYtm2U= -github.com/decred/go-socks v1.1.0/go.mod h1:sDhHqkZH0X4JjSa02oYOGhcGHYp12FsY1jQ/meV8md0= -github.com/decred/slog v1.2.0 h1:soHAxV52B54Di3WtKLfPum9OFfWqwtf/ygf9njdfnPM= -github.com/decred/slog v1.2.0/go.mod h1:kVXlGnt6DHy2fV5OjSeuvCJ0OmlmTF6LFpEPMu/fOY0= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -278,7 +208,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -295,8 +224,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -314,8 +243,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -336,6 +263,8 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -359,17 +288,11 @@ github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= -github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jrick/bitset v1.0.0 h1:Ws0PXV3PwXqWK2n7Vz6idCdrV/9OrBXgHEJi27ZB9Dw= -github.com/jrick/bitset v1.0.0/go.mod h1:ZOYB5Uvkla7wIEY4FEssPVi3IQXa02arznRaYaAEPe4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0= -github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -386,8 +309,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -559,8 +480,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -628,8 +547,6 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -662,7 +579,7 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -693,8 +610,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -708,8 +623,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -752,8 +665,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -761,8 +672,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -796,8 +705,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -891,8 +799,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= -lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 2717aeb50..b69307dbf 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -793,8 +793,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // Update the block and header metadata chains. tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] - tm.chain.bestChain.ChainMap.Delete(*lastBlockHash) - tm.chain.bestHeaderChain.ChainMap.Delete(*lastBlockHash) + tm.chain.bestChain.ChainMap.Remove(*lastBlockHash) + tm.chain.bestHeaderChain.ChainMap.Remove(*lastBlockHash) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 8ac3f39ff..89988dec4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -18,7 +18,7 @@ import ( "github.com/decred/dcrd/lru" - lru2 "github.com/decred/dcrd/container/lru" + lru2 "github.com/hashicorp/golang-lru/v2" "github.com/deso-protocol/core/collections" @@ -546,33 +546,35 @@ type CheckpointBlockInfoAndError struct { type BlockIndex struct { db *badger.DB snapshot *Snapshot - blockIndexByHash *lru2.Map[BlockHash, *BlockNode] - blockIndexByHeight *lru2.Map[uint64, []*BlockNode] + blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] + blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { + blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) return &BlockIndex{ db: db, snapshot: snapshot, - blockIndexByHash: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? - blockIndexByHeight: lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes), // TODO: parameterize this? + blockIndexByHash: blockIndexByHash, // TODO: parameterize this? + blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? maxHeightSeen: 0, } } func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) - newHeightToBlockNodeMap := lru2.NewMap[uint64, []*BlockNode](MaxBlockIndexNodes) + newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) maxHeight := uint64(0) for key, val := range input { - newHashToBlockNodeMap.Put(key, val) + newHashToBlockNodeMap.Add(key, val) blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) if !exists { blocksAtHeight = []*BlockNode{} } blocksAtHeight = append(blocksAtHeight, val) - newHeightToBlockNodeMap.Put(uint64(val.Height), append(blocksAtHeight, val)) + newHeightToBlockNodeMap.Add(uint64(val.Height), append(blocksAtHeight, val)) if uint64(val.Height) > maxHeight { maxHeight = uint64(val.Height) } @@ -583,12 +585,12 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) + bi.blockIndexByHash.Add(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) if !exists { blocksAtHeight = []*BlockNode{} } - bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) + bi.blockIndexByHeight.Add(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) if uint64(blockNode.Height) > bi.maxHeightSeen { bi.maxHeightSeen = uint64(blockNode.Height) } @@ -615,6 +617,9 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + if height > bi.maxHeightSeen { + return nil, false + } val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true @@ -622,9 +627,6 @@ func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height u if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } - if height > bi.maxHeightSeen { - return nil, false - } bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false @@ -665,24 +667,25 @@ type BestChain struct { snapshot *Snapshot IsHeaderChain bool Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. - ChainMap *lru2.Map[BlockHash, *BlockNode] + ChainMap *lru2.Cache[BlockHash, *BlockNode] params *DeSoParams } func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { + chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) return &BestChain{ db: db, snapshot: snapshot, IsHeaderChain: isHeaderChain, Chain: []*BlockNode{}, - ChainMap: lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes), + ChainMap: chainMap, params: params, } } func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { bestChain.Chain = append(bestChain.Chain, tipNode) - bestChain.ChainMap.Put(*tipNode.Hash, tipNode) + bestChain.ChainMap.Add(*tipNode.Hash, tipNode) } func (bestChain *BestChain) GetTip() *BlockNode { @@ -770,7 +773,7 @@ func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height if bn == nil { return nil, false } - bestChain.ChainMap.Put(*blockHash, bn) + bestChain.ChainMap.Add(*blockHash, bn) return bn, true } @@ -797,7 +800,7 @@ func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bo if !bn.IsCommitted() { return nil, false, nil } - bestChain.ChainMap.Put(*blockHash, bn) + bestChain.ChainMap.Add(*blockHash, bn) return bn, true, nil } @@ -983,16 +986,16 @@ func getCheckpointBlockInfoFromProviderHelper(provider string) *CheckpointBlockI } func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bc.blockIndex.blockIndexByHash.Put(*blockNode.Hash, blockNode) + bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *lru2.Map[BlockHash, *BlockNode], + _blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], ) { - newBlockIndexByHash := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBlockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) - newBlockIndexByHash.Put(key, val) + newBlockIndexByHash.Add(key, val) } return newBlockIndexByHash } @@ -1012,25 +1015,25 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { +func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newBestChain = append(newBestChain, bc.bestChain.Chain...) for _, key := range bc.bestChain.ChainMap.Keys() { val, _ := bc.bestChain.ChainMap.Get(key) - newBestChainMap.Put(key, val) + newBestChainMap.Add(key, val) } return newBestChain, newBestChainMap } -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Map[BlockHash, *BlockNode]) { +func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { newBestChain := []*BlockNode{} - newBestChainMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) for _, key := range bc.bestHeaderChain.ChainMap.Keys() { val, _ := bc.bestHeaderChain.ChainMap.Get(key) - newBestChainMap.Put(key, val) + newBestChainMap.Add(key, val) } return newBestChain, newBestChainMap @@ -1162,7 +1165,7 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") } for _, bestChainNode := range bc.bestChain.Chain { - bc.bestChain.ChainMap.Put(*bestChainNode.Hash, bestChainNode) + bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) } } @@ -1175,7 +1178,7 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") } for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - bc.bestHeaderChain.ChainMap.Put(*bestHeaderChainNode.Hash, bestHeaderChainNode) + bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) } } @@ -2023,8 +2026,8 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { func (bc *Blockchain) SetBestChainMap( bestChain []*BlockNode, - bestChainMap *lru2.Map[BlockHash, *BlockNode], - blockIndexByHash *lru2.Map[BlockHash, *BlockNode], + bestChainMap *lru2.Cache[BlockHash, *BlockNode], + blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], ) { bc.bestChain.Chain = bestChain bc.bestChain.ChainMap = bestChainMap @@ -2350,14 +2353,14 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap *lru2.Map[BlockHash, *BlockNode]) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru2.Cache[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 for blockOffset := 0; blockOffset < len(detachBlocks); blockOffset++ { blockIndex := tipIndex - blockOffset - mainChainMap.Delete(*mainChainList[blockIndex].Hash) + mainChainMap.Remove(*mainChainList[blockIndex].Hash) } mainChainList = mainChainList[:len(mainChainList)-len(detachBlocks)] @@ -2366,7 +2369,7 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Map[ // first, with the new tip at the end. for _, attachNode := range attachBlocks { mainChainList = append(mainChainList, attachNode) - mainChainMap.Put(*attachNode.Hash, attachNode) + mainChainMap.Add(*attachNode.Hash, attachNode) } return mainChainList, mainChainMap @@ -2676,14 +2679,19 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.End("Blockchain.ProcessBlock: Initial") bc.timer.Start("Blockchain.ProcessBlock: BlockNode") + now := time.Now() // See if a node for the block exists in our node index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. + glog.V(0).Infof("ProcessBlock: time to check if node exists: %v", time.Since(now)) + now = time.Now() if !nodeExists { _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) + glog.V(0).Infof("ProcessBlock: time to process header: %v", time.Since(now)) + now = time.Now() if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. @@ -2701,6 +2709,9 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // block index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) + + glog.V(0).Infof("ProcessBlock: time to get node after processing header: %v", time.Since(now)) + now = time.Now() } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point diff --git a/lib/miner.go b/lib/miner.go index 5bc1e45d1..fab9cb1d8 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -113,6 +113,7 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // // TODO(miner): Replace with a call to GetBlockTemplate publicKey := desoMiner._getRandomPublicKey() + now := time.Now() blockID, headerBytes, extraNonces, diffTarget, err := desoMiner.BlockProducer.GetHeadersAndExtraDatas( publicKey, 1 /*numHeaders*/, CurrentHeaderVersion) if err != nil { @@ -121,6 +122,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } + glog.V(0).Infof("Time to get headers and extra data: %v", time.Since(now)) + now = time.Now() header := &MsgDeSoHeader{} if err := header.FromBytes(headerBytes[0]); err != nil { glog.Errorf("DeSoMiner._startThread: Error parsing header to " + @@ -138,6 +141,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B glog.Error(errors.Wrapf(err, "DeSoMiner._startThread: Problem while mining: ")) break } + glog.V(0).Infof("Time to header to bytes and find lowest hash: %v", time.Since(now)) + now = time.Now() if atomic.LoadInt32(&desoMiner.stopping) == 1 { glog.V(1).Infof("DeSoMiner._startThread: Stopping thread %d", threadIndex) @@ -161,6 +166,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } + glog.V(0).Infof("Time to get copy of recent block: %v", time.Since(now)) + now = time.Now() // Swap in the public key and extraNonce. This should make the block consistent with // the header we were just mining on. @@ -178,6 +185,8 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // Use the nonce we computed blockToMine.Header.Nonce = bestNonce + glog.V(0).Infof("Time to recompute rewards and whatnot: %v", time.Since(now)) + now = time.Now() return diffTarget, blockToMine } @@ -187,16 +196,21 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoolToUpdate *DeSoMempool) (_block *MsgDeSoBlock, _err error) { // Add a call to update the BlockProducer. + now := time.Now() // TODO(performance): We shouldn't have to do this, it just makes tests pass right now. if err := desoMiner.BlockProducer.UpdateLatestBlockTemplate(); err != nil { // Error if we can't update the template but don't stop the show. glog.Error(err) } + glog.V(0).Infof("Time to update block template: %v", time.Since(now)) + now = time.Now() diffTarget, blockToMine := desoMiner._mineSingleBlock(threadIndex) if blockToMine == nil { return nil, fmt.Errorf("DeSoMiner._startThread: _mineSingleBlock returned nil; should only happen if we're stopping") } + glog.V(0).Infof("Time to update mine single block: %v", time.Since(now)) + now = time.Now() // Log information on the block we just mined. bestHash, _ := blockToMine.Hash() @@ -208,6 +222,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo desoMiner.BlockProducer.chain.blockTip().Header) scs := spew.ConfigState{DisableMethods: true, Indent: " ", DisablePointerAddresses: true} glog.V(1).Infof(scs.Sdump(blockToMine)) + //glog.V(0).Infof("Time to print a bunch of stuff: %v", time.Since(now)) + now = time.Now() // Sanitize the block for the comparison we're about to do. We need to do // this because the comparison function below will think they're different // if one has nil and one has an empty list. Annoying, but this solves the @@ -242,6 +258,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if err := desoMiner.BlockProducer.SignBlock(blockToMine); err != nil { return nil, fmt.Errorf("Error signing block: %v", err) } + //glog.V(0).Infof("Time to sign block: %v", time.Since(now)) + now = time.Now() // Process the block. If the block is connected and/or accepted, the Server // will be informed about it. This will cause it to be relayed appropriately. @@ -259,6 +277,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo return blockToMine, fmt.Errorf("ERROR calling ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)", isMainChain, isOrphan, err) } + glog.V(0).Infof("Time to process block: %v", time.Since(now)) + now = time.Now() // If a mempool object is passed then update it. Normally this isn't necessary because // ProcessBlock will trigger it because the backendServer will be set on the blockchain @@ -266,6 +286,8 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if mempoolToUpdate != nil { mempoolToUpdate.UpdateAfterConnectBlock(blockToMine) } + //glog.V(0).Infof("Time to update mempool after connect block: %v", time.Since(now)) + now = time.Now() decimalPlaces := int64(1000) diffTargetBaseline, _ := hex.DecodeString(desoMiner.params.MinDifficultyTargetHex) @@ -278,6 +300,7 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if atomic.LoadInt32(&desoMiner.stopping) == 1 { return nil, fmt.Errorf("DeSoMiner._startThread: Stopping thread %d", threadIndex) } + //glog.V(0).Infof("Time to update everything else: %v", time.Since(now)) return blockToMine, nil } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1d455be28..134505eab 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1652,7 +1652,7 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Delete(*lastBlock.Hash) + bc.bestChain.ChainMap.Remove(*lastBlock.Hash) bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] return lastBlock } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index fa50800ce..8dddc5213 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) bc.bestChain.Chain = []*BlockNode{genesisBlock} - bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) + bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) randomBLSPrivateKey := _generateRandomBLSPrivateKey(t) @@ -845,7 +845,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.bestChain.Chain = append(bc.bestChain.Chain, block2) - bc.blockIndex.blockIndexByHash.Put(*hash2, block2) + bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) require.Equal(t, err, RuleErrorDoesNotExtendCommittedTip) @@ -1317,9 +1317,9 @@ func TestTryApplyNewTip(t *testing.T) { bc.addTipBlockToBestChain(bn1) bc.addTipBlockToBestChain(bn2) bc.addTipBlockToBestChain(bn3) - bc.blockIndex.blockIndexByHash.Put(*hash1, bn1) - bc.blockIndex.blockIndexByHash.Put(*hash2, bn2) - bc.blockIndex.blockIndexByHash.Put(*hash3, bn3) + bc.blockIndex.blockIndexByHash.Add(*hash1, bn1) + bc.blockIndex.blockIndexByHash.Add(*hash2, bn2) + bc.blockIndex.blockIndexByHash.Add(*hash3, bn3) // Simple reorg. Just replacing the uncommitted tip. newBlock := &MsgDeSoBlock{ @@ -1372,7 +1372,7 @@ func TestTryApplyNewTip(t *testing.T) { // Remove newBlock from the best chain and block index to reset the state. bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.ChainMap.Remove(*newBlockHash) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1401,8 +1401,8 @@ func TestTryApplyNewTip(t *testing.T) { Height: 6, }, } - bc.blockIndex.blockIndexByHash.Put(*hash4, bn4) - bc.blockIndex.blockIndexByHash.Put(*hash5, bn5) + bc.blockIndex.blockIndexByHash.Add(*hash4, bn4) + bc.blockIndex.blockIndexByHash.Add(*hash5, bn5) // Set new block's parent to hash5 newBlockNode.Header.PrevBlockHash = hash5 @@ -1441,9 +1441,9 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 2) // Reset the state of the best chain. - bc.bestChain.ChainMap.Delete(*hash4) - bc.bestChain.ChainMap.Delete(*hash5) - bc.bestChain.ChainMap.Delete(*newBlockHash) + bc.bestChain.ChainMap.Remove(*hash4) + bc.bestChain.ChainMap.Remove(*hash5) + bc.bestChain.ChainMap.Remove(*newBlockHash) bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] // Add block 2 and 3 back. @@ -1514,8 +1514,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChain.ChainMap.Put(*hash1, bn1) - bc.bestChain.ChainMap.Put(*hash2, bn2) + bc.bestChain.ChainMap.Add(*hash1, bn1) + bc.bestChain.ChainMap.Add(*hash2, bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) diff --git a/lib/postgres.go b/lib/postgres.go index 0c5991756..08d078006 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - lru2 "github.com/decred/dcrd/container/lru" + lru2 "github.com/hashicorp/golang-lru/v2" "net/url" "regexp" "strings" @@ -1305,16 +1305,16 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*lru2.Map[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru2.Cache[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap := lru2.NewMap[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { - blockMap.Put(*block.Hash, &BlockNode{ + blockMap.Add(*block.Hash, &BlockNode{ Hash: block.Hash, Height: uint32(block.Height), DifficultyTarget: block.DifficultyTarget, diff --git a/lib/txindex.go b/lib/txindex.go index b7643ef7a..904211819 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -372,8 +372,8 @@ func (txi *TXIndex) Update() error { newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() newBestChain = newBestChain[:len(newBestChain)-1] - newBestChainMap.Delete(*blockToDetach.Hash) - newBlockIndex.Delete(*blockToDetach.Hash) + newBestChainMap.Remove(*blockToDetach.Hash) + newBlockIndex.Remove(*blockToDetach.Hash) txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) diff --git a/test.Dockerfile b/test.Dockerfile index 21f1dc087..34182c4c7 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "60m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "15m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 4f7ff12632e26e13d9c8eae6c726f1c076fabbbb Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 12:47:20 -0400 Subject: [PATCH 013/131] Fix up some more tests and remove logging --- lib/blockchain.go | 48 ++++++++++++-------------------------- lib/blockchain_test.go | 28 ++++++++++++---------- lib/miner.go | 24 +------------------ lib/pos_blockchain_test.go | 13 +++++++---- lib/pos_consensus_test.go | 2 +- 5 files changed, 42 insertions(+), 73 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 89988dec4..56bb03ea6 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -548,7 +548,7 @@ type BlockIndex struct { snapshot *Snapshot blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] - maxHeightSeen uint64 + //maxHeightSeen uint64 } func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { @@ -559,29 +559,18 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { snapshot: snapshot, blockIndexByHash: blockIndexByHash, // TODO: parameterize this? blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? - maxHeightSeen: 0, + //maxHeightSeen: 0, } } func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) - maxHeight := uint64(0) - for key, val := range input { - newHashToBlockNodeMap.Add(key, val) - blocksAtHeight, exists := newHeightToBlockNodeMap.Get(uint64(val.Height)) - if !exists { - blocksAtHeight = []*BlockNode{} - } - blocksAtHeight = append(blocksAtHeight, val) - newHeightToBlockNodeMap.Add(uint64(val.Height), append(blocksAtHeight, val)) - if uint64(val.Height) > maxHeight { - maxHeight = uint64(val.Height) - } - } bi.blockIndexByHash = newHashToBlockNodeMap bi.blockIndexByHeight = newHeightToBlockNodeMap - bi.maxHeightSeen = maxHeight + for _, val := range input { + bi.addNewBlockNodeToBlockIndex(val) + } } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { @@ -589,11 +578,15 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) if !exists { blocksAtHeight = []*BlockNode{} + } else { + // Make sure we don't add the same block node twice. + for _, blockAtHeight := range blocksAtHeight { + if blockAtHeight.Hash.IsEqual(blockNode.Hash) { + return + } + } } bi.blockIndexByHeight.Add(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) - if uint64(blockNode.Height) > bi.maxHeightSeen { - bi.maxHeightSeen = uint64(blockNode.Height) - } } func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { @@ -617,9 +610,6 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { - if height > bi.maxHeightSeen { - return nil, false - } val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true @@ -639,9 +629,9 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } - if height > bi.maxHeightSeen { - return []*BlockNode{} - } + //if height > bi.maxHeightSeen { + // return []*BlockNode{} + //} blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) if exists { return blockNodesAtHeight @@ -2679,19 +2669,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.End("Blockchain.ProcessBlock: Initial") bc.timer.Start("Blockchain.ProcessBlock: BlockNode") - now := time.Now() // See if a node for the block exists in our node index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set // the header validation status for it. - glog.V(0).Infof("ProcessBlock: time to check if node exists: %v", time.Since(now)) - now = time.Now() if !nodeExists { _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) - glog.V(0).Infof("ProcessBlock: time to process header: %v", time.Since(now)) - now = time.Now() if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. @@ -2709,9 +2694,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // block index. // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) - - glog.V(0).Infof("ProcessBlock: time to get node after processing header: %v", time.Since(now)) - now = time.Now() } // At this point if the node still doesn't exist or if the header's validation // failed then we should return an error for the block. Note that at this point diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 18c136607..adcf3f429 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1244,7 +1244,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1254,6 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1281,7 +1282,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1291,6 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1344,7 +1346,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1354,6 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1381,7 +1384,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1391,6 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1444,7 +1448,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1454,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1502,7 +1506,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1512,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1561,7 +1565,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1571,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1598,7 +1602,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( lastNode, - nil, + NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, nil, @@ -1608,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - + bc.bestChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/miner.go b/lib/miner.go index fab9cb1d8..0df444912 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -113,7 +113,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // // TODO(miner): Replace with a call to GetBlockTemplate publicKey := desoMiner._getRandomPublicKey() - now := time.Now() blockID, headerBytes, extraNonces, diffTarget, err := desoMiner.BlockProducer.GetHeadersAndExtraDatas( publicKey, 1 /*numHeaders*/, CurrentHeaderVersion) if err != nil { @@ -122,8 +121,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } - glog.V(0).Infof("Time to get headers and extra data: %v", time.Since(now)) - now = time.Now() header := &MsgDeSoHeader{} if err := header.FromBytes(headerBytes[0]); err != nil { glog.Errorf("DeSoMiner._startThread: Error parsing header to " + @@ -141,8 +138,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B glog.Error(errors.Wrapf(err, "DeSoMiner._startThread: Problem while mining: ")) break } - glog.V(0).Infof("Time to header to bytes and find lowest hash: %v", time.Since(now)) - now = time.Now() if atomic.LoadInt32(&desoMiner.stopping) == 1 { glog.V(1).Infof("DeSoMiner._startThread: Stopping thread %d", threadIndex) @@ -166,8 +161,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B time.Sleep(1 * time.Second) continue } - glog.V(0).Infof("Time to get copy of recent block: %v", time.Since(now)) - now = time.Now() // Swap in the public key and extraNonce. This should make the block consistent with // the header we were just mining on. @@ -185,8 +178,6 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B // Use the nonce we computed blockToMine.Header.Nonce = bestNonce - glog.V(0).Infof("Time to recompute rewards and whatnot: %v", time.Since(now)) - now = time.Now() return diffTarget, blockToMine } @@ -196,21 +187,16 @@ func (desoMiner *DeSoMiner) _mineSingleBlock(threadIndex uint32) (_diffTarget *B func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoolToUpdate *DeSoMempool) (_block *MsgDeSoBlock, _err error) { // Add a call to update the BlockProducer. - now := time.Now() // TODO(performance): We shouldn't have to do this, it just makes tests pass right now. if err := desoMiner.BlockProducer.UpdateLatestBlockTemplate(); err != nil { // Error if we can't update the template but don't stop the show. glog.Error(err) } - glog.V(0).Infof("Time to update block template: %v", time.Since(now)) - now = time.Now() diffTarget, blockToMine := desoMiner._mineSingleBlock(threadIndex) if blockToMine == nil { return nil, fmt.Errorf("DeSoMiner._startThread: _mineSingleBlock returned nil; should only happen if we're stopping") } - glog.V(0).Infof("Time to update mine single block: %v", time.Since(now)) - now = time.Now() // Log information on the block we just mined. bestHash, _ := blockToMine.Hash() @@ -222,8 +208,7 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo desoMiner.BlockProducer.chain.blockTip().Header) scs := spew.ConfigState{DisableMethods: true, Indent: " ", DisablePointerAddresses: true} glog.V(1).Infof(scs.Sdump(blockToMine)) - //glog.V(0).Infof("Time to print a bunch of stuff: %v", time.Since(now)) - now = time.Now() + // Sanitize the block for the comparison we're about to do. We need to do // this because the comparison function below will think they're different // if one has nil and one has an empty list. Annoying, but this solves the @@ -258,8 +243,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if err := desoMiner.BlockProducer.SignBlock(blockToMine); err != nil { return nil, fmt.Errorf("Error signing block: %v", err) } - //glog.V(0).Infof("Time to sign block: %v", time.Since(now)) - now = time.Now() // Process the block. If the block is connected and/or accepted, the Server // will be informed about it. This will cause it to be relayed appropriately. @@ -277,8 +260,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo return blockToMine, fmt.Errorf("ERROR calling ProcessBlock: isMainChain=(%v), isOrphan=(%v), err=(%v)", isMainChain, isOrphan, err) } - glog.V(0).Infof("Time to process block: %v", time.Since(now)) - now = time.Now() // If a mempool object is passed then update it. Normally this isn't necessary because // ProcessBlock will trigger it because the backendServer will be set on the blockchain @@ -286,8 +267,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if mempoolToUpdate != nil { mempoolToUpdate.UpdateAfterConnectBlock(blockToMine) } - //glog.V(0).Infof("Time to update mempool after connect block: %v", time.Since(now)) - now = time.Now() decimalPlaces := int64(1000) diffTargetBaseline, _ := hex.DecodeString(desoMiner.params.MinDifficultyTargetHex) @@ -300,7 +279,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo if atomic.LoadInt32(&desoMiner.stopping) == 1 { return nil, fmt.Errorf("DeSoMiner._startThread: Stopping thread %d", threadIndex) } - //glog.V(0).Infof("Time to update everything else: %v", time.Since(now)) return blockToMine, nil } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 8dddc5213..47d1147c1 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -317,14 +317,14 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { t.Cleanup(resetGlobalDeSoParams) hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) - genesisNode := NewBlockNode(nil, hash1, 1, nil, nil, &MsgDeSoHeader{ + genesisNode := NewBlockNode(nil, hash1, 0, nil, nil, &MsgDeSoHeader{ Version: 2, - Height: 1, + Height: 0, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated) - block2 := NewBlockNode(genesisNode, hash2, 2, nil, nil, &MsgDeSoHeader{ + block2 := NewBlockNode(genesisNode, hash2, 1, nil, nil, &MsgDeSoHeader{ Version: 2, - Height: 2, + Height: 1, ProposedInView: 2, ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, @@ -1880,6 +1880,11 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) require.True(t, exists) require.False(t, timeoutBlockNode.IsCommitted()) + + // The reorg block hash should be in the block index now. + reorgBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(reorgBlockHash, reorgBlock.Header.Height) + require.True(t, exists) + require.True(t, reorgBlockNode.IsStored()) } var dummyParentBlockHash, orphanBlockHash *BlockHash { diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 81e785c5b..74ab70dca 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -104,7 +104,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { blockIndex := NewBlockIndex(nil, nil) blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ - *blockHash: {Header: blockHeader}, + *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) // Create a mock consensus From 722dcc0afbadef707f5071879de1edcad5d680f7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 16:12:27 -0400 Subject: [PATCH 014/131] fix string slice workaround, fix calc next difficulty --- cmd/config.go | 5 +++-- lib/blockchain.go | 10 ++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 77ce94606..fe04cbac7 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -94,7 +94,8 @@ type Config struct { func GetStringSliceWorkaround(flagName string) []string { value := viper.GetString(flagName) if value == "" || value == " " { - return []string{} + values := viper.GetStringSlice(flagName) + return values } return strings.Split(value, ",") } @@ -147,7 +148,6 @@ func LoadConfig() *Config { // Peers config.ConnectIPs = GetStringSliceWorkaround("connect-ips") - glog.V(2).Infof("Connect IPs read in: %v", config.ConnectIPs) config.AddIPs = GetStringSliceWorkaround("add-ips") config.AddSeeds = GetStringSliceWorkaround("add-seeds") config.TargetOutboundPeers = viper.GetUint32("target-outbound-peers") @@ -266,6 +266,7 @@ func (config *Config) Print() { glog.Infof("MaxSyncBlockHeight: %v", config.MaxSyncBlockHeight) } + glog.Infof("Connect IPs: %s", config.ConnectIPs) if len(config.ConnectIPs) > 0 { glog.Infof("Connect IPs: %s", config.ConnectIPs) } diff --git a/lib/blockchain.go b/lib/blockchain.go index 56bb03ea6..9315999b3 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -475,14 +475,8 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - // TODO: this needs to be replaced with a call to GetBlockNodeByHeight - firstNode, exists, err := bc.bestChain.GetBlockByHeight(uint64(firstNodeHeight)) - if err != nil { - return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ - "beginning of retarget interval at height %d during retarget from height %d", - firstNodeHeight, lastNode.Height) - } - if !exists || firstNode == nil { + firstNode := lastNode.Ancestor(firstNodeHeight, bc.blockIndex) + if firstNode == nil { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) From 1941487365703543c6639382476f7f6f52b6eae7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 16:22:31 -0400 Subject: [PATCH 015/131] use best header chain instead of best chain for calc next difficulty target --- lib/blockchain.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 9315999b3..6c58827c4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -475,14 +475,19 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - firstNode := lastNode.Ancestor(firstNodeHeight, bc.blockIndex) - if firstNode == nil { + firstNode, exists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(firstNodeHeight)) + if err != nil { + return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ + "beginning of retarget interval at height %d during retarget from height %d", + firstNodeHeight, lastNode.Height) + } + if firstNode == nil || !exists { return nil, fmt.Errorf("CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", firstNodeHeight, lastNode.Height) } - actualTimeDiffSecs := int64(lastNode.Header.GetTstampSecs() - firstNode.Header.GetTstampSecs()) + actualTimeDiffSecs := lastNode.Header.GetTstampSecs() - firstNode.Header.GetTstampSecs() clippedTimeDiffSecs := actualTimeDiffSecs if actualTimeDiffSecs < minRetargetTimeSecs { clippedTimeDiffSecs = minRetargetTimeSecs From 627e882cc0da5853c669fdca58cfc40b8d7cbbf5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 27 Sep 2024 23:22:39 -0400 Subject: [PATCH 016/131] fix tests --- lib/blockchain_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index adcf3f429..224a762f0 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1254,7 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1292,7 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1356,7 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1394,7 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1458,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1516,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1575,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1612,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusNone, )) - bc.bestChain.PushNewTip(nodes[len(nodes)-1]) + bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } From a469c3a1a436b56988147a25493573d5f84212aa Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:07:47 -0400 Subject: [PATCH 017/131] refactor to just use a block hash for best block chain and best header chain representation --- lib/block_view_lockups_test.go | 5 +- lib/block_view_test.go | 10 +- lib/blockchain.go | 727 ++++++++++++++++++++------------- lib/blockchain_test.go | 32 +- lib/db_utils.go | 2 +- lib/pos_blockchain.go | 96 +++-- lib/pos_blockchain_test.go | 99 +++-- lib/pos_consensus_test.go | 2 +- lib/server.go | 143 +++---- lib/state_change_syncer.go | 6 +- lib/txindex.go | 210 +++++----- 11 files changed, 765 insertions(+), 567 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index faaa6a3e3..c4f127892 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2461,8 +2461,9 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, err) require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) + // TODO: make sure this works? // Update the tip - testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] + testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.Parent // Validate the state update utxoView = NewUtxoView( @@ -2517,7 +2518,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.bestChain.Chain = testMeta.chain.bestChain.Chain[:len(testMeta.chain.bestChain.Chain)-1] + testMeta.chain.blockIndex.SetTip(testMeta.chain.blockIndex.tip.Parent) // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index b69307dbf..f02d06e36 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -702,8 +702,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te require.NoError(err) // sanity-check that the last block hash is the same as the last header hash. require.Equal(true, bytes.Equal( - tm.chain.bestChain.GetTip().Hash.ToBytes(), - tm.chain.bestHeaderChain.GetTip().Hash.ToBytes())) + tm.chain.blockIndex.GetTip().Hash.ToBytes(), + tm.chain.blockIndex.GetHeaderTip().Hash.ToBytes())) // Last block shouldn't be nil, and the number of expectedTxns should be the same as in the testVectorBlock + 1, // because of the additional block reward. require.NotNil(lastBlock) @@ -791,10 +791,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.bestChain.Chain = tm.chain.bestChain.Chain[:len(tm.chain.bestChain.Chain)-1] - tm.chain.bestHeaderChain.Chain = tm.chain.bestHeaderChain.Chain[:len(tm.chain.bestHeaderChain.Chain)-1] - tm.chain.bestChain.ChainMap.Remove(*lastBlockHash) - tm.chain.bestHeaderChain.ChainMap.Remove(*lastBlockHash) + tm.chain.blockIndex.SetTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.SetHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 6c58827c4..12ab58072 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -125,7 +125,12 @@ func (nn *BlockNode) IsValidateFailed() bool { // IsCommitted returns true if a BlockNode has passed all validations, and it has been committed to // the Blockchain according to the Fast HotStuff commit rule. func (nn *BlockNode) IsCommitted() bool { - return nn.Status&StatusBlockCommitted != 0 || !blockNodeProofOfStakeCutoverMigrationTriggered(nn.Height) + //return nn.Status&StatusBlockCommitted != 0 || !blockNodeProofOfStakeCutoverMigrationTriggered(nn.Height) + return nn.Status&StatusBlockCommitted != 0 +} + +func (nn *BlockNode) ClearCommittedStatus() { + nn.Status &= BlockStatus(^uint32(StatusBlockCommitted)) } // IsFullyProcessed determines if the BlockStatus corresponds to a fully processed and stored block. @@ -238,6 +243,20 @@ func (nn *BlockNode) GetEncoderType() EncoderType { return EncoderTypeBlockNode } +func (nn *BlockNode) GetParent(blockIndex *BlockIndex) *BlockNode { + if nn.Parent != nil { + return nn.Parent + } + // If we don't have a parent, try to get it from the block index. + parentNode, exists := blockIndex.GetBlockNodeByHashAndHeight(nn.Header.PrevBlockHash, uint64(nn.Height-1)) + if !exists { + return nil + } + + nn.Parent = parentNode + return parentNode +} + // Append DeSo Encoder Metadata bytes to BlockNode bytes. func AddEncoderMetadataToBlockNodeBytes(blockNodeBytes []byte, blockHeight uint64) []byte { var blockData []byte @@ -475,7 +494,10 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - firstNode, exists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(firstNodeHeight)) + // TODO: we need to write the migration to only have committed blocks from PoW. + // This code is dead for PoS. + // TODO: do we need to do something if we need to get this from the header chain? + firstNode, exists, err := bc.GetBlockFromBestChainByHeight(uint64(firstNodeHeight), true) if err != nil { return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ "beginning of retarget interval at height %d during retarget from height %d", @@ -547,18 +569,19 @@ type BlockIndex struct { snapshot *Snapshot blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] - //maxHeightSeen uint64 + tip *BlockNode + headerTip *BlockNode } -func NewBlockIndex(db *badger.DB, snapshot *Snapshot) *BlockIndex { - blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) +func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { + blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ db: db, snapshot: snapshot, - blockIndexByHash: blockIndexByHash, // TODO: parameterize this? - blockIndexByHeight: blockIndexByHeight, // TODO: parameterize this? - //maxHeightSeen: 0, + blockIndexByHash: blockIndexByHash, + blockIndexByHeight: blockIndexByHeight, + tip: tipNode, } } @@ -569,9 +592,28 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { bi.addNewBlockNodeToBlockIndex(val) + // This function is always used for tests. + // We assume that the tip is just the highest block in the block index. + if bi.tip == nil { + bi.tip = val + } else if val.Height > bi.tip.Height { + bi.tip = val + } } } +func (bi *BlockIndex) SetHeaderTip(tip *BlockNode) { + // Just to be safe, we also add it to the block index. + bi.addNewBlockNodeToBlockIndex(tip) + bi.headerTip = tip +} + +func (bi *BlockIndex) SetTip(tip *BlockNode) { + // Just to be safe, we also add it to the block index. + bi.addNewBlockNodeToBlockIndex(tip) + bi.tip = tip +} + func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Add(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) @@ -579,9 +621,10 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight = []*BlockNode{} } else { // Make sure we don't add the same block node twice. - for _, blockAtHeight := range blocksAtHeight { + for ii, blockAtHeight := range blocksAtHeight { if blockAtHeight.Hash.IsEqual(blockNode.Hash) { - return + blocksAtHeight[ii] = blockNode + break } } } @@ -651,147 +694,155 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { return blockNodes } -type BestChain struct { - db *badger.DB - snapshot *Snapshot - IsHeaderChain bool - Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. - ChainMap *lru2.Cache[BlockHash, *BlockNode] - params *DeSoParams -} - -func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { - chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - return &BestChain{ - db: db, - snapshot: snapshot, - IsHeaderChain: isHeaderChain, - Chain: []*BlockNode{}, - ChainMap: chainMap, - params: params, - } -} - -func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { - bestChain.Chain = append(bestChain.Chain, tipNode) - bestChain.ChainMap.Add(*tipNode.Hash, tipNode) +func (bi *BlockIndex) GetTip() *BlockNode { + return bi.tip } -func (bestChain *BestChain) GetTip() *BlockNode { - if len(bestChain.Chain) == 0 { - return nil - } - return bestChain.Chain[len(bestChain.Chain)-1] +func (bi *BlockIndex) GetHeaderTip() *BlockNode { + return bi.headerTip } -func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { - if height > math.MaxUint32 { - return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) - } - - currTip := bestChain.GetTip() - if currTip != nil && height > uint64(currTip.Height) { - return nil, false, nil - } - - if currTip != nil { - currTipHeight := currTip.Height - delta := currTipHeight - uint32(height) - if delta < uint32(len(bestChain.Chain)) { - targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] - if uint64(targetNode.Height) == height { - return targetNode, true, nil - } - } - //targetIndex := (len(bestChain.Chain)-1) - //// We can probably do some binary search thing here instead. - //currNode := &BlockNode{} - //*currNode = *currTip - //if currNode.Height == uint32(height) { - // return currNode, true, nil - //} - //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. - //for currNode != nil && !currNode.IsStored() { - // if currNode.Height < uint32(height) { - // break - // } - // if currNode.Height == uint32(height) { - // return currNode, true, nil - // } - // var currNodeExists bool - // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( - // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) - // if !currNodeExists { - // break - // } - //} - } - - prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) - _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) - if len(valsFound) == 0 { - return nil, false, nil - } - for _, val := range valsFound { - blockNode, err := DeserializeBlockNode(val) - if err != nil { - glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) - continue - } - if blockNode.IsCommitted() { - return blockNode, true, nil - } - } - // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the - // best chain. - return nil, false, nil -} - -func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { - val, exists := bestChain.ChainMap.Get(*blockHash) - if exists { - return val, true - } - if height > math.MaxUint32 { - glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) - } - if height > uint64(bestChain.GetTip().Height) { - return nil, false - } - bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) - if bn == nil { - return nil, false - } - bestChain.ChainMap.Add(*blockHash, bn) - return bn, true -} - -func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { - val, exists := bestChain.ChainMap.Get(*blockHash) - if exists { - return val, true, nil - } - - height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) - if err != nil { - if errors.Is(err, badger.ErrKeyNotFound) { - return nil, false, nil - } - return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") - } - if height > uint64(bestChain.GetTip().Height) { - return nil, false, nil - } - bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) - if bn == nil { - return nil, false, nil - } - if !bn.IsCommitted() { - return nil, false, nil - } - bestChain.ChainMap.Add(*blockHash, bn) - return bn, true, nil -} +//type BestChain struct { +// db *badger.DB +// snapshot *Snapshot +// IsHeaderChain bool +// Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. +// ChainMap *lru2.Cache[BlockHash, *BlockNode] +// params *DeSoParams +//} +// +//func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { +// chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// return &BestChain{ +// db: db, +// snapshot: snapshot, +// IsHeaderChain: isHeaderChain, +// Chain: []*BlockNode{}, +// ChainMap: chainMap, +// params: params, +// } +//} +// +//func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { +// bestChain.Chain = append(bestChain.Chain, tipNode) +// bestChain.ChainMap.Add(*tipNode.Hash, tipNode) +//} +// +//func (bestChain *BestChain) GetTip() *BlockNode { +// if len(bestChain.Chain) == 0 { +// return nil +// } +// return bestChain.Chain[len(bestChain.Chain)-1] +//} +// +//func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { +// if height > math.MaxUint32 { +// return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) +// } +// +// currTip := bestChain.GetTip() +// if currTip != nil && height > uint64(currTip.Height) { +// return nil, false, nil +// } +// +// if currTip != nil { +// currTipHeight := currTip.Height +// delta := currTipHeight - uint32(height) +// if delta < uint32(len(bestChain.Chain)) { +// targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] +// if uint64(targetNode.Height) == height { +// return targetNode, true, nil +// } +// } +// //targetIndex := (len(bestChain.Chain)-1) +// //// We can probably do some binary search thing here instead. +// //currNode := &BlockNode{} +// //*currNode = *currTip +// //if currNode.Height == uint32(height) { +// // return currNode, true, nil +// //} +// //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. +// //for currNode != nil && !currNode.IsStored() { +// // if currNode.Height < uint32(height) { +// // break +// // } +// // if currNode.Height == uint32(height) { +// // return currNode, true, nil +// // } +// // var currNodeExists bool +// // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( +// // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) +// // if !currNodeExists { +// // break +// // } +// //} +// } +// +// prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) +// _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) +// if len(valsFound) == 0 { +// return nil, false, nil +// } +// for _, val := range valsFound { +// blockNode, err := DeserializeBlockNode(val) +// if err != nil { +// glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) +// continue +// } +// if blockNode.IsCommitted() { +// return blockNode, true, nil +// } +// } +// // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the +// // best chain. +// return nil, false, nil +//} +// +//func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { +// val, exists := bestChain.ChainMap.Get(*blockHash) +// if exists { +// return val, true +// } +// if height > math.MaxUint32 { +// glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) +// } +// if height > uint64(bestChain.GetTip().Height) { +// return nil, false +// } +// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) +// if bn == nil { +// return nil, false +// } +// bestChain.ChainMap.Add(*blockHash, bn) +// return bn, true +//} +// +//func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { +// val, exists := bestChain.ChainMap.Get(*blockHash) +// if exists { +// return val, true, nil +// } +// +// height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) +// if err != nil { +// if errors.Is(err, badger.ErrKeyNotFound) { +// return nil, false, nil +// } +// return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") +// } +// if height > uint64(bestChain.GetTip().Height) { +// return nil, false, nil +// } +// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) +// if bn == nil { +// return nil, false, nil +// } +// if !bn.IsCommitted() { +// return nil, false, nil +// } +// bestChain.ChainMap.Add(*blockHash, bn) +// return bn, true, nil +//} type Blockchain struct { db *badger.DB @@ -834,9 +885,9 @@ type Blockchain struct { //bestHeaderChain []*BlockNode //bestHeaderChainMap map[BlockHash]*BlockNode - blockIndex *BlockIndex - bestChain *BestChain - bestHeaderChain *BestChain + blockIndex *BlockIndex + //bestChain *BestChain + //bestHeaderChain *BestChain // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep @@ -1004,29 +1055,29 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { return len(blockNodes) > 0 } -func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { - newBestChain := []*BlockNode{} - newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newBestChain = append(newBestChain, bc.bestChain.Chain...) - for _, key := range bc.bestChain.ChainMap.Keys() { - val, _ := bc.bestChain.ChainMap.Get(key) - newBestChainMap.Add(key, val) - } - - return newBestChain, newBestChainMap -} - -func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { - newBestChain := []*BlockNode{} - newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) - for _, key := range bc.bestHeaderChain.ChainMap.Keys() { - val, _ := bc.bestHeaderChain.ChainMap.Get(key) - newBestChainMap.Add(key, val) - } - - return newBestChain, newBestChainMap -} +//func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { +// newBestChain := []*BlockNode{} +// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// newBestChain = append(newBestChain, bc.bestChain.Chain...) +// for _, key := range bc.bestChain.ChainMap.Keys() { +// val, _ := bc.bestChain.ChainMap.Get(key) +// newBestChainMap.Add(key, val) +// } +// +// return newBestChain, newBestChainMap +//} + +//func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { +// newBestChain := []*BlockNode{} +// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) +// newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) +// for _, key := range bc.bestHeaderChain.ChainMap.Keys() { +// val, _ := bc.bestHeaderChain.ChainMap.Get(key) +// newBestChainMap.Add(key, val) +// } +// +// return newBestChain, newBestChainMap +//} // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { @@ -1034,11 +1085,13 @@ func (bc *Blockchain) IsFullyStored() bool { chainState := bc.ChainState() if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && bc.headerTip().Height-bc.blockTip().Height < 10) { - for _, blockNode := range bc.bestChain.Chain { - if !blockNode.Status.IsFullyProcessed() { - return false - } - } + // TODO: rewrite to work w/ block index. + panic("IsFullyStored: Need to implement this.") + //for _, blockNode := range bc.bestChain.Chain { + // if !blockNode.Status.IsFullyProcessed() { + // return false + // } + //} return true } return false @@ -1142,6 +1195,10 @@ func (bc *Blockchain) _initChain() error { return errors.Wrapf(err, "_initChain: Problem loading block index from db") } + // We start by simply setting the chain tip and header tip to the tip node. + bc.blockIndex.tip = tipNode + bc.blockIndex.headerTip = tipNode + } // At this point the blockIndexByHash should contain a full node tree with all @@ -1149,26 +1206,26 @@ func (bc *Blockchain) _initChain() error { { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestChain.Chain, err = GetBestChain(tipNode) - if err != nil { - return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") - } - for _, bestChainNode := range bc.bestChain.Chain { - bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) - } + //bc.bestChain.Chain, err = GetBestChain(tipNode) + //if err != nil { + // return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") + //} + //for _, bestChainNode := range bc.bestChain.Chain { + // bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) + //} } // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. { // Walk back from the best node to the genesis block and store them all // in bestChain. - bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) - if err != nil { - return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") - } - for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) - } + //bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) + //if err != nil { + // return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") + //} + //for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { + // bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) + //} } bc.isInitialized = true @@ -1216,15 +1273,17 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } + bc.blockIndex.SetTip(uncommittedTipBlockNode) + ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// - currentHeaderTip := bc.headerTip() - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, - bc.bestHeaderChain.ChainMap, - blocksToDetach, - blocksToAttach, - ) + //currentHeaderTip := bc.headerTip() + //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, + // bc.bestHeaderChain.ChainMap, + // blocksToDetach, + // blocksToAttach, + //) return nil } @@ -1273,15 +1332,15 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot), + blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), //bestChainMap: make(map[BlockHash]*BlockNode), - bestChain: NewBestChain(db, snapshot, false, params), + //bestChain: NewBestChain(db, snapshot, false, params), //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - bestHeaderChain: NewBestChain(db, snapshot, true, params), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize - snapshotCache: NewSnapshotCache(), + //bestHeaderChain: NewBestChain(db, snapshot, true, params), + blockViewCache: lru.NewKVCache(100), // TODO: parameterize + snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1346,12 +1405,12 @@ func fastLog2Floor(n uint32) uint8 { // functions. // // This function MUST be called with the chain state lock held (for reads). -func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32, - blockIndex *BlockIndex, bestChain *BestChain) (*BlockNode, uint32) { +// TODO: this function needs a whole bunch of work. +func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32) (*BlockNode, uint32) { // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := blockIndex.GetBlockNodeByHashOnly(stopHash) + stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHash(stopHash, true) if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { // TODO: what should we really do here? @@ -1366,7 +1425,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // Find the most recent locator block hash in the main chain. In the // case none of the hashes in the locator are in the main chain, fall // back to the genesis block. - startNode, startNodeExists, err := bestChain.GetBlockByHeight(0) + startNode, startNodeExists, err := bc.GetBlockFromBestChainByHeight(0, true) if err != nil { glog.Errorf("locateInventory: Problem getting block by height: %v", err) return nil, 0 @@ -1377,12 +1436,8 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 } for _, hash := range locator { // TODO: replace w/ read-through cache call. - node, bestChainContainsNode, err := bestChain.GetBlockByHash(hash) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by hash: %v", err) - continue - } - if bestChainContainsNode { + node := bc.GetBlockNodeWithHash(hash) + if node != nil { startNode = node break } @@ -1392,7 +1447,7 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // is no next block it means the most recently known block is the tip of // the best chain, so there is nothing more to do. nextNodeHeight := uint32(startNode.Header.Height) + 1 - startNode, startNodeExists, err = bestChain.GetBlockByHeight(uint64(nextNodeHeight)) + startNode, startNodeExists, err = bc.GetBlockFromBestChainByHeight(uint64(nextNodeHeight), true) if err != nil { glog.Errorf("locateInventory: Problem getting block by height: %v", err) return nil, 0 @@ -1402,11 +1457,11 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 } // Calculate how many entries are needed. - total := (bestChain.GetTip().Height - startNode.Height) + 1 + total := (bc.blockIndex.GetTip().Height - startNode.Height) + 1 if stopNodeError != nil && stopNodeExists && stopNode != nil && stopNode.Header.Height >= startNode.Header.Height { - _, bestChainContainsStopNode, err := bestChain.GetBlockByHash(stopNode.Hash) + _, bestChainContainsStopNode, err := bc.blockIndex.GetBlockNodeByHashOnly(stopNode.Hash) if err != nil { glog.Errorf("locateInventory: Problem getting block by hash: %v", err) return nil, 0 @@ -1429,14 +1484,12 @@ func locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint3 // See the comment on the exported function for more details on special cases. // // This function MUST be called with the ChainLock held (for reads). -func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, - blockIndex *BlockIndex, bestChain *BestChain) []*MsgDeSoHeader { +func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { // Find the node after the first known block in the locator and the // total number of nodes after it needed while respecting the stop hash // and max entries. - node, total := locateInventory(locator, stopHash, maxHeaders, - blockIndex, bestChain) + node, total := bc.locateInventory(locator, stopHash, maxHeaders) if total == 0 { return nil } @@ -1452,7 +1505,7 @@ func locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32, break } var nodeExists bool - node, nodeExists, err = bestChain.GetBlockByHeight(node.Header.Height + 1) + node, nodeExists, err = bc.GetBlockFromBestChainByHeight(node.Header.Height+1, true) if err != nil { glog.Errorf("locateHeaders: Problem getting block by height: %v", err) break @@ -1487,8 +1540,7 @@ func (bc *Blockchain) LocateBestBlockChainHeaders( // TODO: Shouldn't we hold a ChainLock here? I think it's fine though because the place // where it's currently called is single-threaded via a channel in server.go. Going to // avoid messing with it for now. - headers := locateHeaders(locator, stopHash, maxHeaders, - bc.blockIndex, bc.bestChain) + headers := bc.locateHeaders(locator, stopHash, maxHeaders) return headers } @@ -1550,14 +1602,14 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { // ancestors must be too, so use a much faster O(1) lookup in // that case. Otherwise, fall back to walking backwards through // the nodes of the other chain to the correct ancestor. - _, exists, err := bc.bestHeaderChain.GetBlockByHash(tip.Hash) + _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tip.Hash) if err != nil { glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) exists = false } if exists { var innerExists bool - tip, innerExists, err = bc.bestHeaderChain.GetBlockByHeight(uint64(height)) + tip, innerExists, err = bc.GetBlockFromBestChainByHeight(uint64(height), true) if err != nil { glog.Errorf("LatestLocator: Problem getting block by height: %v", err) break @@ -1614,7 +1666,11 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain := bc.bestHeaderChain.GetBlockByHashAndHeight(bestBlockTip.Hash, uint64(bestBlockTip.Height)) + headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(uint64(bestBlockTip.Height), true) + if err != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) + return nil + } if !blockTipExistsInBestHeaderChain { // If the hash of the tip of the best blockchain is not in the best header chain, then // this is a case where the header chain has forked off from the best block @@ -1635,7 +1691,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( "backward through best header chain; using genesis block") var err error var genesisBlockExists bool - headerNodeStart, genesisBlockExists, err = bc.bestHeaderChain.GetBlockByHeight(0) + headerNodeStart, genesisBlockExists, err = bc.GetBlockFromBestChainByHeight(0, true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting genesis block: %v", err) return nil @@ -1653,14 +1709,14 @@ func (bc *Blockchain) GetBlockNodesToFetch( currentHeight := headerNodeStart.Height + 1 blockNodesToFetch := []*BlockNode{} heightLimit := maxHeight - if heightLimit >= bc.bestHeaderChain.GetTip().Height { - heightLimit = bc.bestHeaderChain.GetTip().Height - 1 + if heightLimit >= bc.blockIndex.GetHeaderTip().Height { + heightLimit = bc.blockIndex.GetHeaderTip().Height - 1 } for currentHeight <= heightLimit && len(blockNodesToFetch) < numBlocks { // Get the current hash and increment the height. - currentNode, currentNodeExists, err := bc.bestHeaderChain.GetBlockByHeight(uint64(currentHeight)) + currentNode, currentNodeExists, err := bc.GetBlockFromBestChainByHeight(uint64(currentHeight), true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) return nil @@ -1697,10 +1753,10 @@ func (bc *Blockchain) HasHeaderByHashAndHeight(headerHash *BlockHash, height uin // TODO: delete me? func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { - if blockHeight >= bc.bestChain.GetTip().Height { + if blockHeight >= bc.blockIndex.GetHeaderTip().Height { return nil, false, nil } - return bc.bestChain.GetBlockByHeight(uint64(blockHeight)) + return bc.GetBlockFromBestChainByHeight(uint64(blockHeight), true) } func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { @@ -1722,8 +1778,8 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32) (*MsgDeSoBlock, error) { - bn, bnExists, err := bc.bestChain.GetBlockByHeight(uint64(height)) +func (bc *Blockchain) GetBlockAtHeight(height uint32, isHeaderChain bool) (*MsgDeSoBlock, error) { + bn, bnExists, err := bc.GetBlockFromBestChainByHeight(uint64(height), isHeaderChain) if !bnExists || err != nil { glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) return nil, err @@ -1737,7 +1793,7 @@ func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { return nil } - bn, bnExists, err := bc.bestChain.GetBlockByHash(hash) + bn, bnExists, err := bc.blockIndex.GetBlockNodeByHashOnly(hash) if !bnExists || err != nil { return nil } @@ -1904,24 +1960,26 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight + _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? // TODO: figure out how to iterate over best chain to checkArchivalMode // when we only have a portion of best chain in memory. - for _, blockNode := range bc.bestChain.Chain { - if uint64(blockNode.Height) > firstSnapshotHeight { - return false - } - - // Check if we have blocks that have been processed and validated but not stored. This would indicate that there - // are historical blocks that we are yet to download. - if (blockNode.Status&StatusBlockProcessed) == 1 && - (blockNode.Status&StatusBlockValidated) == 1 && - (blockNode.Status&StatusBlockStored) == 0 { - - return true - } - } + panic("NOT IMPLEMENTED: checkArchivalMode") + //for _, blockNode := range bc.bestChain.Chain { + // if uint64(blockNode.Height) > firstSnapshotHeight { + // return false + // } + // + // // Check if we have blocks that have been processed and validated but not stored. This would indicate that there + // // are historical blocks that we are yet to download. + // if (blockNode.Status&StatusBlockProcessed) == 1 && + // (blockNode.Status&StatusBlockValidated) == 1 && + // (blockNode.Status&StatusBlockStored) == 0 { + // + // return true + // } + //} // If we get here, it means that all blocks have been processed and stored, so there is nothing to do. return false @@ -1958,7 +2016,7 @@ func (bc *Blockchain) isHyperSyncCondition() bool { // main chain for blocks, which is why separate functions are required for // each of them. func (bc *Blockchain) headerTip() *BlockNode { - return bc.bestHeaderChain.GetTip() + return bc.blockIndex.GetHeaderTip() } func (bc *Blockchain) HeaderTip() *BlockNode { @@ -1990,37 +2048,114 @@ func (bc *Blockchain) Snapshot() *Snapshot { // invalidate and chop off the headers corresponding to those blocks and // their ancestors so the two generally stay in sync. func (bc *Blockchain) blockTip() *BlockNode { - return bc.bestChain.GetTip() + return bc.blockIndex.GetTip() } func (bc *Blockchain) BlockTip() *BlockNode { return bc.blockTip() } +// TODO: this won't work for now. Need to figure out how to handle this. func (bc *Blockchain) BestChain() []*BlockNode { - return bc.bestChain.Chain + panic("BestChain not supported.") } -func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash) (*BlockNode, bool, error) { - return bc.bestChain.GetBlockByHash(blockHash) +func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeaderChain bool) (*BlockNode, bool, error) { + bn, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + if err != nil { + return nil, false, err + } + if !exists { + return nil, false, nil + } + if bn.IsCommitted() { + return bn, true, nil // TODO: what do we do about header chain? they're not committed so we're going to + // have to get a bunch of parents in order to be sure it is part of the best header chain. I guess we could + // have a map, but kinda defeats the purpose of this refactor. + } + blockTip := bc.BlockTip() + if useHeaderChain { + blockTip = bc.HeaderTip() + } + if blockTip == nil { + return nil, false, fmt.Errorf("GetBlockFromBestChainByHash: Block tip not found: use header chain: %v", useHeaderChain) + } + committedTip, exists := bc.GetCommittedTip() + if !exists { + return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") + } + if uint64(bn.Height) > uint64(blockTip.Height) { + return nil, false, nil + } + currNode := &BlockNode{} + *currNode = *blockTip + for currNode != nil && !currNode.Hash.IsEqual(blockHash) && currNode.Height > committedTip.Height { + currNode = currNode.GetParent(bc.blockIndex) + } + if currNode != nil && currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } + return nil, false, nil } -func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64) (*BlockNode, bool, error) { - return bc.bestChain.GetBlockByHeight(height) +func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChain bool) (*BlockNode, bool, error) { + if !useHeaderChain { + committedTip, exists := bc.GetCommittedTip() + if !exists { + return nil, false, nil + } + if height >= uint64(committedTip.Height) { + // For this, we can just loop back from the tip block. + currentNode := bc.blockIndex.GetTip() + if useHeaderChain { + currentNode = bc.blockIndex.GetHeaderTip() + } + for currentNode != nil { + if uint64(currentNode.Height) == height { + return currentNode, true, nil + } + if currentNode.Height < committedTip.Height { + break + } + currentNode = currentNode.GetParent(bc.blockIndex) + } + return nil, false, nil + } + } + blockNodes := bc.blockIndex.GetBlockNodesByHeight(height) + if len(blockNodes) == 0 { + return nil, false, nil + } + for _, blockNode := range blockNodes { + if !useHeaderChain && blockNode.IsCommitted() { + return blockNode, true, nil + } + // TODO: this is crude and incorrect. + if useHeaderChain && blockNode.IsHeaderValidated() { + return blockNode, true, nil + } + } + return nil, false, nil } +// TODO: need to figure out how to handle this for exchange api tests. func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { - bc.bestChain.Chain = bestChain + for _, blockNode := range bestChain { + bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) + if bc.blockIndex.GetTip() == nil { + bc.blockIndex.SetTip(blockNode) + } else if bc.blockIndex.GetTip().Height < blockNode.Height { + bc.blockIndex.SetTip(blockNode) + } + } } func (bc *Blockchain) SetBestChainMap( - bestChain []*BlockNode, - bestChainMap *lru2.Cache[BlockHash, *BlockNode], blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], + tipNode *BlockNode, ) { - bc.bestChain.Chain = bestChain - bc.bestChain.ChainMap = bestChainMap bc.blockIndex.blockIndexByHash = blockIndexByHash + bc.blockIndex.SetTip(tipNode) } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2293,6 +2428,9 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( "block (%v) at height (%d) to block (%v) at height of (%d)", numBlocks, tip, tip.Height, newNode, newNode.Height) } + } else { + glog.Fatal("GetReorgBlocks: Tip is nil") + return } // Get the blocks to detach. Start at the tip and work backwards to the @@ -2537,10 +2675,10 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B headerTip := bc.headerTip() if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - - _, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) + bc.blockIndex.SetHeaderTip(newNode) + //_, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) // Note that we don't store the best header hash here and so this is an // in-memory-only adjustment. See the comment above on preventing attacks. @@ -2963,6 +3101,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // update our data structures to actually make this connection. Do this // in a transaction so that it is atomic. if bc.postgres != nil { + // TODO: pg support for setting committed on block node. if err = bc.postgres.UpsertBlockAndTransactions(nodeToValidate, desoBlock); err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem upserting block and transactions") } @@ -2984,6 +3123,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures err = bc.db.Update(func(txn *badger.Txn) error { // This will update the node's status. bc.timer.Start("Blockchain.ProcessBlock: Transactions Db height & hash") + if !nodeToValidate.IsCommitted() { + nodeToValidate.Status |= StatusBlockCommitted + bc.blockIndex.addNewBlockNodeToBlockIndex(nodeToValidate) + } if innerErr := PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, nodeToValidate, false /*bitcoinNodes*/, bc.eventManager); innerErr != nil { return errors.Wrapf( innerErr, "ProcessBlock: Problem calling PutHeightHashToNodeInfo after validation") @@ -3030,8 +3173,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now that we've set the best chain in the db, update our in-memory data // structure to reflect this. Do a quick check first to make sure it's consistent. - lastIndex := len(bc.bestChain.Chain) - 1 - bestChainHash := bc.bestChain.Chain[lastIndex].Hash + bestChainTip := bc.blockIndex.GetTip() + if bestChainTip == nil { + return false, false, fmt.Errorf("ProcessBlock: Best chain tip is nil") + } + bestChainHash := bestChainTip.Hash if !bestChainHash.IsEqual(nodeToValidate.Header.PrevBlockHash) { return false, false, fmt.Errorf("ProcessBlock: Last block in bestChain "+ @@ -3042,11 +3188,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.bestChain.PushNewTip(nodeToValidate) + bc.blockIndex.SetTip(nodeToValidate) } else { - newBestChain, newBestChainMap := bc.CopyBestChain() - bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap - bc.bestChain.PushNewTip(nodeToValidate) + //newBestChain, newBestChainMap := bc.CopyBestChain() + //bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.blockIndex.SetTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3271,6 +3417,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if err := PutBestHashWithTxn(txn, bc.snapshot, newTipNode.Hash, ChainTypeDeSoBlock, bc.eventManager); err != nil { return err } + if !newTipNode.IsCommitted() { + newTipNode.Status |= StatusBlockCommitted + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(newTipNode) + if err := PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, newTipNode, false, bc.eventManager); err != nil { + return err + } + } for _, detachNode := range detachBlocks { // Delete the utxo operations for the blocks we're detaching since we don't need @@ -3279,6 +3433,16 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures return errors.Wrapf(err, "ProcessBlock: Problem deleting utxo operations for block") } + // We also need to revert the committed state if applicable. + if detachNode.IsCommitted() { + detachNode.ClearCommittedStatus() + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(detachNode) + if err = PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, detachNode, false, bc.eventManager); err != nil { + return errors.Wrapf(err, "ProcessBlock: Problem putting height hash to node info for detach node that is not committed.") + } + } + // Note we could be even more aggressive here by deleting the nodes and // corresponding blocks from the db here (i.e. not storing any side chain // data on the db). But this seems like a minor optimization that comes at @@ -3291,6 +3455,15 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if err := PutUtxoOperationsForBlockWithTxn(txn, bc.snapshot, blockHeight, attachNode.Hash, utxoOpsForAttachBlocks[ii], bc.eventManager); err != nil { return errors.Wrapf(err, "ProcessBlock: Problem putting utxo operations for block") } + + if !attachNode.IsCommitted() { + attachNode.Status |= StatusBlockCommitted + // update the block index to be safe. + bc.addNewBlockNodeToBlockIndex(attachNode) + if err = PutHeightHashToNodeInfoWithTxn(txn, bc.snapshot, attachNode, false, bc.eventManager); err != nil { + return errors.Wrapf(err, "ProcessBlock: Problem putting height hash to node info for detach node that is not committed.") + } + } } // Write the modified utxo set to the view. @@ -3307,10 +3480,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now the db has been updated, update our in-memory best chain. Note that there // is no need to update the node index because it was updated as we went along. - newBestChain, newBestChainMap := bc.CopyBestChain() - newBestChain, newBestChainMap = updateBestChainInMemory( - newBestChain, newBestChainMap, detachBlocks, attachBlocks) - bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap + bc.blockIndex.SetTip(newTipNode) + bc.blockIndex.SetHeaderTip(newTipNode) // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 224a762f0..2bf1cab3d 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1252,9 +1252,9 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1290,9 +1290,9 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { // Blocks generating every 4 second, which is 2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1354,9 +1354,9 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1392,9 +1392,9 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { // Blocks generating every 8 second, which is >2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1456,9 +1456,9 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { // Blocks generating all at once. TstampNanoSecs: SecondsToNanoSeconds(0), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1514,9 +1514,9 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { // Blocks generating every 2 second, which is under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 2)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1573,9 +1573,9 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1610,9 +1610,9 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { // Blocks generating every 3 seconds, which is slow but under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii) * 3), }, - StatusNone, + StatusHeaderValidated, )) - bc.bestHeaderChain.PushNewTip(nodes[len(nodes)-1]) + bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/db_utils.go b/lib/db_utils.go index cc3e4faee..de0de1bf8 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5360,7 +5360,7 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB, diffTarget, BytesToBigint(ExpectedWorkForBlockHash(diffTarget)[:]), // CumWork genesisBlock.Header, // Header - StatusHeaderValidated|StatusBlockProcessed|StatusBlockStored|StatusBlockValidated, // Status + StatusHeaderValidated|StatusBlockProcessed|StatusBlockStored|StatusBlockValidated|StatusBlockCommitted, // Status ) // Set the fields in the db to reflect the current state of our chain. diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 134505eab..7b5dd2c86 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -58,7 +58,13 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - if _, isInBestHeaderChain := bc.bestHeaderChain.ChainMap.Get(*headerHash); isInBestHeaderChain { + // TODO: make sure this is ok or do we need to explicitly check the block index's cache? + var isInBestHeaderChain bool + _, isInBestHeaderChain, err = bc.GetBlockFromBestChainByHash(headerHash, true) + if err != nil { + return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") + } + if isInBestHeaderChain { return true, false, nil } @@ -91,15 +97,17 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, nil } + bc.blockIndex.SetHeaderTip(blockNode) + // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. - _, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) - bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - bc.bestHeaderChain.Chain, - bc.bestHeaderChain.ChainMap, - blocksToDetach, - blocksToAttach, - ) + //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) + //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( + // bc.bestHeaderChain.Chain, + // bc.bestHeaderChain.ChainMap, + // blocksToDetach, + // blocksToAttach, + //) // Success. The header is at the tip of the best header chain. return true, false, nil @@ -1320,8 +1328,8 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( _missingBlockHashes []*BlockHash, _err error, ) { - highestCommittedBlock, idx := bc.GetCommittedTip() - if idx == -1 || highestCommittedBlock == nil { + highestCommittedBlock, exists := bc.GetCommittedTip() + if !exists || highestCommittedBlock == nil { return nil, nil, errors.New("getStoredLineageFromCommittedTip: No committed blocks found") } currentHash := header.PrevBlockHash.NewBlockHash() @@ -1642,7 +1650,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.bestChain.PushNewTip(blockNode) + bc.blockIndex.SetTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1651,9 +1659,9 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { // the bestChain slice and bestChainMap map. func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. - lastBlock := bc.bestChain.Chain[len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Remove(*lastBlock.Hash) - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] + lastBlock := bc.blockIndex.GetTip() + // Uhhh what happens if we don't have the parent set up!? + bc.blockIndex.SetTip(lastBlock.GetParent(bc.blockIndex)) return lastBlock } @@ -1664,24 +1672,25 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { currentBlock := bc.BlockTip() // If we can commit the grandparent, commit it. // Otherwise, we can't commit it and return nil. - blockToCommit, canCommit := bc.canCommitGrandparent(currentBlock) + blockNodeToCommit, canCommit := bc.canCommitGrandparent(currentBlock) if !canCommit { return nil } // Find all uncommitted ancestors of block to commit - _, idx := bc.GetCommittedTip() - if idx == -1 { + committedTip, exists := bc.GetCommittedTip() + if !exists { // This is an edge case we'll never hit in practice since all the PoW blocks // are committed. return errors.New("runCommitRuleOnBestChain: No committed blocks found") } uncommittedAncestors := []*BlockNode{} - for ii := idx + 1; ii < len(bc.bestChain.Chain); ii++ { - uncommittedAncestors = append(uncommittedAncestors, bc.bestChain.Chain[ii]) - if bc.bestChain.Chain[ii].Hash.IsEqual(blockToCommit) { - break - } + currentNode := &BlockNode{} + *currentNode = *blockNodeToCommit + for currentNode.Height > committedTip.Height { + uncommittedAncestors = append(uncommittedAncestors, currentNode) + currentNode = currentNode.GetParent(bc.blockIndex) } + uncommittedAncestors = collections.Reverse(uncommittedAncestors) for ii := 0; ii < len(uncommittedAncestors); ii++ { if err := bc.commitBlockPoS(uncommittedAncestors[ii].Hash, uint64(uncommittedAncestors[ii].Height), verifySignatures); err != nil { return errors.Wrapf(err, @@ -1696,16 +1705,18 @@ func (bc *Blockchain) runCommitRuleOnBestChain(verifySignatures bool) error { // between the grandparent and parent of the new block, meaning the grandparent and parent // are proposed in consecutive views, and the "parent" is an ancestor of the incoming block // (not necessarily consecutive views). Additionally, the grandparent must not already be committed. -func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparentBlockHash *BlockHash, _canCommit bool, +func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) ( + _grandparentBlockNode *BlockNode, + _canCommit bool, ) { // TODO: Is it sufficient that the current block's header points to the parent // or does it need to have something to do with the QC? - parent, exists := bc.bestChain.GetBlockByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) + parent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) if !exists { glog.Errorf("canCommitGrandparent: Parent block %v not found in best chain map", currentBlock.Header.PrevBlockHash.String()) return nil, false } - grandParent, exists := bc.bestChain.GetBlockByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) + grandParent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) if !exists { glog.Errorf("canCommitGrandparent: Grandparent block %v not found in best chain map", parent.Header.PrevBlockHash.String()) return nil, false @@ -1715,7 +1726,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen } if grandParent.Header.ProposedInView+1 == parent.Header.ProposedInView { // Then we can run the commit rule up to the grandparent! - return grandParent.Hash, true + return grandParent, true } return nil, false } @@ -1725,7 +1736,7 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) (_grandparen // to the DB and updates relevant badger indexes with info about the block. func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, verifySignatures bool) error { // block must be in the best chain. we grab the block node from there. - blockNode, exists := bc.bestChain.GetBlockByHashAndHeight(blockHash, blockHeight) + blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeight) if !exists { return errors.Errorf("commitBlockPoS: Block %v not found in best chain map", blockHash.String()) } @@ -1854,7 +1865,7 @@ func (bc *Blockchain) GetUncommittedBlocks(tipHash *BlockHash) ([]*BlockNode, er } bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() - tipBlock, exists, err := bc.bestChain.GetBlockByHash(tipHash) + tipBlock, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tipHash) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedBlocks: Problem getting block %v", tipHash.String()) } @@ -1946,8 +1957,9 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: Block %v not found in block index", blockHash) } - highestCommittedBlock, _ := bc.GetCommittedTip() - if highestCommittedBlock == nil { + highestCommittedBlock, exists := bc.GetCommittedTip() + glog.Infof("Highest committed block: %v", highestCommittedBlock) + if !exists || highestCommittedBlock == nil { return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") } // If the provided block is committed, we need to make sure it's the committed tip. @@ -1972,10 +1984,6 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc return nil, errors.Errorf( "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") } - if currentBlock.IsCommitted() && !currentBlock.Hash.IsEqual(highestCommittedBlock.Hash) { - return nil, errors.Errorf( - "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") - } } viewAndUtxoOpsAtHash, err, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) if err != nil { @@ -1994,6 +2002,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { + glog.Infof("Connecting block %v", uncommittedAncestors[ii]) // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) if err != nil { @@ -2027,13 +2036,16 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc } // GetCommittedTip returns the highest committed block and its index in the best chain. -func (bc *Blockchain) GetCommittedTip() (*BlockNode, int) { - for ii := len(bc.bestChain.Chain) - 1; ii >= 0; ii-- { - if bc.bestChain.Chain[ii].IsCommitted() { - return bc.bestChain.Chain[ii], ii +func (bc *Blockchain) GetCommittedTip() (*BlockNode, bool) { + // iterate backwards from the tip node + currentNode := bc.blockIndex.GetTip() + for currentNode != nil { + if currentNode.IsCommitted() { + return currentNode, true } + currentNode = currentNode.GetParent(bc.blockIndex) } - return nil, -1 + return nil, false } // GetSafeBlocks returns all headers of blocks from which the chain can safely extend. @@ -2058,9 +2070,9 @@ func (bc *Blockchain) GetSafeBlocks() ([]*MsgDeSoHeader, error) { func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { // First get committed tip. - committedTip, idx := bc.GetCommittedTip() - if idx == -1 || committedTip == nil { - return nil, errors.New("getSafeBlockNodes: No committed blocks found") + committedTip, exists := bc.GetCommittedTip() + if !exists || committedTip == nil { + return []*BlockNode{}, nil } // Now get all blocks from the committed tip to the best chain tip. safeBlocks := []*BlockNode{committedTip} diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 47d1147c1..888ee1f9b 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -2,6 +2,7 @@ package lib import ( "bytes" + "encoding/hex" "fmt" "math" "math/rand" @@ -251,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain.Chain = []*BlockNode{genesisBlock} + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -301,7 +302,8 @@ func TestHasValidBlockHeight(t *testing.T) { require.Equal(t, err, RuleErrorInvalidPoSBlockHeight) block.Header.Height = 2 - bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot) + // TODO: make sure setting to genesis block works. + bc.blockIndex = NewBlockIndex(bc.db, bc.snapshot, genesisBlock) err = bc.hasValidBlockHeightPoS(block.Header) require.Equal(t, err, RuleErrorMissingParentBlock) } @@ -471,10 +473,6 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.bestChain.Chain = []*BlockNode{ - genesisNode, - block2, - } bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, @@ -807,7 +805,6 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain.Chain = []*BlockNode{genesisNode} bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) @@ -844,7 +841,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.bestChain.Chain = append(bc.bestChain.Chain, block2) + bc.blockIndex.SetTip(block2) bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1235,21 +1232,27 @@ func TestShouldReorg(t *testing.T) { hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) hash3 := NewBlockHash(RandomBytes(32)) - bc.bestChain.Chain = []*BlockNode{ + chain := []*BlockNode{ { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, + Height: 0, }, { Hash: hash3, Status: StatusBlockStored | StatusBlockValidated, + Height: 1, }, } + bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + *hash1: chain[0], + *hash3: chain[1], + }) newBlock := &BlockNode{ Header: &MsgDeSoHeader{ ProposedInView: 2, - PrevBlockHash: bc.bestChain.Chain[1].Hash, + PrevBlockHash: chain[1].Hash, }, } @@ -1335,44 +1338,47 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - return collections.Any(bc.bestChain.Chain, func(bn *BlockNode) bool { - return bn.Hash.IsEqual(hash) - }) + _, exists, err := bc.GetBlockFromBestChainByHash(hash, false) + require.NoError(t, err) + return exists } // Try to apply newBlock as tip. This should succeed. newBlockNode := &BlockNode{ Header: newBlock.Header, Hash: newBlockHash, + Height: 4, } appliedNewTip, connectedBlockHashes, disconnectedBlockHashes, err := bc.tryApplyNewTip(newBlockNode, 9, ancestors) require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash3) + _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash3, false) + require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) require.Len(t, connectedBlockHashes, 1) require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap := bc.bestChain.ChainMap.Get(*newBlockHash) + _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(newBlockHash, false) + require.NoError(t, err) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash2) + _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash2, false) + require.NoError(t, err) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) - _, hash1ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash1) + _, hash1ExistsInBestChainMap := bc.blockIndex.blockIndexByHash.Get(*hash1) require.True(t, hash1ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-1] - bc.bestChain.ChainMap.Remove(*newBlockHash) + bc.blockIndex.SetTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1409,6 +1415,8 @@ func TestTryApplyNewTip(t *testing.T) { newBlockNode.Header.ProposedInView = 7 newBlockNode.Header.Height = 7 newBlockNode.Height = 7 + // Clear parent out for safety. + newBlockNode.Parent = nil require.NoError(t, err) ancestors, _, err = bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) @@ -1420,19 +1428,23 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash3) + _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash3, false) + require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap = bc.bestChain.ChainMap.Get(*hash2) + _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash2, false) + require.NoError(t, err) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash4) + _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash4, false) + require.NoError(t, err) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap := bc.bestChain.ChainMap.Get(*hash5) + _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash5, false) + require.NoError(t, err) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1440,11 +1452,12 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, connectedBlockHashes, 3) require.Len(t, disconnectedBlockHashes, 2) - // Reset the state of the best chain. - bc.bestChain.ChainMap.Remove(*hash4) - bc.bestChain.ChainMap.Remove(*hash5) - bc.bestChain.ChainMap.Remove(*newBlockHash) - bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] + // Reset the state of the best chain - parent of newBlock + //bc.bestChain.ChainMap.Remove(*hash4) + //bc.bestChain.ChainMap.Remove(*hash5) + //bc.bestChain.ChainMap.Remove(*newBlockHash) + //bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] + bc.blockIndex.SetTip(newBlockNode.GetParent(bc.blockIndex)) // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) @@ -1514,8 +1527,8 @@ func TestCanCommitGrandparent(t *testing.T) { PrevBlockHash: hash1, }, } - bc.bestChain.ChainMap.Add(*hash1, bn1) - bc.bestChain.ChainMap.Add(*hash2, bn2) + bc.blockIndex.addNewBlockNodeToBlockIndex(bn1) + bc.blockIndex.addNewBlockNodeToBlockIndex(bn2) // define incoming block hash3 := NewBlockHash(RandomBytes(32)) @@ -1534,7 +1547,7 @@ func TestCanCommitGrandparent(t *testing.T) { // (meaning they are in consecutive views). So we should be able // to commit bn1. grandparentHash, canCommit := bc.canCommitGrandparent(bn3) - require.True(t, hash1.IsEqual(grandparentHash)) + require.True(t, hash1.IsEqual(grandparentHash.Hash)) require.True(t, canCommit) // Update bn1 to be committed. We no longer can run the commit since bn1 is already @@ -1609,7 +1622,7 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { // Add one more block to the best chain, but have the view be further in the future. // this should trigger a commit on block 2. - blockTemplate4 := _generateBlockAndAddToBestChain(testMeta, 14, 20, 429) + blockTemplate4 := _generateBlockAndAddToBestChain(testMeta, 15, 20, 429) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1621,7 +1634,7 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { // Okay so add block 5 to the best chain. This should NOT trigger a commit on block 3 // as block 4 is not a direct child of block 3 based on its view. - blockTemplate5 := _generateBlockAndAddToBestChain(testMeta, 15, 21, 654) + blockTemplate5 := _generateBlockAndAddToBestChain(testMeta, 16, 21, 654) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1632,9 +1645,9 @@ func TestRunCommitRuleOnBestChain(t *testing.T) { _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, blockHash4, blockHash5}, blockHash2) // If we now add a block that is a descendent of block 5, we should be able to commit - // blocks 3 and 4 as block 4 and 5 possess a direct parent child relationship and + // blocks 3 and 4 as block 4 and 5 possess a direct parent child relationship, and // we have a descendent of block 5. - blockTemplate6 := _generateBlockAndAddToBestChain(testMeta, 16, 22, 912) + blockTemplate6 := _generateBlockAndAddToBestChain(testMeta, 17, 22, 912) require.NoError(t, err) err = testMeta.chain.runCommitRuleOnBestChain(true) require.NoError(t, err) @@ -1657,7 +1670,8 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*committedHash) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(committedHash, false) + require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1682,7 +1696,8 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists := testMeta.chain.bestChain.ChainMap.Get(*uncommittedBlockHash) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(uncommittedBlockHash, false) + require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) // TODO: Verify DB results?? Kinda silly to make sure everything is missing. @@ -1874,7 +1889,8 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists := testMeta.chain.bestChain.ChainMap.Get(*timeoutBlockHash) + _, exists, err := testMeta.chain.GetBlockFromBestChainByHash(timeoutBlockHash, false) + require.NoError(t, err) require.False(t, exists) timeoutBlockNode, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(timeoutBlockHash, timeoutBlockHeight) @@ -1987,6 +2003,7 @@ func TestGetSafeBlocks(t *testing.T) { testMeta := NewTestPoSBlockchainWithValidators(t) committedHash := testMeta.chain.BlockTip().Hash var block1 *MsgDeSoBlock + fmt.Println("HEX STUFF: ", hex.EncodeToString(committedHash[:])) block1 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight), uint64(testMeta.savedHeight), 1723, committedHash, false) block1Hash, err := block1.Hash() require.NoError(t, err) @@ -2542,7 +2559,10 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se // _generateBlockAndAddToBestChain generates a BlockTemplate by calling _generateRealBlock and then adds it to the // best chain. Finally it updates the PosMempool's latest block view. func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, view uint64, seed int64) *MsgDeSoBlock { - blockTemplate := _generateRealBlock(testMeta, blockHeight, view, seed, testMeta.chain.BlockTip().Hash, false) + prevBlockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHeight(blockHeight-1, false) + require.NoError(testMeta.t, err) + require.True(testMeta.t, exists) + blockTemplate := _generateRealBlock(testMeta, blockHeight, view, seed, prevBlockNode.Hash, false) var msgDesoBlock *MsgDeSoBlock msgDesoBlock = blockTemplate newBlockHash, err := msgDesoBlock.Hash() @@ -2824,6 +2844,7 @@ func NewTestPoSBlockchainWithValidators(t *testing.T) *TestMeta { for ii := 0; ii < 10; ii++ { _, err := miner.MineAndProcessSingleBlock(0 /*threadIndex*/, oldPool) require.NoError(t, err) + fmt.Println("CHAIN TIP: ", chain.BlockTip().Hash) } m0PubBytes, _, _ := Base58CheckDecode(m0Pub) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 74ab70dca..9d471b2a5 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -102,7 +102,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { currentView := blockHeader.ValidatorsVoteQC.GetView() + 1 nextView := currentView + 1 - blockIndex := NewBlockIndex(nil, nil) + blockIndex := NewBlockIndex(nil, nil, nil) blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) diff --git a/lib/server.go b/lib/server.go index 71f003c8b..997f63785 100644 --- a/lib/server.go +++ b/lib/server.go @@ -885,58 +885,58 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. - for _, blockNode := range srv.blockchain.bestChain.Chain { - // We find the first block that's not stored and get ready to download blocks starting from this block onwards. - if blockNode.Status&StatusBlockStored == 0 { - maxBlocksInFlight := MaxBlocksInFlight - if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && - (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || - srv.params.NetworkType == NetworkType_TESTNET) { - - maxBlocksInFlight = MaxBlocksInFlightPoS - } - numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - currentHeight := int(blockNode.Height) - blockNodesToFetch := []*BlockNode{} - // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. - var heightLimit int - for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { - if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { - break - } - } - - // Find the blocks that we should download. - for currentHeight <= heightLimit && - len(blockNodesToFetch) < numBlocksToFetch { - - // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to - // the array index. - currentNode := srv.blockchain.bestChain.Chain[currentHeight] - currentHeight++ - - // If we've already requested this block then we don't request it again. - if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { - continue - } - - blockNodesToFetch = append(blockNodesToFetch, currentNode) - } - - var hashList []*BlockHash - for _, node := range blockNodesToFetch { - hashList = append(hashList, node.Hash) - pp.requestedBlocks[*node.Hash] = true - } - pp.AddDeSoMessage(&MsgDeSoGetBlocks{ - HashList: hashList, - }, false) - - glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", - blockNode.Header, pp) - return - } - } + //for _, blockNode := range srv.blockchain.bestChain.Chain { + // // We find the first block that's not stored and get ready to download blocks starting from this block onwards. + // if blockNode.Status&StatusBlockStored == 0 { + // maxBlocksInFlight := MaxBlocksInFlight + // if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && + // (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || + // srv.params.NetworkType == NetworkType_TESTNET) { + // + // maxBlocksInFlight = MaxBlocksInFlightPoS + // } + // numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) + // currentHeight := int(blockNode.Height) + // blockNodesToFetch := []*BlockNode{} + // // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. + // var heightLimit int + // for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { + // if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { + // break + // } + // } + // + // // Find the blocks that we should download. + // for currentHeight <= heightLimit && + // len(blockNodesToFetch) < numBlocksToFetch { + // + // // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to + // // the array index. + // currentNode := srv.blockchain.bestChain.Chain[currentHeight] + // currentHeight++ + // + // // If we've already requested this block then we don't request it again. + // if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { + // continue + // } + // + // blockNodesToFetch = append(blockNodesToFetch, currentNode) + // } + // + // var hashList []*BlockHash + // for _, node := range blockNodesToFetch { + // hashList = append(hashList, node.Hash) + // pp.requestedBlocks[*node.Hash] = true + // } + // pp.AddDeSoMessage(&MsgDeSoGetBlocks{ + // HashList: hashList, + // }, false) + // + // glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", + // blockNode.Header, pp) + // return + // } + //} // If we get here then it means that we've downloaded all blocks so we can update srv.blockchain.downloadingHistoricalBlocks = false @@ -1038,12 +1038,16 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } var hasSeenCheckpointBlockHash bool var checkpointBlockNode *BlockNode + var err error if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, true) } else { - checkpointBlockNode, hasSeenCheckpointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, false) + } + if err != nil { + glog.Fatalf("shouldVerifySignatures: Problem getting checkpoint block node from best chain: %v", err) } // If we haven't seen the checkpoint block hash yet, we skip signature verification. if !hasSeenCheckpointBlockHash { @@ -1071,13 +1075,11 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { if checkpointBlockInfo == nil { return "" } - hasSeenCheckPointBlockHash := false - if isHeaders { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestHeaderChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) - } else { - _, hasSeenCheckPointBlockHash = srv.blockchain.bestChain.GetBlockByHashAndHeight( - checkpointBlockInfo.Hash, checkpointBlockInfo.Height) + _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHash( + checkpointBlockInfo.Hash, isHeaders) + + if err != nil { + glog.Fatalf("getCheckpointSyncingStatus: Problem getting checkpoint block node from best chain: %v", err) } if !hasSeenCheckPointBlockHash { return fmt.Sprintf("", checkpointBlockInfo.String()) @@ -1258,7 +1260,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // snapshot we receive from the peer is up-to-date. // TODO: error handle if the hash doesn't exist for some reason. expectedSnapshotHeightBlock, expectedSnapshotHeightblockExists, err := - srv.blockchain.bestHeaderChain.GetBlockByHeight(expectedSnapshotHeight) + srv.blockchain.GetBlockFromBestChainByHeight(expectedSnapshotHeight, true) if err != nil { glog.Errorf("Server._handleHeaderBundle: Problem getting expected snapshot height block, error (%v)", err) return @@ -1404,8 +1406,9 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) - glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (chain map: %v) - (chain: %v) ", - srv.blockchain.bestHeaderChain.ChainMap.Len(), len(srv.blockchain.bestHeaderChain.Chain)) + // TODO: this may be wrong? + glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (header tip height: %v) ", + srv.blockchain.blockIndex.GetHeaderTip()) } func (srv *Server) _handleGetBlocks(pp *Peer, msg *MsgDeSoGetBlocks) { @@ -1677,7 +1680,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.snapshot.PrintChecksum("Finished hyper sync. Checksum is:") glog.Infof(CLog(Magenta, fmt.Sprintf("Metadata checksum: (%v)", srv.HyperSyncProgress.SnapshotMetadata.CurrentEpochChecksumBytes))) - blockNode, exists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(msg.SnapshotMetadata.SnapshotBlockHeight) + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(msg.SnapshotMetadata.SnapshotBlockHeight, true) if err != nil { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error (%v)", err) return @@ -1687,7 +1690,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { //return } else { glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", - blockNode, srv.blockchain.bestChain.Chain))) + blockNode, srv.blockchain.blockIndex.GetTip()))) } // Verify that the state checksum matches the one in HyperSyncProgress snapshot metadata. // If the checksums don't match, it means that we've been interacting with a peer that was misbehaving. @@ -1730,7 +1733,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. var blockNodeBatch []*BlockNode for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode, currentNodeExists, err := srv.blockchain.bestHeaderChain.GetBlockByHeight(ii) + currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(ii, true) if err != nil { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) break @@ -1744,7 +1747,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.bestChain.PushNewTip(currentNode) + srv.blockchain.blockIndex.SetTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 0172e30f9..47832000a 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -777,7 +777,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser return true, nil } - blockHeight := uint64(server.blockchain.bestChain.GetTip().Height) + blockHeight := uint64(server.blockchain.blockIndex.GetTip().Height) stateChangeSyncer.MempoolFlushId = originalCommittedFlushId @@ -804,7 +804,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser mempoolUtxoView.Snapshot = nil server.blockchain.ChainLock.RLock() - mempoolUtxoView.TipHash = server.blockchain.bestChain.GetTip().Hash + mempoolUtxoView.TipHash = server.blockchain.blockIndex.GetTip().Hash server.blockchain.ChainLock.RUnlock() // A new transaction is created so that we can simulate writes to the db without actually writing to the db. @@ -815,7 +815,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser defer txn.Discard() glog.V(2).Infof("Time since mempool sync start: %v", time.Since(startTime)) startTime = time.Now() - err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.bestChain.GetTip().Height)) + err = mempoolUtxoView.FlushToDbWithTxn(txn, uint64(server.blockchain.blockIndex.GetTip().Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 904211819..d3794d935 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -150,8 +150,8 @@ func NewTXIndex(coreChain *Blockchain, params *DeSoParams, dataDirectory string) } func (txi *TXIndex) FinishedSyncing() bool { - committedTip, idx := txi.CoreChain.GetCommittedTip() - if idx == -1 { + committedTip, exists := txi.CoreChain.GetCommittedTip() + if !exists { return false } return txi.TXIndexChain.BlockTip().Height == committedTip.Height @@ -208,8 +208,7 @@ func (txi *TXIndex) Stop() { // GetTxindexUpdateBlockNodes ... func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( - _txindexTipNode *BlockNode, _blockTipNode *BlockNode, _commonAncestor *BlockNode, - _detachBlocks []*BlockNode, _attachBlocks []*BlockNode) { + _txindexTipNode *BlockNode, _blockTipNode *BlockNode, _commonAncestor *BlockNode) { // Get the current txindex tip. txindexTipHash := txi.TXIndexChain.BlockTip() @@ -219,7 +218,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // case. glog.Error("Error: TXIndexChain had nil tip; this should never " + "happen and it means the transaction index is broken.") - return nil, nil, nil, nil, nil + return nil, nil, nil } // If the tip of the txindex is no longer stored in the block index, it // means the txindex hit a fork that we are no longer keeping track of. @@ -230,22 +229,7 @@ func (txi *TXIndex) GetTxindexUpdateBlockNodes() ( // Get the committed tip. committedTip, _ := txi.CoreChain.GetCommittedTip() - if txindexTipNode == nil { - glog.Info("GetTxindexUpdateBlockNodes: Txindex tip was not found; building txindex starting at genesis block") - - newTxIndexBestChain, _ := txi.TXIndexChain.CopyBestChain() - newBlockchainBestChain, _ := txi.CoreChain.CopyBestChain() - - return txindexTipNode, committedTip, nil, newTxIndexBestChain, newBlockchainBestChain - } - - derefedTxindexTipNode := *txindexTipNode - - // At this point, we know our txindex tip is in our block index so - // there must be a common ancestor between the tip and the block tip. - commonAncestor, detachBlocks, attachBlocks := txi.CoreChain.GetReorgBlocks(&derefedTxindexTipNode, committedTip) - - return txindexTipNode, committedTip, commonAncestor, detachBlocks, attachBlocks + return txindexTipNode, committedTip, txindexTipNode } // Update syncs the transaction index with the blockchain. @@ -265,7 +249,7 @@ func (txi *TXIndex) Update() error { // done with the rest of the function. txi.TXIndexLock.Lock() defer txi.TXIndexLock.Unlock() - txindexTipNode, blockTipNode, commonAncestor, detachBlocks, attachBlocks := txi.GetTxindexUpdateBlockNodes() + txindexTipNode, blockTipNode, commonAncestor := txi.GetTxindexUpdateBlockNodes() // Note that the blockchain's ChainLock does not need to be held at this // point because we're just reading blocks from the db, which never get @@ -294,97 +278,100 @@ func (txi *TXIndex) Update() error { // For each of the blocks we're removing, delete the transactions from // the transaction index. - for _, blockToDetach := range detachBlocks { - if txi.killed { - glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while detaching blocks")) - break - } - // Go through each txn in the block and delete its mappings from our - // txindex. - glog.V(1).Infof("Update: Detaching block (height: %d, hash: %v)", - blockToDetach.Height, blockToDetach.Hash) - blockMsg, err := GetBlock(blockToDetach.Hash, txi.TXIndexChain.DB(), nil) - if err != nil { - return fmt.Errorf("Update: Problem fetching detach block "+ - "with hash %v: %v", blockToDetach.Hash, err) - } - blockHeight := uint64(txi.CoreChain.blockTip().Height) - err = txi.TXIndexChain.DB().Update(func(dbTxn *badger.Txn) error { - for _, txn := range blockMsg.Txns { - if err := DbDeleteTxindexTransactionMappingsWithTxn(dbTxn, nil, - blockHeight, txn, txi.Params, txi.CoreChain.eventManager, true); err != nil { - - return fmt.Errorf("Update: Problem deleting "+ - "transaction mappings for transaction %v: %v", txn.Hash(), err) - } - } - return nil - }) - if err != nil { - return err - } - - // Now that all the transactions have been deleted from our txindex, - // it's safe to disconnect the block from our txindex chain. - utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - utxoOps, err := GetUtxoOperationsForBlock( - txi.TXIndexChain.DB(), nil, blockToDetach.Hash) - if err != nil { - return fmt.Errorf( - "Update: Error getting UtxoOps for block %v: %v", blockToDetach, err) - } - // Compute the hashes for all the transactions. - txHashes, err := ComputeTransactionHashes(blockMsg.Txns) - if err != nil { - return fmt.Errorf( - "Update: Error computing tx hashes for block %v: %v", - blockToDetach, err) - } - if err := utxoView.DisconnectBlock(blockMsg, txHashes, utxoOps, blockHeight); err != nil { - return fmt.Errorf("Update: Error detaching block "+ - "%v from UtxoView: %v", blockToDetach, err) - } - if err := utxoView.FlushToDb(blockHeight); err != nil { - return fmt.Errorf("Update: Error flushing view to db for block "+ - "%v: %v", blockToDetach, err) - } - // We have to flush a couple of extra things that the view doesn't flush... - if err := PutBestHash(txi.TXIndexChain.DB(), nil, utxoView.TipHash, ChainTypeDeSoBlock, txi.CoreChain.eventManager); err != nil { - return fmt.Errorf("Update: Error putting best hash for block "+ - "%v: %v", blockToDetach, err) - } - err = txi.TXIndexChain.DB().Update(func(txn *badger.Txn) error { - if err := DeleteUtxoOperationsForBlockWithTxn(txn, nil, blockToDetach.Hash, txi.TXIndexChain.eventManager, true); err != nil { - return fmt.Errorf("Update: Error deleting UtxoOperations 1 for block %v, %v", blockToDetach.Hash, err) - } - if err := txn.Delete(BlockHashToBlockKey(blockToDetach.Hash)); err != nil { - return fmt.Errorf("Update: Error deleting UtxoOperations 2 for block %v %v", blockToDetach.Hash, err) - } - return nil - }) - - if err != nil { - return fmt.Errorf("Update: Error updating badgger: %v", err) - } - // Delete this block from the chain db so we don't get duplicate block errors. - - // Remove this block from our bestChain data structures. - newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() - newBestChain, newBestChainMap := txi.TXIndexChain.CopyBestChain() - newBestChain = newBestChain[:len(newBestChain)-1] - newBestChainMap.Remove(*blockToDetach.Hash) - newBlockIndex.Remove(*blockToDetach.Hash) - - txi.TXIndexChain.SetBestChainMap(newBestChain, newBestChainMap, newBlockIndex) - - // At this point the entries for the block should have been removed - // from both our Txindex chain and our transaction index mappings. - } + // TODO: delete - we're simplifying the txindex logic to only use committed state. + //for _, blockToDetach := range detachBlocks { + // if txi.killed { + // glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while detaching blocks")) + // break + // } + // // Go through each txn in the block and delete its mappings from our + // // txindex. + // glog.V(1).Infof("Update: Detaching block (height: %d, hash: %v)", + // blockToDetach.Height, blockToDetach.Hash) + // blockMsg, err := GetBlock(blockToDetach.Hash, txi.TXIndexChain.DB(), nil) + // if err != nil { + // return fmt.Errorf("Update: Problem fetching detach block "+ + // "with hash %v: %v", blockToDetach.Hash, err) + // } + // blockHeight := uint64(txi.CoreChain.blockTip().Height) + // err = txi.TXIndexChain.DB().Update(func(dbTxn *badger.Txn) error { + // for _, txn := range blockMsg.Txns { + // if err := DbDeleteTxindexTransactionMappingsWithTxn(dbTxn, nil, + // blockHeight, txn, txi.Params, txi.CoreChain.eventManager, true); err != nil { + // + // return fmt.Errorf("Update: Problem deleting "+ + // "transaction mappings for transaction %v: %v", txn.Hash(), err) + // } + // } + // return nil + // }) + // if err != nil { + // return err + // } + // + // // Now that all the transactions have been deleted from our txindex, + // // it's safe to disconnect the block from our txindex chain. + // utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) + // utxoOps, err := GetUtxoOperationsForBlock( + // txi.TXIndexChain.DB(), nil, blockToDetach.Hash) + // if err != nil { + // return fmt.Errorf( + // "Update: Error getting UtxoOps for block %v: %v", blockToDetach, err) + // } + // // Compute the hashes for all the transactions. + // txHashes, err := ComputeTransactionHashes(blockMsg.Txns) + // if err != nil { + // return fmt.Errorf( + // "Update: Error computing tx hashes for block %v: %v", + // blockToDetach, err) + // } + // if err := utxoView.DisconnectBlock(blockMsg, txHashes, utxoOps, blockHeight); err != nil { + // return fmt.Errorf("Update: Error detaching block "+ + // "%v from UtxoView: %v", blockToDetach, err) + // } + // if err := utxoView.FlushToDb(blockHeight); err != nil { + // return fmt.Errorf("Update: Error flushing view to db for block "+ + // "%v: %v", blockToDetach, err) + // } + // // We have to flush a couple of extra things that the view doesn't flush... + // if err := PutBestHash(txi.TXIndexChain.DB(), nil, utxoView.TipHash, ChainTypeDeSoBlock, txi.CoreChain.eventManager); err != nil { + // return fmt.Errorf("Update: Error putting best hash for block "+ + // "%v: %v", blockToDetach, err) + // } + // err = txi.TXIndexChain.DB().Update(func(txn *badger.Txn) error { + // if err := DeleteUtxoOperationsForBlockWithTxn(txn, nil, blockToDetach.Hash, txi.TXIndexChain.eventManager, true); err != nil { + // return fmt.Errorf("Update: Error deleting UtxoOperations 1 for block %v, %v", blockToDetach.Hash, err) + // } + // if err := txn.Delete(BlockHashToBlockKey(blockToDetach.Hash)); err != nil { + // return fmt.Errorf("Update: Error deleting UtxoOperations 2 for block %v %v", blockToDetach.Hash, err) + // } + // return nil + // }) + // + // if err != nil { + // return fmt.Errorf("Update: Error updating badgger: %v", err) + // } + // // Delete this block from the chain db so we don't get duplicate block errors. + // + // // Remove this block from our bestChain data structures. + // newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() + // newTip := blockToDetach.GetParent(txi.TXIndexChain.blockIndex) + // if newTip == nil { + // return fmt.Errorf("Update: Error getting parent of block %v", blockToDetach) + // } + // + // txi.TXIndexChain.SetBestChainMap(newBlockIndex, newTip) + // + // // At this point the entries for the block should have been removed + // // from both our Txindex chain and our transaction index mappings. + //} // For each of the blocks we're adding, process them on our txindex chain // and add their mappings to our txn index. Compute any metadata that might // be useful. - for _, blockToAttach := range attachBlocks { + blockToAttach := &BlockNode{} + *blockToAttach = *txindexTipNode + for !blockToAttach.Hash.IsEqual(blockTipNode.Hash) { if txi.killed { glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while attaching blocks")) break @@ -454,6 +441,11 @@ func (txi *TXIndex) Update() error { return fmt.Errorf("Update: Problem attaching block %v: %v", blockToAttach, err) } + var exists bool + blockToAttach, exists, err = txi.CoreChain.GetBlockFromBestChainByHeight(uint64(blockToAttach.Height+1), false) + if !exists || err != nil { + return fmt.Errorf("Update: Problem getting block at height %d: %v", blockToAttach.Height+1, err) + } } glog.Infof("Update: Txindex update complete. New tip: (height: %d, hash: %v)", From c6ad24a409ff2829f997cd41ff9156ea21599c6d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:34:53 -0400 Subject: [PATCH 018/131] move block index migration to NewBlockchain instead of NewServer, add setting committed status for PoW nodes, remove timeout adjustment in test.Dockerfile --- lib/blockchain.go | 3 +++ lib/db_utils.go | 46 ++++++++++++++++++++++++++++++++++++++++++++-- lib/server.go | 11 +++-------- test.Dockerfile | 2 +- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 12ab58072..0620b545b 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1306,6 +1306,9 @@ func NewBlockchain( archivalMode bool, checkpointSyncingProviders []string, ) (*Blockchain, error) { + if err := RunBlockIndexMigrationOnce(db, params); err != nil { + return nil, errors.Wrapf(err, "NewBlockchain: Problem running block index migration") + } trustedBlockProducerPublicKeys := make(map[PkMapKey]bool) for _, keyStr := range trustedBlockProducerPublicKeyStrs { diff --git a/lib/db_utils.go b/lib/db_utils.go index de0de1bf8..f6fe875fe 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5659,8 +5659,8 @@ func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams }) } -func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager) error { - return handle.View(func(txn *badger.Txn) error { +func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager, params *DeSoParams) error { + return handle.Update(func(txn *badger.Txn) error { prefix := _heightHashToNodeIndexPrefix(false) opts := badger.DefaultIteratorOptions opts.Prefix = prefix @@ -5692,6 +5692,48 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } } + // TODO: get best chain up to PoS Cutover height and set all blocks in that chain to committed. + firstPoSBlockHeight := params.GetFirstPoSBlockHeight() + // Look up blocks at cutover height. + prefixKey := _heightHashToNodePrefixByHeight(uint32(firstPoSBlockHeight), false) + _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, prefixKey, false) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") + } + if len(valsFound) == 0 { + return fmt.Errorf("RunBlockIndexMigration: No blocks found at PoS cutover height") + } + if len(valsFound) > 1 { + return fmt.Errorf("RunBlockIndexMigration: More than one block found at PoS cutover height") + } + blockNode, err := DeserializeBlockNode(valsFound[0]) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + } + var blockNodeBatch []*BlockNode + for blockNode != nil { + if !blockNode.IsCommitted() { + blockNode.Status |= StatusBlockCommitted + } + // TODO: make sure I don't need a copy. + blockNodeBatch = append(blockNodeBatch, blockNode) + if len(blockNodeBatch) < 10000 { + continue + } + err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + } + parentBlockNode := GetHeightHashToNodeInfoWithTxn(txn, snapshot, blockNode.Height, blockNode.Hash, false /*bitcoinNodes*/) + if blockNode.Height > 0 && parentBlockNode == nil { + return errors.New("RunBlockIndexMigration: Parent block node not found") + } + blockNode = parentBlockNode + } + err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + } return nil }) } diff --git a/lib/server.go b/lib/server.go index 997f63785..57a7bc625 100644 --- a/lib/server.go +++ b/lib/server.go @@ -364,8 +364,8 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } } -func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { - blockIndexMigrationFileName := filepath.Join(dataDir, BlockIndexMigrationFileName) +func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { + blockIndexMigrationFileName := filepath.Join(db.Opts().Dir, BlockIndexMigrationFileName) glog.V(0).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { @@ -373,7 +373,7 @@ func RunBlockIndexMigrationOnce(db *badger.DB, dataDir string) error { return nil } glog.V(0).Info("Running block index migration") - if err = RunBlockIndexMigration(db, nil, nil); err != nil { + if err = RunBlockIndexMigration(db, nil, nil, params); err != nil { return errors.Wrapf(err, "Problem running block index migration") } if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { @@ -457,11 +457,6 @@ func NewServer( _err error, _shouldRestart bool, ) { - - if err := RunBlockIndexMigrationOnce(_db, _dataDir); err != nil { - return nil, errors.Wrapf(err, "NewServer: Problem running block index migration"), true - } - var err error // Only initialize state change syncer if the directories are defined. diff --git a/test.Dockerfile b/test.Dockerfile index 34182c4c7..3eb8c7501 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -28,4 +28,4 @@ COPY main.go . # build backend RUN GOOS=linux go build -mod=mod -a -installsuffix cgo -o bin/core main.go -ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "-timeout", "15m", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] +ENTRYPOINT ["go", "test", "-v", "-failfast", "-p", "1", "github.com/deso-protocol/core/bls", "github.com/deso-protocol/core/collections", "github.com/deso-protocol/core/collections/bitset", "github.com/deso-protocol/core/consensus", "github.com/deso-protocol/core/lib"] From 216fd31274cda0ee8aa6a3c40ab0ce89f387c8c2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:39:07 -0400 Subject: [PATCH 019/131] merge main into branch --- cmd/node.go | 2 +- go.mod | 69 +++++++++------- go.sum | 199 +++++++++++++++++++++++++++++---------------- lib/pos_mempool.go | 10 +++ lib/server.go | 2 +- 5 files changed, 181 insertions(+), 101 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index f494a79f7..1041da734 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -11,7 +11,7 @@ import ( "syscall" "time" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" diff --git a/go.mod b/go.mod index ddf84244b..02c87cd5f 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/deso-protocol/core go 1.20 require ( - github.com/DataDog/datadog-go v4.5.0+incompatible + github.com/DataDog/datadog-go/v5 v5.5.0 github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible github.com/cloudflare/circl v1.1.0 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 @@ -17,29 +17,29 @@ require ( github.com/dgraph-io/badger/v3 v3.2103.5 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.9.25 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.16.0 github.com/gernest/mention v2.0.0+incompatible - github.com/go-pg/pg/v10 v10.10.0 + github.com/go-pg/pg/v10 v10.11.1 github.com/golang/glog v1.0.0 - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.5.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/holiman/uint256 v1.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/oleiade/lane v1.0.1 github.com/onflow/crypto v0.25.0 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0 github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 github.com/spf13/cobra v1.1.3 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 golang.org/x/crypto v0.27.0 golang.org/x/sync v0.8.0 - gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) require ( @@ -49,7 +49,14 @@ require ( ) require ( - github.com/Microsoft/go-winio v0.4.16 // indirect + github.com/DataDog/appsec-internal-go v1.7.0 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect + github.com/DataDog/go-libddwaf/v3 v3.2.1 // indirect + github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect + github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/DataDog/sketches-go v1.4.5 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect github.com/bwesterb/go-ristretto v1.2.0 // indirect @@ -58,62 +65,66 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.6.0-alpha.5 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/git-chglog/git-chglog v0.0.0-20200414013904-db796966b373 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect - github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.8 // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.12.3 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/magiconair/properties v1.8.1 // indirect - github.com/mattn/go-colorable v0.1.9 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/goveralls v0.0.6 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/onsi/ginkgo v1.15.0 // indirect github.com/onsi/gomega v1.10.5 // indirect + github.com/outcaste-io/ristretto v0.2.3 // indirect github.com/pelletier/go-toml v1.7.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect - github.com/philhofer/fwd v1.1.1 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 // indirect github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.1.2 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tinylib/msgp v1.1.2 // indirect + github.com/tinylib/msgp v1.1.8 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect github.com/urfave/cli v1.22.1 // indirect github.com/vmihailenco/bufpool v0.1.11 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.1 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.opencensus.io v0.23.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect - golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.25.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gonum.org/v1/gonum v0.6.1 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/AlecAivazis/survey.v1 v1.8.7 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.51.0 // indirect gopkg.in/kyokomi/emoji.v1 v1.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - mellium.im/sasl v0.2.1 // indirect + mellium.im/sasl v0.3.1 // indirect ) diff --git a/go.sum b/go.sum index c159e2333..5536e1ec2 100644 --- a/go.sum +++ b/go.sum @@ -27,11 +27,25 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go v4.5.0+incompatible h1:MyyuIz5LVAI3Im+0F/tfo64ETyH4sNVynZ29yOiHm50= -github.com/DataDog/datadog-go v4.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= -github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/DataDog/appsec-internal-go v1.7.0 h1:iKRNLih83dJeVya3IoUfK+6HLD/hQsIbyBlfvLmAeb0= +github.com/DataDog/appsec-internal-go v1.7.0/go.mod h1:wW0cRfWBo4C044jHGwYiyh5moQV2x0AhnwqMuiX7O/g= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= +github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= +github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8= +github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/DataDog/sketches-go v1.4.5 h1:ki7VfeNz7IcNafq7yI/j5U/YCkO3LJiMDtXz9OMQbyE= +github.com/DataDog/sketches-go v1.4.5/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -86,9 +100,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -106,11 +117,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= @@ -134,9 +145,14 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= @@ -148,8 +164,8 @@ github.com/ethereum/go-ethereum v1.9.25 h1:mMiw/zOOtCLdGLWfcekua0qPrJTe7FVIiHJ4I github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8plZ0M9VMk4/oM= github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fergusstrange/embedded-postgres v1.19.0 h1:NqDufJHeA03U7biULlPHZ0pZ10/mDOMKPILEpT50Fyk= github.com/fergusstrange/embedded-postgres v1.19.0/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -169,8 +185,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-pg/pg/v10 v10.5.0/go.mod h1:BfgPoQnD2wXNd986RYEHzikqv9iE875PrFaZ9vXvtNM= -github.com/go-pg/pg/v10 v10.10.0 h1:xc5zWYQ/55XI8pk5NkK+ixXqbJh1vnOun3VODPmbYfY= -github.com/go-pg/pg/v10 v10.10.0/go.mod h1:EmoJGYErc+stNN/1Jf+o4csXuprjxcRztBnn6cHe38E= +github.com/go-pg/pg/v10 v10.11.1 h1:vYwbFpqoMpTDphnzIPshPPepdy3VpzD8qo29OFKp4vo= +github.com/go-pg/pg/v10 v10.11.1/go.mod h1:ExJWndhDNNftBdw1Ow83xqpSf4WMSJK8urmXD5VXS1I= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -191,6 +207,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -204,8 +221,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -226,17 +243,18 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -255,7 +273,10 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -265,8 +286,9 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -278,9 +300,9 @@ github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -307,19 +329,17 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -329,16 +349,16 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/goveralls v0.0.6 h1:cr8Y0VMo/MnEZBjxNN/vh6G90SZ7IMb6lms1dzMoO+Y= @@ -355,8 +375,9 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -386,8 +407,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -396,14 +419,15 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -416,6 +440,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= +github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0 h1:0/H63lDsoNYVn5YmP6VLDEnnKkoVYiHx7udTWCK4BUI= github.com/robinjoseph08/go-pg-migrations/v3 v3.0.0/go.mod h1:nOkSFfwwDUBFnDDQqMRC2p4PDE7GZb/KSVqILVB3bmw= @@ -427,15 +453,18 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= +github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 h1:Xuk8ma/ibJ1fOy4Ee11vHhUFHQNpHhrBneOCNHVXS5w= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0/go.mod h1:7AwjWCpdPhkSmNAgUv5C7EJ4AbmjEB3r047r3DXWu3Y= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= @@ -465,8 +494,9 @@ github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8 github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -475,13 +505,15 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ= -github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= @@ -502,8 +534,9 @@ github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5 github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= -github.com/vmihailenco/msgpack/v5 v5.3.1 h1:0i85a4dsZh8mC//wmyyTEzidDLPQfQAxZIOLtafGbFY= -github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -517,16 +550,20 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -536,6 +573,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -545,8 +583,8 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -557,8 +595,8 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 h1:A1gGSx58LAGVHUUsOf7IiR0u8Xb6W51gRwfDBhkdcaw= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -579,7 +617,9 @@ golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -610,6 +650,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -623,6 +666,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -643,12 +688,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -658,26 +700,38 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201026173827-119d4633e4d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -704,14 +758,18 @@ golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.1 h1:/LSrTrgZtpbXyAR6+0e152SROCkJJSh7goYWVmdPFGc= gonum.org/v1/gonum v0.6.1/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -757,19 +815,18 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM= gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= -gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 h1:3C1EEjgFTPqrnS2SXuSqkBbZGacIOPJ7ScGJk4nrP9s= -gopkg.in/DataDog/dd-trace-go.v1 v1.29.0/go.mod h1:FLwUDeuH0z5hkvgvd04/M3MHQN4AF5pQDnedeWRWvok= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 h1:Ne7kzWr/br/jwhUJR7CnqPl/mUpNxa6LfgZs0S4htZM= +gopkg.in/DataDog/dd-trace-go.v1 v1.65.1/go.mod h1:beNFIWd/H04d0k96cfltgiDH2+t0T5sDbyYLF3VTXqk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= @@ -794,13 +851,15 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 7e0ba023e..ae8a2c94a 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "fmt" "path/filepath" "sync" @@ -944,6 +945,15 @@ func (mp *PosMempool) UpdateLatestBlock(blockView *UtxoView, blockHeight uint64) // new minimum will be removed from the mempool. To safely handle this, this method re-creates the TransactionRegister // with the new global params and re-adds all transactions in the mempool to the new register. func (mp *PosMempool) UpdateGlobalParams(globalParams *GlobalParamsEntry) { + // If the global params haven't changed at all, then we don't need to do anything. + newGlobalParamBytes := globalParams.RawEncodeWithoutMetadata(mp.latestBlockHeight, true) + mp.RLock() + mpGlobalParamBytes := mp.globalParams.RawEncodeWithoutMetadata(mp.latestBlockHeight, true) + mp.RUnlock() + if bytes.Equal(newGlobalParamBytes, mpGlobalParamBytes) { + return + } + mp.Lock() defer mp.Unlock() diff --git a/lib/server.go b/lib/server.go index 71f003c8b..608cccc8b 100644 --- a/lib/server.go +++ b/lib/server.go @@ -18,7 +18,7 @@ import ( "github.com/decred/dcrd/lru" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/btcsuite/btcd/addrmgr" chainlib "github.com/btcsuite/btcd/blockchain" From 8b73b1cba90de39c13ec2eb14b1bc26c44f71c57 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 3 Oct 2024 00:47:49 -0400 Subject: [PATCH 020/131] handle case where we haven't hit PoS cutover yet in block index migration --- lib/db_utils.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/db_utils.go b/lib/db_utils.go index f6fe875fe..6b7ebfc62 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5669,6 +5669,9 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager nodeIterator := txn.NewIterator(opts) defer nodeIterator.Close() hashToHeightMap := make(map[BlockHash]uint32) + // Just in case we need it, get the height of the best hash. + bestHash := DbGetBestHash(handle, snapshot, ChainTypeDeSoBlock) + var bestHashHeight uint32 for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { item := nodeIterator.Item().Key() @@ -5677,6 +5680,9 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager hash := BlockHash{} copy(hash[:], item[5:]) hashToHeightMap[hash] = height + if bestHash != nil && bestHash.IsEqual(&hash) { + bestHashHeight = height + } if len(hashToHeightMap) < 10000 { continue } @@ -5692,6 +5698,10 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } } + // If we don't have a best hash, then we certainly haven't hit the first pos block height. + if bestHash == nil { + return nil + } // TODO: get best chain up to PoS Cutover height and set all blocks in that chain to committed. firstPoSBlockHeight := params.GetFirstPoSBlockHeight() // Look up blocks at cutover height. @@ -5700,15 +5710,21 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") } - if len(valsFound) == 0 { - return fmt.Errorf("RunBlockIndexMigration: No blocks found at PoS cutover height") - } if len(valsFound) > 1 { return fmt.Errorf("RunBlockIndexMigration: More than one block found at PoS cutover height") } - blockNode, err := DeserializeBlockNode(valsFound[0]) - if err != nil { - return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + var blockNode *BlockNode + // In this case, we need to find pull the best hash from the DB and iterate backwards. + if len(valsFound) == 0 { + blockNode = GetHeightHashToNodeInfoWithTxn(txn, snapshot, bestHashHeight, bestHash, false) + if blockNode == nil { + return fmt.Errorf("RunBlockIndexMigration: block with Best hash (%v) and height (%v) not found", bestHash, bestHashHeight) + } + } else { + blockNode, err = DeserializeBlockNode(valsFound[0]) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + } } var blockNodeBatch []*BlockNode for blockNode != nil { From 7ac67bdf52a779a086a7aec7f3332a1ba9bb6c57 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 4 Oct 2024 18:37:19 -0400 Subject: [PATCH 021/131] cleanup --- lib/blockchain.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 0620b545b..20fd99bbb 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1335,14 +1335,8 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. - //blockIndexByHash: collections.NewConcurrentMap[BlockHash, *BlockNode](), - //blockIndexByHeight: make(map[uint64]map[BlockHash]*BlockNode), - //bestChainMap: make(map[BlockHash]*BlockNode), - //bestChain: NewBestChain(db, snapshot, false, params), - //bestHeaderChainMap: make(map[BlockHash]*BlockNode), - //bestHeaderChain: NewBestChain(db, snapshot, true, params), - blockViewCache: lru.NewKVCache(100), // TODO: parameterize + blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. + blockViewCache: lru.NewKVCache(100), // TODO: parameterize snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, From 7675765dc6884d1a9aa1c7d7e0950a3d114f18b5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:21:07 -0400 Subject: [PATCH 022/131] do some sampling for is fully stored and check archival mode --- lib/blockchain.go | 87 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 20fd99bbb..591022eb2 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1085,13 +1085,37 @@ func (bc *Blockchain) IsFullyStored() bool { chainState := bc.ChainState() if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && bc.headerTip().Height-bc.blockTip().Height < 10) { - // TODO: rewrite to work w/ block index. - panic("IsFullyStored: Need to implement this.") - //for _, blockNode := range bc.bestChain.Chain { - // if !blockNode.Status.IsFullyProcessed() { - // return false - // } - //} + // Get a sampling of blocks from the best chain and check if they are fully stored. + // We only need to check a few blocks to determine if the chain is fully stored. + blockTipHeight := uint64(bc.BlockTip().Height) + increment := blockTipHeight / 20 + if increment == 0 { + increment = 1 + } + blockHeights := []uint64{} + for ii := uint64(0); ii < blockTipHeight; ii += increment { + blockHeights = append(blockHeights, ii) + } + if blockTipHeight > 100 { + for ii := blockTipHeight - 20; ii < blockTipHeight; ii++ { + blockHeights = append(blockHeights, ii) + } + } + blockHeights = append(blockHeights, blockTipHeight) + blockHeightSet := NewSet(blockHeights) + for _, blockHeight := range blockHeightSet.ToSlice() { + blockNode, exists, err := bc.GetBlockFromBestChainByHeight(blockHeight, false) + if err != nil { + glog.Errorf("IsFullyStored: Problem getting block at height %d: %v", blockHeight, err) + return false + } + if !exists { + return false + } + if !blockNode.Status.IsFullyProcessed() { + return false + } + } return true } return false @@ -1960,23 +1984,38 @@ func (bc *Blockchain) checkArchivalMode() bool { _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? - // TODO: figure out how to iterate over best chain to checkArchivalMode - // when we only have a portion of best chain in memory. - panic("NOT IMPLEMENTED: checkArchivalMode") - //for _, blockNode := range bc.bestChain.Chain { - // if uint64(blockNode.Height) > firstSnapshotHeight { - // return false - // } - // - // // Check if we have blocks that have been processed and validated but not stored. This would indicate that there - // // are historical blocks that we are yet to download. - // if (blockNode.Status&StatusBlockProcessed) == 1 && - // (blockNode.Status&StatusBlockValidated) == 1 && - // (blockNode.Status&StatusBlockStored) == 0 { - // - // return true - // } - //} + // We take a sampling of blocks to determine if we've downloaded all the blocks up to the first snapshot height. + blockHeights := []uint64{} + increment := firstSnapshotHeight / 10 + for ii := uint64(0); ii < firstSnapshotHeight; ii += increment { + blockHeights = append(blockHeights, ii) + } + for ii := firstSnapshotHeight - 10; ii < firstSnapshotHeight; ii++ { + blockHeights = append(blockHeights, ii) + } + blockHeights = append(blockHeights, firstSnapshotHeight) + for _, height := range blockHeights { + blockNode, exists, err := bc.GetBlockFromBestChainByHeight(height, false) + if err != nil { + glog.Errorf("checkArchivalMode: Problem getting block by height: %v", err) + return false + } + if !exists { + return false + } + if uint64(blockNode.Height) > firstSnapshotHeight { + return false + } + + // Check if we have blocks that have been processed and validated but not stored. This would indicate that there + // are historical blocks that we are yet to download. + if (blockNode.Status&StatusBlockProcessed) == 1 && + (blockNode.Status&StatusBlockValidated) == 1 && + (blockNode.Status&StatusBlockStored) == 0 { + + return true + } + } // If we get here, it means that all blocks have been processed and stored, so there is nothing to do. return false From 148bad3ae471eb14f6097f96b4470f5f0737fb0c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:48:41 -0400 Subject: [PATCH 023/131] fix get blocks to store --- lib/blockchain.go | 20 +------- lib/server.go | 120 ++++++++++++++++++++++++++-------------------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 591022eb2..3d0801d6a 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -870,24 +870,8 @@ type Blockchain struct { ChainLock deadlock.RWMutex // These should only be accessed after acquiring the ChainLock. - // - // An in-memory index of the "tree" of blocks we are currently aware of. - // This index includes forks and side-chains. - //blockIndexByHash *collections.ConcurrentMap[BlockHash, *BlockNode] - //// blockIndexByHeight is an in-memory map of block height to block nodes. This is - //// used to quickly find the safe blocks from which the chain can be extended for PoS - //blockIndexByHeight map[uint64]map[BlockHash]*BlockNode - // An in-memory slice of the blocks on the main chain only. The end of - // this slice is the best known tip that we have at any given time. - //bestChain []*BlockNode - //bestChainMap map[BlockHash]*BlockNode - // - //bestHeaderChain []*BlockNode - //bestHeaderChainMap map[BlockHash]*BlockNode - - blockIndex *BlockIndex - //bestChain *BestChain - //bestHeaderChain *BestChain + blockIndex *BlockIndex + lowestBlockNotStored uint64 // We keep track of orphan blocks with the following data structures. Orphans // are not written to disk and are only cached in memory. Moreover we only keep diff --git a/lib/server.go b/lib/server.go index 9f6fcf36d..6fe1f0766 100644 --- a/lib/server.go +++ b/lib/server.go @@ -879,59 +879,73 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Go through the block nodes in the blockchain and download the blocks if they're not stored. - // TODO: need to figure out a way to get all the blocks in the best chain so we can download historical blocks. - //for _, blockNode := range srv.blockchain.bestChain.Chain { - // // We find the first block that's not stored and get ready to download blocks starting from this block onwards. - // if blockNode.Status&StatusBlockStored == 0 { - // maxBlocksInFlight := MaxBlocksInFlight - // if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && - // (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || - // srv.params.NetworkType == NetworkType_TESTNET) { - // - // maxBlocksInFlight = MaxBlocksInFlightPoS - // } - // numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - // currentHeight := int(blockNode.Height) - // blockNodesToFetch := []*BlockNode{} - // // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. - // var heightLimit int - // for heightLimit = len(srv.blockchain.bestChain.Chain) - 1; heightLimit >= 0; heightLimit-- { - // if !srv.blockchain.bestChain.Chain[heightLimit].Status.IsFullyProcessed() { - // break - // } - // } - // - // // Find the blocks that we should download. - // for currentHeight <= heightLimit && - // len(blockNodesToFetch) < numBlocksToFetch { - // - // // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to - // // the array index. - // currentNode := srv.blockchain.bestChain.Chain[currentHeight] - // currentHeight++ - // - // // If we've already requested this block then we don't request it again. - // if _, exists := pp.requestedBlocks[*currentNode.Hash]; exists { - // continue - // } - // - // blockNodesToFetch = append(blockNodesToFetch, currentNode) - // } - // - // var hashList []*BlockHash - // for _, node := range blockNodesToFetch { - // hashList = append(hashList, node.Hash) - // pp.requestedBlocks[*node.Hash] = true - // } - // pp.AddDeSoMessage(&MsgDeSoGetBlocks{ - // HashList: hashList, - // }, false) - // - // glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", - // blockNode.Header, pp) - // return - // } - //} + for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), false) + if err != nil { + glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) + return + } + if !exists { + glog.Errorf("GetBlocksToStore: Block at height %v not found in best chain", ii) + return + } + // We find the first block that's not stored and get ready to download blocks starting from this block onwards. + if blockNode.Status&StatusBlockStored == 0 { + maxBlocksInFlight := MaxBlocksInFlight + if pp.NegotiatedProtocolVersion >= ProtocolVersion2 && + (srv.params.IsPoSBlockHeight(uint64(blockNode.Height)) || + srv.params.NetworkType == NetworkType_TESTNET) { + + maxBlocksInFlight = MaxBlocksInFlightPoS + } + srv.blockchain.lowestBlockNotStored = uint64(blockNode.Height) + numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) + currentHeight := int(blockNode.Height) + blockNodesToFetch := []*BlockNode{} + // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. + // We filter those out in the loop below by checking IsFullyProcessed. + // Find the blocks that we should download. + for len(blockNodesToFetch) < numBlocksToFetch { + + // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to + // the array index. + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(currentHeight), false) + if err != nil { + glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) + return + } + if !currNodeExists { + glog.Errorf("GetBlocksToStore: Block at height %v not found in best chain", currentHeight) + return + } + currentHeight++ + // If this node is already fully processed, then we don't need to download it. + if currentNode.Status.IsFullyProcessed() { + break + } + + // If we've already requested this block then we don't request it again. + if _, exists = pp.requestedBlocks[*currentNode.Hash]; exists { + continue + } + + blockNodesToFetch = append(blockNodesToFetch, currentNode) + } + + var hashList []*BlockHash + for _, node := range blockNodesToFetch { + hashList = append(hashList, node.Hash) + pp.requestedBlocks[*node.Hash] = true + } + pp.AddDeSoMessage(&MsgDeSoGetBlocks{ + HashList: hashList, + }, false) + + glog.V(1).Infof("GetBlocksToStore: Downloading blocks to store for header %v from peer %v", + blockNode.Header, pp) + return + } + } // If we get here then it means that we've downloaded all blocks so we can update srv.blockchain.downloadingHistoricalBlocks = false From 8aa6a5b404bbe2d9235b6b97f8ef94ba709a390f Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 17:57:02 -0400 Subject: [PATCH 024/131] cleanup --- lib/blockchain.go | 236 +++------------------------------------------- 1 file changed, 11 insertions(+), 225 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 3d0801d6a..79818c28c 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -702,148 +702,6 @@ func (bi *BlockIndex) GetHeaderTip() *BlockNode { return bi.headerTip } -//type BestChain struct { -// db *badger.DB -// snapshot *Snapshot -// IsHeaderChain bool -// Chain []*BlockNode // Ugh we can't really have a cache here. I mean maybe, but it complicates things quite a lot. -// ChainMap *lru2.Cache[BlockHash, *BlockNode] -// params *DeSoParams -//} -// -//func NewBestChain(db *badger.DB, snapshot *Snapshot, isHeaderChain bool, params *DeSoParams) *BestChain { -// chainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// return &BestChain{ -// db: db, -// snapshot: snapshot, -// IsHeaderChain: isHeaderChain, -// Chain: []*BlockNode{}, -// ChainMap: chainMap, -// params: params, -// } -//} -// -//func (bestChain *BestChain) PushNewTip(tipNode *BlockNode) { -// bestChain.Chain = append(bestChain.Chain, tipNode) -// bestChain.ChainMap.Add(*tipNode.Hash, tipNode) -//} -// -//func (bestChain *BestChain) GetTip() *BlockNode { -// if len(bestChain.Chain) == 0 { -// return nil -// } -// return bestChain.Chain[len(bestChain.Chain)-1] -//} -// -//func (bestChain *BestChain) GetBlockByHeight(height uint64) (*BlockNode, bool, error) { -// if height > math.MaxUint32 { -// return nil, false, fmt.Errorf("GetBlockByHeight: Height %d is greater than math.MaxUint32", height) -// } -// -// currTip := bestChain.GetTip() -// if currTip != nil && height > uint64(currTip.Height) { -// return nil, false, nil -// } -// -// if currTip != nil { -// currTipHeight := currTip.Height -// delta := currTipHeight - uint32(height) -// if delta < uint32(len(bestChain.Chain)) { -// targetNode := bestChain.Chain[uint32(len(bestChain.Chain)-1)-delta] -// if uint64(targetNode.Height) == height { -// return targetNode, true, nil -// } -// } -// //targetIndex := (len(bestChain.Chain)-1) -// //// We can probably do some binary search thing here instead. -// //currNode := &BlockNode{} -// //*currNode = *currTip -// //if currNode.Height == uint32(height) { -// // return currNode, true, nil -// //} -// //// During syncing, we don't write to the DB, so we need to iterate backwards through the best chain. -// //for currNode != nil && !currNode.IsStored() { -// // if currNode.Height < uint32(height) { -// // break -// // } -// // if currNode.Height == uint32(height) { -// // return currNode, true, nil -// // } -// // var currNodeExists bool -// // currNode, currNodeExists = bestChain.GetBlockByHashAndHeight( -// // currNode.Header.PrevBlockHash, uint64(currNode.Height-1)) -// // if !currNodeExists { -// // break -// // } -// //} -// } -// -// prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) -// _, valsFound := EnumerateKeysForPrefix(bestChain.db, prefixKey, false) -// if len(valsFound) == 0 { -// return nil, false, nil -// } -// for _, val := range valsFound { -// blockNode, err := DeserializeBlockNode(val) -// if err != nil { -// glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) -// continue -// } -// if blockNode.IsCommitted() { -// return blockNode, true, nil -// } -// } -// // TODO: how to return uncommitted blocks by height. We probably just need to iterate backwards through the -// // best chain. -// return nil, false, nil -//} -// -//func (bestChain *BestChain) GetBlockByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { -// val, exists := bestChain.ChainMap.Get(*blockHash) -// if exists { -// return val, true -// } -// if height > math.MaxUint32 { -// glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) -// } -// if height > uint64(bestChain.GetTip().Height) { -// return nil, false -// } -// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) -// if bn == nil { -// return nil, false -// } -// bestChain.ChainMap.Add(*blockHash, bn) -// return bn, true -//} -// -//func (bestChain *BestChain) GetBlockByHash(blockHash *BlockHash) (*BlockNode, bool, error) { -// val, exists := bestChain.ChainMap.Get(*blockHash) -// if exists { -// return val, true, nil -// } -// -// height, err := GetHeightForHash(bestChain.db, bestChain.snapshot, blockHash) -// if err != nil { -// if errors.Is(err, badger.ErrKeyNotFound) { -// return nil, false, nil -// } -// return nil, false, errors.Wrapf(err, "GetBlockByHash: Problem getting height for hash") -// } -// if height > uint64(bestChain.GetTip().Height) { -// return nil, false, nil -// } -// bn := GetHeightHashToNodeInfo(bestChain.db, bestChain.snapshot, uint32(height), blockHash, false) -// if bn == nil { -// return nil, false, nil -// } -// if !bn.IsCommitted() { -// return nil, false, nil -// } -// bestChain.ChainMap.Add(*blockHash, bn) -// return bn, true, nil -//} - type Blockchain struct { db *badger.DB postgres *Postgres @@ -1030,39 +888,10 @@ func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*Blo } func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { - //blocksAtHeight, hasNestedMapAtHeight := bc.blockIndexByHeight[blockHeight] - //if !hasNestedMapAtHeight { - // return false - //} - //return len(blocksAtHeight) > 0 blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) return len(blockNodes) > 0 } -//func (bc *Blockchain) CopyBestChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { -// newBestChain := []*BlockNode{} -// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// newBestChain = append(newBestChain, bc.bestChain.Chain...) -// for _, key := range bc.bestChain.ChainMap.Keys() { -// val, _ := bc.bestChain.ChainMap.Get(key) -// newBestChainMap.Add(key, val) -// } -// -// return newBestChain, newBestChainMap -//} - -//func (bc *Blockchain) CopyBestHeaderChain() ([]*BlockNode, *lru2.Cache[BlockHash, *BlockNode]) { -// newBestChain := []*BlockNode{} -// newBestChainMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) -// newBestChain = append(newBestChain, bc.bestHeaderChain.Chain...) -// for _, key := range bc.bestHeaderChain.ChainMap.Keys() { -// val, _ := bc.bestHeaderChain.ChainMap.Get(key) -// newBestChainMap.Add(key, val) -// } -// -// return newBestChain, newBestChainMap -//} - // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. @@ -1192,9 +1021,9 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } - // Walk back the last 100 hours of blocks. + // Walk back the last 24 hours of blocks. currBlockCounter := 1 - for currBlockCounter < 3600*100 && tipNode.Header.PrevBlockHash != nil { + for currBlockCounter < 3600*24 && tipNode.Header.PrevBlockHash != nil { bc.blockIndex.GetBlockNodeByHashAndHeight(tipNode.Header.PrevBlockHash, tipNode.Header.Height-1) currBlockCounter++ } @@ -1204,38 +1033,9 @@ func (bc *Blockchain) _initChain() error { } // We start by simply setting the chain tip and header tip to the tip node. - bc.blockIndex.tip = tipNode - bc.blockIndex.headerTip = tipNode - - } - - // At this point the blockIndexByHash should contain a full node tree with all - // nodes pointing to valid parent nodes. - { - // Walk back from the best node to the genesis block and store them all - // in bestChain. - //bc.bestChain.Chain, err = GetBestChain(tipNode) - //if err != nil { - // return errors.Wrapf(err, "_initChain(block): Problem reading best chain from db") - //} - //for _, bestChainNode := range bc.bestChain.Chain { - // bc.bestChain.ChainMap.Add(*bestChainNode.Hash, bestChainNode) - //} - } - - // TODO: This code is a bit repetitive but this seemed clearer than factoring it out. - { - // Walk back from the best node to the genesis block and store them all - // in bestChain. - //bc.bestHeaderChain.Chain, err = GetBestChain(tipNode) - //if err != nil { - // return errors.Wrapf(err, "_initChain(header): Problem reading best chain from db") - //} - //for _, bestHeaderChainNode := range bc.bestHeaderChain.Chain { - // bc.bestHeaderChain.ChainMap.Add(*bestHeaderChainNode.Hash, bestHeaderChainNode) - //} + bc.blockIndex.SetTip(tipNode) + bc.blockIndex.SetHeaderTip(tipNode) } - bc.isInitialized = true return nil @@ -1277,22 +1077,12 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { } // Add the uncommitted blocks to the in-memory data structures. - if _, _, _, err := bc.tryApplyNewTip(uncommittedTipBlockNode, 0, lineageFromCommittedTip); err != nil { + if _, _, _, err = bc.tryApplyNewTip(uncommittedTipBlockNode, 0, lineageFromCommittedTip); err != nil { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } bc.blockIndex.SetTip(uncommittedTipBlockNode) - - ////////////////////////// Update the bestHeaderChain in-memory data structures ////////////////////////// - //currentHeaderTip := bc.headerTip() - //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentHeaderTip, uncommittedTipBlockNode) - //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - // bc.bestHeaderChain.Chain, - // bc.bestHeaderChain.ChainMap, - // blocksToDetach, - // blocksToAttach, - //) - + bc.blockIndex.SetHeaderTip(uncommittedTipBlockNode) return nil } @@ -2696,12 +2486,6 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true bc.blockIndex.SetHeaderTip(newNode) - //_, detachBlocks, attachBlocks := bc.GetReorgBlocks(headerTip, newNode) - //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - // bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap, detachBlocks, attachBlocks) - - // Note that we don't store the best header hash here and so this is an - // in-memory-only adjustment. See the comment above on preventing attacks. } return isMainChain, false, nil @@ -3121,7 +2905,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // update our data structures to actually make this connection. Do this // in a transaction so that it is atomic. if bc.postgres != nil { - // TODO: pg support for setting committed on block node. + if !nodeToValidate.IsCommitted() { + nodeToValidate.Status |= StatusBlockCommitted + bc.blockIndex.addNewBlockNodeToBlockIndex(nodeToValidate) + } + if err = bc.postgres.UpsertBlockAndTransactions(nodeToValidate, desoBlock); err != nil { return false, false, errors.Wrapf(err, "ProcessBlock: Problem upserting block and transactions") } @@ -3210,8 +2998,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures if bc.isSyncing() { bc.blockIndex.SetTip(nodeToValidate) } else { - //newBestChain, newBestChainMap := bc.CopyBestChain() - //bc.bestChain.Chain, bc.bestChain.ChainMap = newBestChain, newBestChainMap bc.blockIndex.SetTip(nodeToValidate) } From 53b2d298d9589bc4ff11b27f3ac027ce5d27d913 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 19:55:39 -0400 Subject: [PATCH 025/131] make a bunch of methods on block index private and expose block index to backend --- lib/block_view_lockups_test.go | 2 +- lib/block_view_test.go | 4 ++-- lib/blockchain.go | 34 +++++++++++++++++++--------------- lib/blockchain_test.go | 16 ++++++++-------- lib/db_utils.go | 2 ++ lib/pos_blockchain.go | 6 +++--- lib/pos_blockchain_test.go | 16 ++++++++-------- lib/pos_consensus_test.go | 2 +- lib/server.go | 2 +- 9 files changed, 45 insertions(+), 39 deletions(-) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index c4f127892..41e53e74e 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2518,7 +2518,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.blockIndex.SetTip(testMeta.chain.blockIndex.tip.Parent) + testMeta.chain.blockIndex.setTip(testMeta.chain.blockIndex.tip.Parent) // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/block_view_test.go b/lib/block_view_test.go index f02d06e36..ff17677a4 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -791,8 +791,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // TODO: if ever needed we can call tm.chain.eventManager.blockDisconnected() here. // Update the block and header metadata chains. - tm.chain.blockIndex.SetTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) - tm.chain.blockIndex.SetHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.setTip(tm.chain.BlockTip().GetParent(tm.chain.blockIndex)) + tm.chain.blockIndex.setHeaderTip(tm.chain.HeaderTip().GetParent(tm.chain.blockIndex)) // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. diff --git a/lib/blockchain.go b/lib/blockchain.go index 79818c28c..eae552147 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -585,7 +585,7 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } } -func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { +func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) bi.blockIndexByHash = newHashToBlockNodeMap @@ -602,13 +602,13 @@ func (bi *BlockIndex) SetBlockIndexFromMap(input map[BlockHash]*BlockNode) { } } -func (bi *BlockIndex) SetHeaderTip(tip *BlockNode) { +func (bi *BlockIndex) setHeaderTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.headerTip = tip } -func (bi *BlockIndex) SetTip(tip *BlockNode) { +func (bi *BlockIndex) setTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.tip = tip @@ -882,6 +882,10 @@ func (bc *Blockchain) CopyBlockIndexes() ( return newBlockIndexByHash } +func (bc *Blockchain) GetBlockIndex() *BlockIndex { + return bc.blockIndex +} + // TODO: read through to DB. func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { return bc.blockIndex.GetBlockNodesByHeight(blockHeight) @@ -1033,8 +1037,8 @@ func (bc *Blockchain) _initChain() error { } // We start by simply setting the chain tip and header tip to the tip node. - bc.blockIndex.SetTip(tipNode) - bc.blockIndex.SetHeaderTip(tipNode) + bc.blockIndex.setTip(tipNode) + bc.blockIndex.setHeaderTip(tipNode) } bc.isInitialized = true @@ -1081,8 +1085,8 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } - bc.blockIndex.SetTip(uncommittedTipBlockNode) - bc.blockIndex.SetHeaderTip(uncommittedTipBlockNode) + bc.blockIndex.setTip(uncommittedTipBlockNode) + bc.blockIndex.setHeaderTip(uncommittedTipBlockNode) return nil } @@ -1953,9 +1957,9 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } else if bc.blockIndex.GetTip().Height < blockNode.Height { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } } } @@ -1965,7 +1969,7 @@ func (bc *Blockchain) SetBestChainMap( tipNode *BlockNode, ) { bc.blockIndex.blockIndexByHash = blockIndexByHash - bc.blockIndex.SetTip(tipNode) + bc.blockIndex.setTip(tipNode) } func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { @@ -2485,7 +2489,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B headerTip := bc.headerTip() if headerTip.CumWork.Cmp(newNode.CumWork) < 0 { isMainChain = true - bc.blockIndex.SetHeaderTip(newNode) + bc.blockIndex.setHeaderTip(newNode) } return isMainChain, false, nil @@ -2996,9 +3000,9 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. if bc.isSyncing() { - bc.blockIndex.SetTip(nodeToValidate) + bc.blockIndex.setTip(nodeToValidate) } else { - bc.blockIndex.SetTip(nodeToValidate) + bc.blockIndex.setTip(nodeToValidate) } // This node is on the main chain so set this variable. @@ -3286,8 +3290,8 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Now the db has been updated, update our in-memory best chain. Note that there // is no need to update the node index because it was updated as we went along. - bc.blockIndex.SetTip(newTipNode) - bc.blockIndex.SetHeaderTip(newTipNode) + bc.blockIndex.setTip(newTipNode) + bc.blockIndex.setHeaderTip(newTipNode) // If we made it here then this block is on the main chain. isMainChain = true diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 2bf1cab3d..1d6cfe406 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1254,7 +1254,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1292,7 +1292,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1356,7 +1356,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1394,7 +1394,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1458,7 +1458,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1516,7 +1516,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1575,7 +1575,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1612,7 +1612,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { }, StatusHeaderValidated, )) - bc.blockIndex.SetHeaderTip(nodes[len(nodes)-1]) + bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } diff --git a/lib/db_utils.go b/lib/db_utils.go index 6b7ebfc62..1a231b3b4 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5754,6 +5754,8 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager }) } +// TODO: refactor to actually get the whole best chain if that's +// what someone wants. It'll take a while, but whatever. func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 7b5dd2c86..4ae8013ac 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -97,7 +97,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, nil } - bc.blockIndex.SetHeaderTip(blockNode) + bc.blockIndex.setHeaderTip(blockNode) // The header is not an orphan and has a higher view than the current tip. We reorg the header chain // and apply the incoming header as the new tip. @@ -1650,7 +1650,7 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool // addTipBlockToBestChain adds the block as the new tip of the best chain. func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { - bc.blockIndex.SetTip(blockNode) + bc.blockIndex.setTip(blockNode) } // removeTipBlockFromBestChain removes the current tip from the best chain. It @@ -1661,7 +1661,7 @@ func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. lastBlock := bc.blockIndex.GetTip() // Uhhh what happens if we don't have the parent set up!? - bc.blockIndex.SetTip(lastBlock.GetParent(bc.blockIndex)) + bc.blockIndex.setTip(lastBlock.GetParent(bc.blockIndex)) return lastBlock } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 888ee1f9b..291a34eab 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -252,7 +252,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) @@ -331,7 +331,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -473,7 +473,7 @@ func TestHasValidBlockViewPoS(t *testing.T) { ValidatorsVoteQC: nil, ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, *hash2: block2, }) @@ -805,7 +805,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: genesisNode, }) block := &MsgDeSoBlock{ @@ -841,7 +841,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { ProposedInView: 2, PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) - bc.blockIndex.SetTip(block2) + bc.blockIndex.setTip(block2) bc.blockIndex.blockIndexByHash.Add(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) @@ -1244,7 +1244,7 @@ func TestShouldReorg(t *testing.T) { Height: 1, }, } - bc.blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *hash1: chain[0], *hash3: chain[1], }) @@ -1378,7 +1378,7 @@ func TestTryApplyNewTip(t *testing.T) { require.True(t, checkBestChainForHash(hash1)) // Remove newBlock from the best chain and block index to reset the state. - bc.blockIndex.SetTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) + bc.blockIndex.setTip(bc.blockIndex.GetTip().GetParent(bc.blockIndex)) // Add block 3 back bc.addTipBlockToBestChain(bn3) @@ -1457,7 +1457,7 @@ func TestTryApplyNewTip(t *testing.T) { //bc.bestChain.ChainMap.Remove(*hash5) //bc.bestChain.ChainMap.Remove(*newBlockHash) //bc.bestChain.Chain = bc.bestChain.Chain[:len(bc.bestChain.Chain)-3] - bc.blockIndex.SetTip(newBlockNode.GetParent(bc.blockIndex)) + bc.blockIndex.setTip(newBlockNode.GetParent(bc.blockIndex)) // Add block 2 and 3 back. bc.addTipBlockToBestChain(bn2) diff --git a/lib/pos_consensus_test.go b/lib/pos_consensus_test.go index 9d471b2a5..3e85457e9 100644 --- a/lib/pos_consensus_test.go +++ b/lib/pos_consensus_test.go @@ -103,7 +103,7 @@ func TestFastHotStuffConsensusHandleLocalTimeoutEvent(t *testing.T) { nextView := currentView + 1 blockIndex := NewBlockIndex(nil, nil, nil) - blockIndex.SetBlockIndexFromMap(map[BlockHash]*BlockNode{ + blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ *blockHash: {Header: blockHeader, Height: uint32(blockHeader.Height), Hash: blockHash}, }) diff --git a/lib/server.go b/lib/server.go index 6fe1f0766..28b225a48 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1756,7 +1756,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.blockIndex.SetTip(currentNode) + srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if len(blockNodeBatch) < 10000 { continue From 91e22ffc5ebbb15cf39a7a031bd7d1699cfce33c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 20:23:37 -0400 Subject: [PATCH 026/131] don't bother healing orphan pointers if we're syncing --- lib/pos_blockchain.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 4ae8013ac..e9388353f 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -81,9 +81,12 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b return false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") } - // Now that we know we have a valid header, we check the block index for it any orphan children for it - // and heal the parent pointers for all of them. - bc.healPointersForOrphanChildren(blockNode) + // Don't worry about healing orphan children when we're syncing. + if !bc.isSyncing() { + // Now that we know we have a valid header, we check the block index for it any orphan children for it + // and heal the parent pointers for all of them. + bc.healPointersForOrphanChildren(blockNode) + } // Exit early if the header is an orphan. if isOrphan { From 9abe23865512255c938fbd74d355e3d65aa6f279 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 21:12:23 -0400 Subject: [PATCH 027/131] refactor to reduce number of times we have to call hash and don't bother healing orphan pointers during syncing --- lib/blockchain.go | 2 +- lib/pos_blockchain.go | 151 +++++++++++++++++++------------------ lib/pos_blockchain_test.go | 34 ++++----- 3 files changed, 96 insertions(+), 91 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index eae552147..3ecdd2b5e 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2508,7 +2508,7 @@ func (bc *Blockchain) ProcessHeader(blockHeader *MsgDeSoHeader, headerHash *Bloc // If the header's height is after the PoS cut-over fork height, then we use the PoS header processing logic. // Otherwise, fall back to the PoW logic. if bc.params.IsPoSBlockHeight(blockHeader.Height) { - return bc.processHeaderPoS(blockHeader, verifySignatures) + return bc.processHeaderPoS(blockHeader, headerHash, verifySignatures) } return bc.processHeaderPoW(blockHeader, headerHash) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index e9388353f..2b0383f95 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -38,7 +38,7 @@ import ( // StatusHeaderValidated or StatusHeaderValidateFailed. // 5. Exit early if the's view is less than the current header chain's tip. // 6. Reorg the best header chain if the header's view is higher than the current tip. -func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures bool) ( +func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( _isMainChain bool, _isOrphan bool, _err error, ) { if !bc.params.IsPoSBlockHeight(header.Height) { @@ -48,19 +48,13 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, verifySignatures b ) } - headerHash, err := header.Hash() - if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem hashing header") - } - // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - var isInBestHeaderChain bool - _, isInBestHeaderChain, err = bc.GetBlockFromBestChainByHash(headerHash, true) + _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } @@ -206,14 +200,14 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas // is also not valid. if parentBlockNode.IsHeaderValidateFailed() { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Parent header failed validations"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Parent header failed validations"), ) } // Verify that the header is properly formed. if err := bc.isValidBlockHeaderPoS(header); err != nil { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Header failed validations"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Header failed validations"), ) } @@ -225,13 +219,13 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } if !isValidRandomSeedSignature { return nil, false, bc.storeValidateFailedHeaderInBlockIndexWithWrapperError( - header, errors.New("validateAndIndexHeaderPoS: Header has invalid random seed signature"), + header, headerHash, errors.New("validateAndIndexHeaderPoS: Header has invalid random seed signature"), ) } } // Store it as HeaderValidated now that it has passed all validations. - blockNode, err = bc.storeValidatedHeaderInBlockIndex(header) + blockNode, err = bc.storeValidatedHeaderInBlockIndex(header, headerHash) if err != nil { return nil, false, errors.Wrapf(err, "validateAndIndexHeaderPoS: Problem adding header to block index: ") } @@ -283,14 +277,15 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } // If we can't hash the block, we can never store in the block index and we should throw it out immediately. - if _, err := block.Hash(); err != nil { + blockHash, err := block.Hash() + if err != nil { return false, false, nil, errors.Wrapf(err, "processBlockPoS: Problem hashing block") } // In hypersync archival mode, we may receive blocks that have already been processed and committed during state // synchronization. However, we may want to store these blocks in the db for archival purposes. We check if the // block we're dealing with is an archival block. If it is, we store it and return early. - if success, err := bc.checkAndStoreArchivalBlock(block); err != nil { + if success, err := bc.checkAndStoreArchivalBlock(block, blockHash); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem checking and storing archival block") } else if success { return true, false, nil, nil @@ -335,7 +330,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v parentUtxoView := parentUtxoViewAndUtxoOps.UtxoView // First, we perform a validation of the leader and the QC to prevent spam. // If the block fails this check, we throw it away. - passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, parentUtxoView, verifySignatures) + passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, blockHash, parentUtxoView, verifySignatures) if err != nil { // If we hit an error, we can't store it since we're not sure if it passed the spam prevention check. return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem validating leader and QC") @@ -346,7 +341,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v } // Validate the block and store it in the block index. The block is guaranteed to not be an orphan. - blockNode, err := bc.validateAndIndexBlockPoS(block, parentUtxoView, verifySignatures) + blockNode, err := bc.validateAndIndexBlockPoS(block, blockHash, parentUtxoView, verifySignatures) if err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem validating block: ") @@ -369,7 +364,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // header and applying it to the header chain will result in the two chains being out of // sync. The header chain is less critical and mutations to it are reversible. So we attempt // to mutate it first before attempting to mutate the block chain. - if _, _, err = bc.processHeaderPoS(block.Header, verifySignatures); err != nil { + if _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem processing header") } @@ -559,31 +554,29 @@ func (bc *Blockchain) processOrphanBlockPoS(block *MsgDeSoBlock) error { // As a spam-prevention measure, we just throw away this block and don't store it. return nil } + + blockHash, err := block.Header.Hash() if err != nil { - return errors.Wrap(err, "processOrphanBlockPoS: Problem getting snapshot global params") + return errors.Wrap(err, "processOrphanBlockPoS: Problem hashing block") } + // All blocks should pass the basic integrity validations, which ensure the block // is not malformed. If the block is malformed, we should store it as ValidateFailed. if err = bc.isProperlyFormedBlockPoS(block); err != nil { - if _, innerErr := bc.storeValidateFailedBlockInBlockIndex(block); innerErr != nil { + if _, innerErr := bc.storeValidateFailedBlockInBlockIndex(block, blockHash); innerErr != nil { return errors.Wrapf(innerErr, "processOrphanBlockPoS: Problem adding validate failed block to block index: %v", err) } return nil } // Add to blockIndexByHash with status STORED only as we are not sure if it's valid yet. - _, err = bc.storeBlockInBlockIndex(block) + _, err = bc.storeBlockInBlockIndex(block, blockHash) return errors.Wrap(err, "processBlockPoS: Problem adding block to block index: ") } // checkAndStoreArchivalBlock is a helper function that takes in a block and checks if it's an archival block. // If it is, it stores the block in the db and returns true. If it's not, it returns false, or false and an error. -func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success bool, _err error) { - // First, get the block hash and lookup the block index. - blockHash, err := block.Hash() - if err != nil { - return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem hashing block") - } +func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock, blockHash *BlockHash) (_success bool, _err error) { blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) // If the blockNode doesn't exist, or the block is not committed, or it's already stored, then we're not dealing // with an archival block. Archival blocks must have an existing blockNode, be committed, and not be stored. @@ -593,8 +586,7 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success // If we get to this point, we're dealing with an archival block, so we'll attempt to store it. // This means, this block node is already marked as COMMITTED and VALIDATED, and we just need to store it. - _, err = bc.storeBlockInBlockIndex(block) - if err != nil { + if _, err := bc.storeBlockInBlockIndex(block, blockHash); err != nil { return false, errors.Wrap(err, "checkAndStoreArchivalBlock: Problem storing block in block index") } return true, nil @@ -602,9 +594,9 @@ func (bc *Blockchain) checkAndStoreArchivalBlock(block *MsgDeSoBlock) (_success // storeValidateFailedBlockWithWrappedError is a helper function that takes in a block and an error and // stores the block in the block index with status VALIDATE_FAILED. It returns the resulting BlockNode. -func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlock, outerErr error) ( +func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlock, hash *BlockHash, outerErr error) ( *BlockNode, error) { - blockNode, innerErr := bc.storeValidateFailedBlockInBlockIndex(block) + blockNode, innerErr := bc.storeValidateFailedBlockInBlockIndex(block, hash) if innerErr != nil { return nil, errors.Wrapf(innerErr, "storeValidateFailedBlockWithWrappedError: Problem adding validate failed block to block index: %v", @@ -615,6 +607,7 @@ func (bc *Blockchain) storeValidateFailedBlockWithWrappedError(block *MsgDeSoBlo func (bc *Blockchain) validateLeaderAndQC( block *MsgDeSoBlock, + blockHash *BlockHash, parentUtxoView *UtxoView, verifySignatures bool, ) (_passedSpamPreventionCheck bool, _err error) { @@ -636,7 +629,7 @@ func (bc *Blockchain) validateLeaderAndQC( "validateLeaderAndQC: Problem getting snapshot epoch number for epoch #%d", currentEpochEntry.EpochNumber) } - isValidPartialSig, err := parentUtxoView.hasValidProposerPartialSignaturePoS(block, snapshotAtEpochNumber) + isValidPartialSig, err := parentUtxoView.hasValidProposerPartialSignaturePoS(block, blockHash, snapshotAtEpochNumber) if err != nil { return false, errors.Wrap(err, "validateLeaderAndQC: Problem validating proposer partial sig") @@ -694,12 +687,12 @@ func (bc *Blockchain) validateLeaderAndQC( // return the new BlockNode. // - Error case: Something goes wrong that doesn't result in the block being marked VALIDATE or VALIDATE_FAILED. In // this case, we will add the block to the block index with status STORED and return the BlockNode. -func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoView *UtxoView, verifySignatures bool) ( - *BlockNode, error) { - blockHash, err := block.Header.Hash() - if err != nil { - return nil, errors.Wrapf(err, "validateAndIndexBlockPoS: Problem hashing block %v", block) - } +func (bc *Blockchain) validateAndIndexBlockPoS( + block *MsgDeSoBlock, + blockHash *BlockHash, + parentUtxoView *UtxoView, + verifySignatures bool, +) (*BlockNode, error) { // Base case - Check if the block is validated or validate failed. If so, we can return early. // TODO: validate height doesn't overflow uint32 @@ -733,13 +726,13 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // this block as ValidateFailed. If the parent is not ValidateFailed, we ONLY store the block and move on. // We don't want to store it as ValidateFailed because we don't know if it's actually invalid. if parentBlockNode.IsValidateFailed() { - return bc.storeValidateFailedBlockWithWrappedError(block, errors.New("parent block is ValidateFailed")) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, errors.New("parent block is ValidateFailed")) } // If the parent block still has a Stored status, it means that we weren't able to validate it // despite trying. The current block will also be stored as a Stored block. if !parentBlockNode.IsValidated() { - return bc.storeBlockInBlockIndex(block) + return bc.storeBlockInBlockIndex(block, blockHash) } // Validate the block's random seed signature @@ -747,14 +740,14 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi isValidRandomSeedSignature, err := bc.hasValidProposerRandomSeedSignaturePoS(block.Header) if err != nil { var innerErr error - blockNode, innerErr = bc.storeBlockInBlockIndex(block) + blockNode, innerErr = bc.storeBlockInBlockIndex(block, blockHash) if innerErr != nil { return nil, errors.Wrapf(innerErr, "validateAndIndexBlockPoS: Problem adding block to block index: %v", err) } return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem validating random seed signature") } if !isValidRandomSeedSignature { - return bc.storeValidateFailedBlockWithWrappedError(block, errors.New("invalid random seed signature")) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, errors.New("invalid random seed signature")) } } @@ -762,15 +755,15 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi serializedBlock, err := block.ToBytes(false) if err != nil { return bc.storeValidateFailedBlockWithWrappedError( - block, errors.Wrap(err, "validateAndIndexBlockPoS: Problem serializing block")) + block, blockHash, errors.Wrap(err, "validateAndIndexBlockPoS: Problem serializing block")) } if uint64(len(serializedBlock)) > parentUtxoView.GetCurrentGlobalParamsEntry().MaxBlockSizeBytesPoS { - return bc.storeValidateFailedBlockWithWrappedError(block, RuleErrorBlockTooBig) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, RuleErrorBlockTooBig) } // Check if the block is properly formed and passes all basic validations. if err = bc.isValidBlockPoS(block); err != nil { - return bc.storeValidateFailedBlockWithWrappedError(block, err) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, err) } // Connect this block to the parent block's UtxoView. @@ -781,7 +774,7 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi // If we fail to connect the block, then it means the block is invalid. We should store it as ValidateFailed. if _, err = parentUtxoView.ConnectBlock(block, txHashes, verifySignatures, nil, block.Header.Height); err != nil { // If it doesn't connect, we want to mark it as ValidateFailed. - return bc.storeValidateFailedBlockWithWrappedError(block, err) + return bc.storeValidateFailedBlockWithWrappedError(block, blockHash, err) } // If the block is too far in the future, we leave it as STORED and return early. @@ -790,11 +783,11 @@ func (bc *Blockchain) validateAndIndexBlockPoS(block *MsgDeSoBlock, parentUtxoVi return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem checking block timestamp") } if failsTimestampDriftCheck { - return bc.storeBlockInBlockIndex(block) + return bc.storeBlockInBlockIndex(block, blockHash) } // We can now add this block to the block index since we have performed all basic validations. - blockNode, err = bc.storeValidatedBlockInBlockIndex(block) + blockNode, err = bc.storeValidatedBlockInBlockIndex(block, blockHash) if err != nil { return blockNode, errors.Wrap(err, "validateAndIndexBlockPoS: Problem adding block to block index: ") } @@ -840,7 +833,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( parentUtxoView := parentUtxoViewAndUtxoOps.UtxoView // If the block isn't validated or validate failed, we need to run the anti-spam checks on it. - passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, parentUtxoView, verifySignatures) + passedSpamPreventionCheck, err := bc.validateLeaderAndQC(block, blockHash, parentUtxoView, verifySignatures) if err != nil { // If we hit an error, that means there was an intermittent issue when trying to // validate the QC or the leader. @@ -849,7 +842,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( if !passedSpamPreventionCheck { // If the QC or Leader check failed, we'll never accept this block, but we've already stored it, // so we need to mark it as ValidateFailed. - blockNode, err = bc.storeValidateFailedBlockInBlockIndex(block) + blockNode, err = bc.storeValidateFailedBlockInBlockIndex(block, blockHash) if err != nil { return nil, errors.Wrap(err, "validatePreviouslyIndexedBlockPoS: Problem adding validate failed block to block index") @@ -858,7 +851,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( } // We run the full validation algorithm on the block. - return bc.validateAndIndexBlockPoS(block, parentUtxoView, verifySignatures) + return bc.validateAndIndexBlockPoS(block, blockHash, parentUtxoView, verifySignatures) } // isValidBlockPoS performs all basic block integrity checks. Any error @@ -1132,8 +1125,20 @@ func (bc *Blockchain) hasValidProposerRandomSeedSignaturePoS(header *MsgDeSoHead return isVerified, nil } -func (bav *UtxoView) hasValidProposerPartialSignaturePoS(block *MsgDeSoBlock, snapshotAtEpochNumber uint64) ( - bool, error) { +func (bav *UtxoView) hasValidProposerPartialSignaturePoS( + block *MsgDeSoBlock, + blockHash *BlockHash, + snapshotAtEpochNumber uint64, +) (bool, error) { + // If we aren't provided a hash, we can just compute it on the fly. + // It's more efficient for us not to recompute the hash though, so we only do it if we have to. + if blockHash == nil { + var err error + blockHash, err = block.Hash() + if err != nil { + return false, errors.Wrapf(err, "hasValidProposerPartialSignaturePoS: Problem hashing block") + } + } votingPublicKey := block.Header.ProposerVotingPublicKey proposerPartialSig := block.Header.ProposerVotePartialSignature // If the proposer partial sig is nil, we can't validate it. That's an error. @@ -1157,11 +1162,6 @@ func (bav *UtxoView) hasValidProposerPartialSignaturePoS(block *MsgDeSoBlock, sn if !snapshotBlockProposerValidatorEntry.VotingPublicKey.Eq(votingPublicKey) { return false, nil } - // Get the block's hash - blockHash, err := block.Header.Hash() - if err != nil { - return false, errors.Wrapf(err, "hasValidProposerPartialSignaturePoS: Problem hashing block") - } // Now that we have the snapshot validator entry and validated that the // voting public key from this block's header matches the snapshotted // voting public key, we can validate the partial sig. @@ -1380,10 +1380,15 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( // getOrCreateBlockNodeFromBlockIndex returns the block node from the block index if it exists. // Otherwise, it creates a new block node and adds it to the blockIndexByHash and blockIndexByHeight. -func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - hash, err := block.Header.Hash() - if err != nil { - return nil, errors.Wrapf(err, "getOrCreateBlockNodeFromBlockIndex: Problem hashing block %v", block) +func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + // If we aren't provided a hash, we can just compute it on the fly. + // It's more efficient for us not to recompute the hash though, so we only do it if we have to. + if hash == nil { + var err error + hash, err = block.Hash() + if err != nil { + return nil, errors.Wrapf(err, "storeBlockInBlockIndex: Problem hashing block") + } } blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) @@ -1401,8 +1406,8 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock) (* // storeBlockInBlockIndex upserts the blocks into the in-memory block index & badger and updates its status to // StatusBlockStored. It also writes the block to the block index in badger -func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}) +func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidatedHeaderInBlockIndex: Problem getting or creating block node") } @@ -1420,8 +1425,8 @@ func (bc *Blockchain) storeValidatedHeaderInBlockIndex(header *MsgDeSoHeader) (* return blockNode, nil } -func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(header *MsgDeSoHeader, wrapperError error) error { - if _, innerErr := bc.storeValidateFailedHeaderInBlockIndex(header); innerErr != nil { +func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(header *MsgDeSoHeader, hash *BlockHash, wrapperError error) error { + if _, innerErr := bc.storeValidateFailedHeaderInBlockIndex(header, hash); innerErr != nil { return errors.Wrapf(innerErr, "%v", wrapperError) } return wrapperError @@ -1429,8 +1434,8 @@ func (bc *Blockchain) storeValidateFailedHeaderInBlockIndexWithWrapperError(head // storeValidateFailedHeaderInBlockIndex stores the header in the block index only and sets its status to // StatusHeaderValidateFailed. It does not write the header to the DB. -func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeader) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}) +func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeader, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(&MsgDeSoBlock{Header: header}, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidateFailedHeaderInBlockIndex: Problem getting or creating block node") } @@ -1451,8 +1456,8 @@ func (bc *Blockchain) storeValidateFailedHeaderInBlockIndex(header *MsgDeSoHeade // storeBlockInBlockIndex upserts the blocks into the in-memory block index & badger and updates its status to // StatusBlockStored. It also writes the block to the block index in badger // by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeBlockInBlockIndex: Problem getting or creating block node") } @@ -1472,8 +1477,8 @@ func (bc *Blockchain) storeBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, e // status to StatusBlockValidated. If it does not have the status StatusBlockStored already, we add that as we // will store the block in the DB after updating its status. It also writes the block to the block index in // badger by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidatedBlockInBlockIndex: Problem getting or creating block node") } @@ -1502,8 +1507,8 @@ func (bc *Blockchain) storeValidatedBlockInBlockIndex(block *MsgDeSoBlock) (*Blo // status to StatusBlockValidateFailed. If it does not have the status StatusBlockStored already, we add that as we // will store the block in the DB after updating its status. It also writes the block to the block index in badger // by calling upsertBlockAndBlockNodeToDB. -func (bc *Blockchain) storeValidateFailedBlockInBlockIndex(block *MsgDeSoBlock) (*BlockNode, error) { - blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block) +func (bc *Blockchain) storeValidateFailedBlockInBlockIndex(block *MsgDeSoBlock, hash *BlockHash) (*BlockNode, error) { + blockNode, err := bc.getOrCreateBlockNodeFromBlockIndex(block, hash) if err != nil { return nil, errors.Wrapf(err, "storeValidateFailedBlockInBlockIndex: Problem getting or creating block node") } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 291a34eab..053d33bac 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -374,10 +374,10 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { }, }, } - blockNode, err := bc.storeBlockInBlockIndex(block) - require.NoError(t, err) newHash, err := block.Hash() require.NoError(t, err) + blockNode, err := bc.storeBlockInBlockIndex(block, newHash) + require.NoError(t, err) // Check the block index by hash blockNodeFromIndex, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uint64(blockNode.Height)) require.True(t, exists) @@ -400,7 +400,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.NoError(t, err) require.True(t, bytes.Equal(uncommittedBytes, origBlockBytes)) // Okay now we update the status of the block to include validated. - blockNode, err = bc.storeValidatedBlockInBlockIndex(block) + blockNode, err = bc.storeValidatedBlockInBlockIndex(block, newHash) require.NoError(t, err) blockNodeFromIndex, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(newHash, uncommittedBlock.Header.Height) require.True(t, exists) @@ -425,7 +425,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.False(t, updatedBlockHash.IsEqual(newHash)) // Okay now put this new block in there. - blockNode, err = bc.storeBlockInBlockIndex(block) + blockNode, err = bc.storeBlockInBlockIndex(block, updatedBlockHash) require.NoError(t, err) // Make sure the blockIndexByHash is correct. updatedBlockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(updatedBlockHash, uint64(blockNode.Height)) @@ -446,7 +446,7 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { // If we're missing a field in the header, we should get an error // as we can't compute the hash. block.Header.ProposerVotingPublicKey = nil - _, err = bc.storeBlockInBlockIndex(block) + _, err = bc.storeBlockInBlockIndex(block, nil) require.Error(t, err) } @@ -2008,7 +2008,7 @@ func TestGetSafeBlocks(t *testing.T) { block1Hash, err := block1.Hash() require.NoError(t, err) // Add block 1 w/ stored and validated - bn1, err := testMeta.chain.storeValidatedBlockInBlockIndex(block1) + bn1, err := testMeta.chain.storeValidatedBlockInBlockIndex(block1, nil) require.NoError(t, err) require.True(t, bn1.Hash.IsEqual(block1Hash)) // Create block 2 w/ block 1 as parent and add it to the block index w/ stored & validated @@ -2016,13 +2016,13 @@ func TestGetSafeBlocks(t *testing.T) { block2 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+1), uint64(testMeta.savedHeight+1), 1293, block1Hash, false) block2Hash, err := block2.Hash() require.NoError(t, err) - bn2, err := testMeta.chain.storeValidatedBlockInBlockIndex(block2) + bn2, err := testMeta.chain.storeValidatedBlockInBlockIndex(block2, nil) require.NoError(t, err) require.True(t, bn2.Hash.IsEqual(block2Hash)) // Add block 3 only as stored and validated var block3 *MsgDeSoBlock block3 = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+2), uint64(testMeta.savedHeight+2), 1372, block2Hash, false) - bn3, err := testMeta.chain.storeValidatedBlockInBlockIndex(block3) + bn3, err := testMeta.chain.storeValidatedBlockInBlockIndex(block3, nil) require.NoError(t, err) block3Hash, err := block3.Hash() require.NoError(t, err) @@ -2030,7 +2030,7 @@ func TestGetSafeBlocks(t *testing.T) { // Add block 3' only as stored var block3Prime *MsgDeSoBlock block3Prime = _generateRealBlock(testMeta, uint64(testMeta.savedHeight+2), uint64(testMeta.savedHeight+3), 137175, block2Hash, false) - bn3Prime, err := testMeta.chain.storeBlockInBlockIndex(block3Prime) + bn3Prime, err := testMeta.chain.storeBlockInBlockIndex(block3Prime, nil) require.NoError(t, err) block3PrimeHash, err := block3Prime.Hash() require.NoError(t, err) @@ -2041,7 +2041,7 @@ func TestGetSafeBlocks(t *testing.T) { block5.Header.Height = uint64(testMeta.savedHeight + 5) block5Hash, err := block5.Hash() require.NoError(t, err) - _, err = testMeta.chain.storeValidatedBlockInBlockIndex(block5) + _, err = testMeta.chain.storeValidatedBlockInBlockIndex(block5, nil) require.NoError(t, err) // Okay let's get the safe blocks. safeBlocks, err := testMeta.chain.GetSafeBlocks() @@ -2062,7 +2062,7 @@ func TestGetSafeBlocks(t *testing.T) { require.False(t, _checkSafeBlocksForBlockHash(block5Hash, safeBlocks)) // Update block 3 prime to be validated and it should now be a safe block. - bn3Prime, err = testMeta.chain.storeValidatedBlockInBlockIndex(block3Prime) + bn3Prime, err = testMeta.chain.storeValidatedBlockInBlockIndex(block3Prime, nil) require.NoError(t, err) require.True(t, bn3Prime.IsValidated()) safeBlocks, err = testMeta.chain.GetSafeBlocks() @@ -2345,7 +2345,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { utxoView := _newUtxoView(testMeta) snapshotEpochNumber, err := utxoView.GetCurrentSnapshotEpochNumber() require.NoError(t, err) - isValid, err := utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err := utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.True(t, isValid) @@ -2353,7 +2353,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { realVotingPublicKey := realBlock.Header.ProposerVotingPublicKey { realBlock.Header.ProposerVotingPublicKey = _generateRandomBLSPrivateKey(t).PublicKey() - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) // Reset the proposer voting public key @@ -2365,7 +2365,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { incorrectPayload := consensus.GetVoteSignaturePayload(13, testMeta.chain.BlockTip().Hash) realBlock.Header.ProposerVotePartialSignature, err = testMeta.blsPubKeyToBLSKeyMap[realBlock.Header.ProposerVotingPublicKey.ToString()].Sign(incorrectPayload[:]) - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) } @@ -2378,7 +2378,7 @@ func TestHasValidProposerPartialSignaturePoS(t *testing.T) { correctPayload := consensus.GetVoteSignaturePayload(12, realBlockHash) wrongPrivateKey := _generateRandomBLSPrivateKey(t) realBlock.Header.ProposerVotePartialSignature, err = wrongPrivateKey.Sign(correctPayload[:]) - isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, snapshotEpochNumber) + isValid, err = utxoView.hasValidProposerPartialSignaturePoS(realBlock, nil, snapshotEpochNumber) require.NoError(t, err) require.False(t, isValid) } @@ -2543,7 +2543,7 @@ func _generateDummyBlock(testMeta *TestMeta, blockHeight uint64, view uint64, se require.NoError(testMeta.t, err) // Add block to block index. - blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock) + blockNode, err := testMeta.chain.storeBlockInBlockIndex(msgDesoBlock, nil) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) _, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(newBlockHash, msgDesoBlock.Header.Height) @@ -2568,7 +2568,7 @@ func _generateBlockAndAddToBestChain(testMeta *TestMeta, blockHeight uint64, vie newBlockHash, err := msgDesoBlock.Hash() require.NoError(testMeta.t, err) // Add block to block index. - blockNode, err := testMeta.chain.storeValidatedBlockInBlockIndex(msgDesoBlock) + blockNode, err := testMeta.chain.storeValidatedBlockInBlockIndex(msgDesoBlock, nil) require.NoError(testMeta.t, err) require.True(testMeta.t, blockNode.IsStored()) require.True(testMeta.t, blockNode.IsValidated()) From 23991c4c60e132d497de045b7e784ed5de428985 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 22:45:40 -0400 Subject: [PATCH 028/131] upgrade cloudflare/circl --- go.mod | 10 +++++----- go.sum | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 02c87cd5f..eeec46a9b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible - github.com/cloudflare/circl v1.1.0 + github.com/cloudflare/circl v1.4.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 github.com/decred/dcrd/lru v1.1.3 @@ -37,7 +37,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) @@ -59,7 +59,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect - github.com/bwesterb/go-ristretto v1.2.0 // indirect + github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -114,8 +114,8 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 5536e1ec2..da57a1eb9 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -103,6 +104,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= +github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -587,6 +590,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -715,6 +720,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -728,6 +735,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= From 8adc62459f8c3420d08da9acf99c91b68de7afa1 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Mon, 7 Oct 2024 23:05:38 -0400 Subject: [PATCH 029/131] Ln/replace best chain plus replace old lru cache (#1414) * replace decred lru with hashicorp lru * fix usage of delete -> remove * fix calc next difficulty target issue * upgrade cloudflare/circl --- go.mod | 11 +++++------ go.sum | 11 +++++++++-- lib/block_view_test.go | 5 +++-- lib/blockchain.go | 36 +++++++++++++++++------------------- lib/connection_manager.go | 7 ++++--- lib/db_utils.go | 8 ++++---- lib/network_manager.go | 14 +++++++------- lib/peer.go | 14 +++++++------- lib/pos_blockchain.go | 19 +++++-------------- lib/pos_mempool.go | 16 ++++++++-------- lib/postgres.go | 6 +++--- lib/server.go | 23 ++++++++++------------- lib/snapshot.go | 14 ++++++++------ 13 files changed, 90 insertions(+), 94 deletions(-) diff --git a/go.mod b/go.mod index 02c87cd5f..e786aa20c 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,9 @@ require ( github.com/btcsuite/btcd v0.21.0-beta github.com/btcsuite/btcutil v1.0.2 github.com/bxcodec/faker v2.0.1+incompatible - github.com/cloudflare/circl v1.1.0 + github.com/cloudflare/circl v1.4.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 - github.com/decred/dcrd/lru v1.1.3 github.com/deso-protocol/go-deadlock v1.0.0 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/dgraph-io/badger/v3 v3.2103.5 @@ -37,7 +36,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.0.2 github.com/unrolled/secure v1.0.8 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 gopkg.in/DataDog/dd-trace-go.v1 v1.65.1 ) @@ -59,7 +58,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect - github.com/bwesterb/go-ristretto v1.2.0 // indirect + github.com/bwesterb/go-ristretto v1.2.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -114,8 +113,8 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 5536e1ec2..65e5bdded 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,7 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -103,6 +104,8 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= +github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -128,8 +131,6 @@ github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/decred/dcrd/lru v1.1.3 h1:w9EAbvGLyzm6jTjF83UKuqZEiUtJmvRhQDOCEIvSuE0= -github.com/decred/dcrd/lru v1.1.3/go.mod h1:Tw0i0pJyiLEx/oZdHLe1Wdv/Y7EGzAX+sYftnmxBR4o= github.com/deso-protocol/go-deadlock v1.0.0 h1:mw0pHy/19zgC+JFBStuQt1+1Ehv5OKA5NxXqecnL5ic= github.com/deso-protocol/go-deadlock v1.0.0/go.mod h1:K0Wd2OV2x7ck7SMYDraWerpKjFKUeBqaFcwz21tmkb8= github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+LsH/kRdL5l/Yzvg= @@ -587,6 +588,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -715,6 +718,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -728,6 +733,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= diff --git a/lib/block_view_test.go b/lib/block_view_test.go index ff17677a4..903f4ed4f 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -13,10 +13,10 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec" - "github.com/decred/dcrd/lru" "github.com/dgraph-io/badger/v3" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -797,7 +797,8 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. if tm.chain.snapshot != nil { - tm.chain.snapshot.DatabaseCache = lru.NewKVCache(DatabaseCacheSize) + tm.chain.snapshot.DatabaseCache, err = lru.New[string, []byte](int(DatabaseCacheSize)) + require.NoError(err) } // Note that unlike connecting test vectors, when disconnecting, we don't need to verify db entries. diff --git a/lib/blockchain.go b/lib/blockchain.go index 3ecdd2b5e..06ef50632 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -16,9 +16,7 @@ import ( "sync" "time" - "github.com/decred/dcrd/lru" - - lru2 "github.com/hashicorp/golang-lru/v2" + "github.com/hashicorp/golang-lru/v2" "github.com/deso-protocol/core/collections" @@ -567,15 +565,15 @@ type CheckpointBlockInfoAndError struct { type BlockIndex struct { db *badger.DB snapshot *Snapshot - blockIndexByHash *lru2.Cache[BlockHash, *BlockNode] - blockIndexByHeight *lru2.Cache[uint64, []*BlockNode] + blockIndexByHash *lru.Cache[BlockHash, *BlockNode] + blockIndexByHeight *lru.Cache[uint64, []*BlockNode] tip *BlockNode headerTip *BlockNode } func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { - blockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? - blockIndexByHeight, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHeight, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ db: db, snapshot: snapshot, @@ -586,8 +584,8 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newHeightToBlockNodeMap, _ := lru2.New[uint64, []*BlockNode](MaxBlockIndexNodes) + newHashToBlockNodeMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newHeightToBlockNodeMap, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) bi.blockIndexByHash = newHashToBlockNodeMap bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { @@ -740,7 +738,7 @@ type Blockchain struct { blockView *UtxoView // cache block view for each block - blockViewCache lru.KVCache + blockViewCache *lru.Cache[BlockHash, *BlockViewAndUtxoOps] // snapshot cache snapshotCache *SnapshotCache @@ -872,9 +870,9 @@ func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], + _blockIndexByHash *lru.Cache[BlockHash, *BlockNode], ) { - newBlockIndexByHash, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBlockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) newBlockIndexByHash.Add(key, val) @@ -1124,7 +1122,7 @@ func NewBlockchain( timer := &Timer{} timer.Initialize() - + blockViewCache, _ := lru.New[BlockHash, *BlockViewAndUtxoOps](100) // TODO: parameterize bc := &Blockchain{ db: db, postgres: postgres, @@ -1138,7 +1136,7 @@ func NewBlockchain( archivalMode: archivalMode, blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. - blockViewCache: lru.NewKVCache(100), // TODO: parameterize + blockViewCache: blockViewCache, snapshotCache: NewSnapshotCache(), checkpointSyncingProviders: checkpointSyncingProviders, @@ -1953,7 +1951,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai } // TODO: need to figure out how to handle this for exchange api tests. -func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { +func (bc *Blockchain) setBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { @@ -1964,8 +1962,8 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { } } -func (bc *Blockchain) SetBestChainMap( - blockIndexByHash *lru2.Cache[BlockHash, *BlockNode], +func (bc *Blockchain) setBestChainMap( + blockIndexByHash *lru.Cache[BlockHash, *BlockNode], tipNode *BlockNode, ) { bc.blockIndex.blockIndexByHash = blockIndexByHash @@ -2294,8 +2292,8 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru2.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap *lru2.Cache[BlockHash, *BlockNode]) { +func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( + chainList []*BlockNode, chainMap *lru.Cache[BlockHash, *BlockNode]) { // Remove the nodes we detached from the end of the best chain node list. tipIndex := len(mainChainList) - 1 diff --git a/lib/connection_manager.go b/lib/connection_manager.go index 40eaf2fc7..d8b060e71 100644 --- a/lib/connection_manager.go +++ b/lib/connection_manager.go @@ -10,8 +10,8 @@ import ( "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" - "github.com/decred/dcrd/lru" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" ) // connection_manager.go contains most of the logic for creating and managing @@ -52,7 +52,7 @@ type ConnectionManager struct { // Keep track of the nonces we've sent in our version messages so // we can prevent connections to ourselves. - sentNonces lru.Cache + sentNonces *lru.Cache[uint64, struct{}] // This section defines the data structures for storing all the // peers we're aware of. @@ -125,13 +125,14 @@ func NewConnectionManager( ValidateHyperSyncFlags(_hyperSync, _syncType) + sentNoncesCache, _ := lru.New[uint64, struct{}](1000) return &ConnectionManager{ srv: _srv, params: _params, listeners: _listeners, // We keep track of the last N nonces we've sent in order to detect // self connections. - sentNonces: lru.NewCache(1000), + sentNonces: sentNoncesCache, //newestBlock: _newestBlock, // Initialize the peer data structures. diff --git a/lib/db_utils.go b/lib/db_utils.go index 1a231b3b4..345737f75 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -1183,8 +1183,8 @@ func DBGetWithTxn(txn *badger.Txn, snap *Snapshot, key []byte) ([]byte, error) { // Lookup the snapshot cache and check if we've already stored a value there. if isState { - if val, exists := snap.DatabaseCache.Lookup(keyString); exists { - return val.([]byte), nil + if val, exists := snap.DatabaseCache.Get(keyString); exists { + return val, nil } } @@ -1241,7 +1241,7 @@ func DBDeleteWithTxn(txn *badger.Txn, snap *Snapshot, key []byte, eventManager * return errors.Wrapf(err, "DBDeleteWithTxn: Problem preparing ancestral record") } // Now delete the past record from the cache. - snap.DatabaseCache.Delete(keyString) + snap.DatabaseCache.Remove(keyString) // We have to remove the previous value from the state checksum. // Because checksum is commutative, we can safely remove the past value here. if !snap.disableChecksum { @@ -2810,7 +2810,7 @@ func DBGetAccessGroupExistenceByAccessGroupIdWithTxn(txn *badger.Txn, snap *Snap // Lookup the snapshot cache and check if we've already stored a value there. if isState { - if _, exists := snap.DatabaseCache.Lookup(keyString); exists { + if exists := snap.DatabaseCache.Contains(keyString); exists { return true, nil } } diff --git a/lib/network_manager.go b/lib/network_manager.go index bbbaa7c19..ff7f8b8fa 100644 --- a/lib/network_manager.go +++ b/lib/network_manager.go @@ -11,11 +11,11 @@ import ( "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" - "github.com/decred/dcrd/lru" "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -69,7 +69,7 @@ type NetworkManager struct { NonValidatorInboundIndex *collections.ConcurrentMap[RemoteNodeId, *RemoteNode] // Cache of nonces used during handshake. - usedNonces lru.Cache + usedNonces *lru.Cache[uint64, struct{}] // The address manager keeps track of peer addresses we're aware of. When // we need to connect to a new outbound peer, it chooses one of the addresses @@ -121,7 +121,7 @@ func NewNetworkManager( minTxFeeRateNanosPerKB uint64, nodeServices ServiceFlag, ) *NetworkManager { - + usedNoncesCache, _ := lru.New[uint64, struct{}](1000) return &NetworkManager{ params: params, srv: srv, @@ -136,7 +136,7 @@ func NewNetworkManager( ValidatorOutboundIndex: collections.NewConcurrentMap[bls.SerializedPublicKey, *RemoteNode](), NonValidatorOutboundIndex: collections.NewConcurrentMap[RemoteNodeId, *RemoteNode](), NonValidatorInboundIndex: collections.NewConcurrentMap[RemoteNodeId, *RemoteNode](), - usedNonces: lru.NewCache(1000), + usedNonces: usedNoncesCache, connectIps: connectIps, persistentIpToRemoteNodeIdsMap: collections.NewConcurrentMap[string, RemoteNodeId](), activeValidatorsMap: collections.NewConcurrentMap[bls.SerializedPublicKey, consensus.Validator](), @@ -261,7 +261,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag // If we've seen this nonce before then return an error since this is a connection from ourselves. msgNonce := verMsg.Nonce if nm.usedNonces.Contains(msgNonce) { - nm.usedNonces.Delete(msgNonce) + nm.usedNonces.Remove(msgNonce) glog.Errorf("NetworkManager.handleVersionMessage: Disconnecting RemoteNode with id: (%v) "+ "nonce collision, nonce (%v)", origin.ID, msgNonce) nm.Disconnect(rn, "nonce collision") @@ -277,7 +277,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag return } - nm.usedNonces.Add(responseNonce) + nm.usedNonces.Add(responseNonce, struct{}{}) } // _handleVerackMessage is called when a new verack message is received. @@ -1248,7 +1248,7 @@ func (nm *NetworkManager) InitiateHandshake(rn *RemoteNode) { glog.Errorf("NetworkManager.InitiateHandshake: Error initiating handshake: %v", err) nm.Disconnect(rn, fmt.Sprintf("error initiating handshake: %v", err)) } - nm.usedNonces.Add(nonce) + nm.usedNonces.Add(nonce, struct{}{}) } // handleHandshakeComplete is called on a completed handshake with a RemoteNodes. diff --git a/lib/peer.go b/lib/peer.go index aece3eeff..73c4c4912 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/decred/dcrd/lru" + "github.com/hashicorp/golang-lru/v2" "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/go-deadlock" @@ -111,7 +111,7 @@ type Peer struct { // Inventory stuff. // The inventory that we know the peer already has. - knownInventory lru.Cache + knownInventory *lru.Cache[InvVect, struct{}] // Whether the peer is ready to receive INV messages. For a peer that // still needs a mempool download, this is false. @@ -292,7 +292,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { for _, invVect := range msg.InvList { // No matter what, add the inv to the peer's known inventory. - pp.knownInventory.Add(*invVect) + pp.knownInventory.Add(*invVect, struct{}{}) // If this is a hash we are currently processing, no need to do anything. // This check serves to fill the gap between the time when we've decided @@ -344,7 +344,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { // If we made it here, it means the inventory was added to one of the // lists so mark it as processed on the Server. - pp.srv.inventoryBeingProcessed.Add(*invVect) + pp.srv.inventoryBeingProcessed.Add(*invVect, struct{}{}) } // If there were any transactions we don't yet have, request them using @@ -644,7 +644,7 @@ func NewPeer(_id uint64, _conn net.Conn, _isOutbound bool, _netAddr *wire.NetAdd _cmgr *ConnectionManager, _srv *Server, _syncType NodeSyncType, peerDisconnectedChan chan *Peer) *Peer { - + knownInventory, _ := lru.New[InvVect, struct{}](maxKnownInventory) pp := Peer{ ID: _id, cmgr: _cmgr, @@ -657,7 +657,7 @@ func NewPeer(_id uint64, _conn net.Conn, _isOutbound bool, _netAddr *wire.NetAdd outputQueueChan: make(chan DeSoMessage), peerDisconnectedChan: peerDisconnectedChan, quit: make(chan interface{}), - knownInventory: lru.NewCache(maxKnownInventory), + knownInventory: knownInventory, blocksToSend: make(map[BlockHash]bool), stallTimeoutSeconds: _stallTimeoutSeconds, minTxFeeRateNanosPerKB: _minFeeRateNanosPerKB, @@ -983,7 +983,7 @@ out: // Add the new inventory to the peer's knownInventory. for _, invVect := range invMsg.InvList { - pp.knownInventory.Add(*invVect) + pp.knownInventory.Add(*invVect, struct{}{}) } } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 2b0383f95..ff793e940 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1937,16 +1937,9 @@ func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { return blockViewAndUtxoOps.UtxoView, nil } -func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockViewAndUtxoOps, error, bool) { - if viewAndUtxoOpsAtHashItem, exists := bc.blockViewCache.Lookup(blockHash); exists { - viewAndUtxoOpsAtHash, ok := viewAndUtxoOpsAtHashItem.(*BlockViewAndUtxoOps) - if !ok { - glog.Errorf("getCachedBlockViewAndUtxoOps: Problem casting to BlockViewAndUtxoOps") - return nil, fmt.Errorf("getCachedBlockViewAndUtxoOps: Problem casting to BlockViewAndUtxoOps"), false - } - return viewAndUtxoOpsAtHash, nil, true - } - return nil, nil, false +func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockViewAndUtxoOps, bool) { + viewAndUtxoOpsAtHash, exists := bc.blockViewCache.Get(blockHash) + return viewAndUtxoOpsAtHash, exists } // getUtxoViewAndUtxoOpsAtBlockHash builds a UtxoView to the block provided and returns a BlockViewAndUtxoOps @@ -1993,10 +1986,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc "getUtxoViewAndUtxoOpsAtBlockHash: extends from a committed block that isn't the committed tip") } } - viewAndUtxoOpsAtHash, err, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) - if err != nil { - return nil, errors.Wrapf(err, "getUtxoViewAndUtxoOpsAtBlockHash: Problem getting cached BlockViewAndUtxoOps") - } + viewAndUtxoOpsAtHash, exists := bc.getCachedBlockViewAndUtxoOps(blockHash) if exists { viewAndUtxoOpsCopy := viewAndUtxoOpsAtHash.Copy() return viewAndUtxoOpsCopy, nil @@ -2011,6 +2001,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { glog.Infof("Connecting block %v", uncommittedAncestors[ii]) + var err error // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) if err != nil { diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index ae8a2c94a..73c162b27 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/decred/dcrd/lru" + "github.com/hashicorp/golang-lru/v2" "github.com/dgraph-io/badger/v3" "github.com/golang/glog" @@ -183,11 +183,11 @@ type PosMempool struct { // recentBlockTxnCache is an LRU KV cache used to track the transaction that have been included in blocks. // This cache is used to power logic that waits for a transaction to either be validated in the mempool // or be included in a block. - recentBlockTxnCache lru.KVCache + recentBlockTxnCache *lru.Cache[BlockHash, struct{}] // recentRejectedTxnCache is a cache to store the txns that were recently rejected so that we can return better // errors for them. - recentRejectedTxnCache lru.KVCache + recentRejectedTxnCache *lru.Cache[BlockHash, error] } func NewPosMempool() *PosMempool { @@ -234,8 +234,8 @@ func (mp *PosMempool) Init( mp.mempoolBackupIntervalMillis = mempoolBackupIntervalMillis mp.maxValidationViewConnects = maxValidationViewConnects mp.transactionValidationRefreshIntervalMillis = transactionValidationRefreshIntervalMillis - mp.recentBlockTxnCache = lru.NewKVCache(100000) // cache 100K latest txns from blocks. - mp.recentRejectedTxnCache = lru.NewKVCache(100000) // cache 100K rejected txns. + mp.recentBlockTxnCache, _ = lru.New[BlockHash, struct{}](100000) // cache 100K latest txns from blocks. + mp.recentRejectedTxnCache, _ = lru.New[BlockHash, error](100000) // cache 100K rejected txns. // Recreate and initialize the transaction register and the nonce tracker. mp.txnRegister = NewTransactionRegister() @@ -486,11 +486,11 @@ func (mp *PosMempool) AddTransaction(txn *MsgDeSoTxn, txnTimestamp time.Time) er } func (mp *PosMempool) addTxnHashToRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Add(txnHash, nil) + mp.recentBlockTxnCache.Add(txnHash, struct{}{}) } func (mp *PosMempool) deleteTxnHashFromRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Delete(txnHash) + mp.recentBlockTxnCache.Remove(txnHash) } func (mp *PosMempool) isTxnHashInRecentBlockCache(txnHash BlockHash) bool { return mp.recentBlockTxnCache.Contains(txnHash) @@ -1023,7 +1023,7 @@ func (mp *PosMempool) WaitForTxnValidation(txHash *BlockHash) error { checkIntervalMillis = 1 } for { - rejectionErr, wasRejected := mp.recentRejectedTxnCache.Lookup(*txHash) + rejectionErr, wasRejected := mp.recentRejectedTxnCache.Get(*txHash) if wasRejected { return rejectionErr.(error) } diff --git a/lib/postgres.go b/lib/postgres.go index 08d078006..e919c8e78 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - lru2 "github.com/hashicorp/golang-lru/v2" + "github.com/hashicorp/golang-lru/v2" "net/url" "regexp" "strings" @@ -1305,14 +1305,14 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*lru2.Cache[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*lru.Cache[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap, _ := lru2.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { blockMap.Add(*block.Hash, &BlockNode{ Hash: block.Hash, diff --git a/lib/server.go b/lib/server.go index 28b225a48..8a9b17a21 100644 --- a/lib/server.go +++ b/lib/server.go @@ -12,20 +12,17 @@ import ( "sync/atomic" "time" - "github.com/btcsuite/btcd/wire" - "github.com/deso-protocol/core/collections" - "github.com/deso-protocol/core/consensus" - - "github.com/decred/dcrd/lru" - "github.com/DataDog/datadog-go/v5/statsd" - "github.com/btcsuite/btcd/addrmgr" chainlib "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" + "github.com/deso-protocol/core/collections" + "github.com/deso-protocol/core/consensus" "github.com/deso-protocol/go-deadlock" "github.com/dgraph-io/badger/v3" "github.com/golang/glog" + "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -91,7 +88,7 @@ type Server struct { // adding it to this map and checking this map before replying will make it // so that we only send a reply to the first peer that sent us the inv, which // is more efficient. - inventoryBeingProcessed lru.Cache + inventoryBeingProcessed *lru.Cache[InvVect, struct{}] // hasRequestedSync indicates whether we've bootstrapped our mempool // by requesting all mempool transactions from a // peer. It's initially false @@ -229,7 +226,7 @@ func (srv *Server) _removeRequest(hash *BlockHash) { Type: InvTypeTx, Hash: *hash, } - srv.inventoryBeingProcessed.Delete(*invVect) + srv.inventoryBeingProcessed.Remove(*invVect) } // dataLock must be acquired for writing before calling this function. @@ -704,7 +701,7 @@ func NewServer( srv.blockProducer = _blockProducer srv.incomingMessages = _incomingMessages // Make this hold a multiple of what we hold for individual peers. - srv.inventoryBeingProcessed = lru.NewCache(maxKnownInventory) + srv.inventoryBeingProcessed, _ = lru.New[InvVect, struct{}](maxKnownInventory) srv.requestTimeoutSeconds = 10 srv.statsdClient = statsd @@ -1781,7 +1778,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { } // We also reset the in-memory snapshot cache, because it is populated with stale records after // we've initialized the chain with seed transactions. - srv.snapshot.DatabaseCache = lru.NewKVCache(DatabaseCacheSize) + srv.snapshot.DatabaseCache, _ = lru.New[string, []byte](int(DatabaseCacheSize)) // If we got here then we finished the snapshot sync so set appropriate flags. srv.blockchain.syncingState = false @@ -2067,8 +2064,8 @@ func (srv *Server) _relayTransactions() { // Add the transaction to the peer's known inventory. We do // it here when we enqueue the message to the peers outgoing - // message queue so that we don't have remember to do it later. - pp.knownInventory.Add(*invVect) + // message queue so that we don't have to remember to do it later. + pp.knownInventory.Add(*invVect, struct{}{}) invMsg.InvList = append(invMsg.InvList, invVect) } if len(invMsg.InvList) > 0 { diff --git a/lib/snapshot.go b/lib/snapshot.go index 70cb41c1c..7472e00c7 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -13,12 +13,12 @@ import ( "time" "github.com/cloudflare/circl/group" - "github.com/decred/dcrd/lru" "github.com/deso-protocol/go-deadlock" "github.com/dgraph-io/badger/v3" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" + "github.com/hashicorp/golang-lru/v2" "github.com/oleiade/lane" "github.com/pkg/errors" "golang.org/x/sync/semaphore" @@ -313,7 +313,7 @@ type Snapshot struct { // DatabaseCache is used to store most recent DB records that we've read/written. // This is a low-level optimization for ancestral records that // saves us read time when we're writing to the DB during UtxoView flush. - DatabaseCache lru.KVCache + DatabaseCache *lru.Cache[string, []byte] // AncestralFlushCounter is used to offset ancestral records flush to occur only after x blocks. AncestralFlushCounter uint64 @@ -483,11 +483,13 @@ func NewSnapshot( "This may lead to unexpected behavior.") } + databaseCache, _ := lru.New[string, []byte](int(DatabaseCacheSize)) + // Set the snapshot. snap := &Snapshot{ mainDb: mainDb, SnapshotDbMutex: &snapshotDbMutex, - DatabaseCache: lru.NewKVCache(DatabaseCacheSize), + DatabaseCache: databaseCache, AncestralFlushCounter: uint64(0), snapshotBlockHeightPeriod: snapshotBlockHeightPeriod, OperationChannel: operationChannel, @@ -1394,7 +1396,7 @@ type StateChecksum struct { ctx context.Context // hashToCurveCache is a cache of computed hashToCurve mappings - hashToCurveCache lru.KVCache + hashToCurveCache *lru.Cache[string, group.Element] // When we want to add a database record to the state checksum, we will first have to // map the record to the Ristretto255 curve using the hash_to_curve. We will then add the @@ -1422,7 +1424,7 @@ func (sc *StateChecksum) Initialize(mainDb *badger.DB, snapshotDbMutex *sync.Mut sc.maxWorkers = int64(runtime.GOMAXPROCS(0)) // Set the hashToCurveCache - sc.hashToCurveCache = lru.NewKVCache(HashToCurveCache) + sc.hashToCurveCache, _ = lru.New[string, group.Element](int(HashToCurveCache)) // Set the worker pool semaphore and context. sc.semaphore = semaphore.NewWeighted(sc.maxWorkers) @@ -1487,7 +1489,7 @@ func (sc *StateChecksum) HashToCurve(bytes []byte) group.Element { // Check if we've already mapped this element, if so we will save some computation this way. bytesStr := hex.EncodeToString(bytes) - if elem, exists := sc.hashToCurveCache.Lookup(bytesStr); exists { + if elem, exists := sc.hashToCurveCache.Get(bytesStr); exists { hashElement = elem.(group.Element) } else { // Compute the hash_to_curve primitive, mapping the bytes to an elliptic curve point. From dbad6493af488515c0740bbedf77e3f00db5cb4b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 23:07:55 -0400 Subject: [PATCH 030/131] go mod tidy --- go.sum | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/go.sum b/go.sum index 65e5bdded..d0944531d 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,7 @@ github.com/btcsuite/snappy-go v1.0.0 h1:ZxaA6lo2EpxGddsA8JwWOcxlzRybb444sgmeJQMJ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bwesterb/go-ristretto v1.2.0 h1:xxWOVbN5m8NNKiSDZXE1jtZvZnC6JSJ9cYFADiZcWtw= -github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= @@ -102,8 +101,6 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= -github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.4.0 h1:BV7h5MgrktNzytKmWjpOtdYrf0lkkbF8YMlBGPhJQrY= github.com/cloudflare/circl v1.4.0/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= @@ -586,8 +583,6 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -707,7 +702,6 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -716,8 +710,6 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -731,8 +723,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ebb117593b226ca99313378d92ba4caaea251c98 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 7 Oct 2024 23:17:41 -0400 Subject: [PATCH 031/131] expose SetBestChain --- lib/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 06ef50632..5dd5da6e5 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1951,7 +1951,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai } // TODO: need to figure out how to handle this for exchange api tests. -func (bc *Blockchain) setBestChain(bestChain []*BlockNode) { +func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { for _, blockNode := range bestChain { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) if bc.blockIndex.GetTip() == nil { From ae34cff3f76f1c4a2e90784503e9c7dd8384ce3f Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 11:49:48 -0400 Subject: [PATCH 032/131] fix exit condition for GetBlocksToStore --- lib/server.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/server.go b/lib/server.go index 8a9b17a21..bad4143b3 100644 --- a/lib/server.go +++ b/lib/server.go @@ -897,16 +897,18 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } srv.blockchain.lowestBlockNotStored = uint64(blockNode.Height) numBlocksToFetch := maxBlocksInFlight - len(pp.requestedBlocks) - currentHeight := int(blockNode.Height) + currentHeight := uint64(blockNode.Height) blockNodesToFetch := []*BlockNode{} // In case there are blocks at tip that are already stored (which shouldn't really happen), we'll not download them. // We filter those out in the loop below by checking IsFullyProcessed. // Find the blocks that we should download. for len(blockNodesToFetch) < numBlocksToFetch { - + if currentHeight > uint64(srv.blockchain.blockTip().Height) { + break + } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(currentHeight), false) + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, false) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return From b48c52d9574d7a39e6ee848d8935909f8a8849e9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 17:59:17 -0400 Subject: [PATCH 033/131] use bytes.Equal instead of reflect.DeepEqual for comparing block hashes --- lib/types.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/types.go b/lib/types.go index 5395d360b..36382ecbd 100644 --- a/lib/types.go +++ b/lib/types.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "reflect" "sort" "github.com/holiman/uint256" @@ -238,7 +237,7 @@ func (bh *BlockHash) IsEqual(target *BlockHash) bool { return false } - return reflect.DeepEqual(bh[:], target[:]) + return bytes.Equal(bh[:], target[:]) } func (bh *BlockHash) NewBlockHash() *BlockHash { From 925e44e34e914be38a651f95817f55cad529f20d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:24:53 -0400 Subject: [PATCH 034/131] use get block from best chain by height to speed up syncing --- lib/blockchain.go | 8 ++++---- lib/pos_blockchain.go | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 5dd5da6e5..df1c1b7a4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,12 +1901,12 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && !currNode.Hash.IsEqual(blockHash) && currNode.Height > committedTip.Height { + for currNode != nil && currNode.Height > committedTip.Height { + if !currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } currNode = currNode.GetParent(bc.blockIndex) } - if currNode != nil && currNode.Hash.IsEqual(blockHash) { - return currNode, true, nil - } return nil, false, nil } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff793e940..426a97bac 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -54,11 +54,12 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + //_, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isHeightInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(header.Height, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } - if isInBestHeaderChain { + if isHeightInBestHeaderChain && blockNode.Hash.IsEqual(headerHash) { return true, false, nil } From dddd64cd99f7e0db5b0337680932e84bd5f44a56 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:32:30 -0400 Subject: [PATCH 035/131] fix condition for returning from get block by hash --- lib/blockchain.go | 2 +- lib/pos_blockchain_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index df1c1b7a4..dd2419669 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1902,7 +1902,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade currNode := &BlockNode{} *currNode = *blockTip for currNode != nil && currNode.Height > committedTip.Height { - if !currNode.Hash.IsEqual(blockHash) { + if currNode.Hash.IsEqual(blockHash) { return currNode, true, nil } currNode = currNode.GetParent(bc.blockIndex) diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 053d33bac..cf2e33f04 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1276,7 +1276,7 @@ func TestShouldReorg(t *testing.T) { // 1. Simple reorg. Just replacing the uncommitted tip. // 2. Create a longer chain and reorg to it. // 3. Make sure no reorg when current view is greater than block's view -// 4. Super happy path of simply extending current uncommitted tip. +// 4. Super happy path of simply extet anding current uncommitted tip. func TestTryApplyNewTip(t *testing.T) { setBalanceModelBlockHeights(t) bc, _, _ := NewTestBlockchain(t) From 62503c12f2c63012485aee13f1678b80aa9a8a5d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 18:47:04 -0400 Subject: [PATCH 036/131] revert using get block from best chain by height --- lib/blockchain.go | 2 +- lib/pos_blockchain.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index dd2419669..00a366dfa 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1896,7 +1896,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade if !exists { return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") } - if uint64(bn.Height) > uint64(blockTip.Height) { + if uint64(bn.Height) > uint64(blockTip.Height) || uint64(bn.Height) < uint64(committedTip.Height) { return nil, false, nil } currNode := &BlockNode{} diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 426a97bac..ff793e940 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -54,12 +54,11 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - //_, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) - blockNode, isHeightInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(header.Height, true) + _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } - if isHeightInBestHeaderChain && blockNode.Hash.IsEqual(headerHash) { + if isInBestHeaderChain { return true, false, nil } From 131f38910eb043bcfc6ee1c491e445a1e974f1f6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:16:05 -0400 Subject: [PATCH 037/131] add height check to reduce calls to compare hashes --- lib/blockchain.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 00a366dfa..a0a9e954f 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,8 +1901,8 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && currNode.Height > committedTip.Height { - if currNode.Hash.IsEqual(blockHash) { + for currNode != nil && currNode.Height > bn.Height { + if currNode.Height == bn.Height && currNode.Hash.IsEqual(blockHash) { return currNode, true, nil } currNode = currNode.GetParent(bc.blockIndex) From f57a6993bef8fe9336a05b7c1a8492585e48b7ba Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:24:27 -0400 Subject: [PATCH 038/131] fix exit condition for height in get block from best chain by hash --- lib/blockchain.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index a0a9e954f..3aff181b6 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1901,9 +1901,12 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } currNode := &BlockNode{} *currNode = *blockTip - for currNode != nil && currNode.Height > bn.Height { - if currNode.Height == bn.Height && currNode.Hash.IsEqual(blockHash) { - return currNode, true, nil + for currNode != nil && currNode.Height >= bn.Height { + if currNode.Height == bn.Height { + if currNode.Hash.IsEqual(blockHash) { + return currNode, true, nil + } + return nil, false, nil } currNode = currNode.GetParent(bc.blockIndex) } From 2bae1a2132bb1922cb9dc087ebe094289e94f7f7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:25:59 -0400 Subject: [PATCH 039/131] allow early return from GetBlockFromBestChainByHash if we're using header chain and the header is validated if we're syncing --- lib/blockchain.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/blockchain.go b/lib/blockchain.go index 3aff181b6..ed5356688 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1885,6 +1885,10 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade // have to get a bunch of parents in order to be sure it is part of the best header chain. I guess we could // have a map, but kinda defeats the purpose of this refactor. } + // TODO: is this legit? It seems like it's fair game... + if bc.isSyncing() && useHeaderChain && bn.IsHeaderValidated() { + return bn, true, nil + } blockTip := bc.BlockTip() if useHeaderChain { blockTip = bc.HeaderTip() From ea243f37009fc3ab59153792132ff69b6d9309e8 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 8 Oct 2024 21:50:36 -0400 Subject: [PATCH 040/131] minor enhancement to isTipCurrent --- lib/blockchain.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ed5356688..b8e20dde0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1617,14 +1617,16 @@ func (bc *Blockchain) isTipCurrent(tip *BlockNode) bool { return tip.Height >= bc.MaxSyncBlockHeight } - minChainWorkBytes, _ := hex.DecodeString(bc.params.MinChainWorkHex) - // Not current if the cumulative work is below the threshold. - if bc.params.IsPoWBlockHeight(uint64(tip.Height)) && tip.CumWork.Cmp(BytesToBigint(minChainWorkBytes)) < 0 { - //glog.V(2).Infof("Blockchain.isTipCurrent: Tip not current because "+ - //"CumWork (%v) is less than minChainWorkBytes (%v)", - //tip.CumWork, BytesToBigint(minChainWorkBytes)) - return false + if bc.params.IsPoWBlockHeight(uint64(tip.Height)) { + minChainWorkBytes, _ := hex.DecodeString(bc.params.MinChainWorkHex) + + if tip.CumWork.Cmp(BytesToBigint(minChainWorkBytes)) < 0 { + //glog.V(2).Infof("Blockchain.isTipCurrent: Tip not current because "+ + //"CumWork (%v) is less than minChainWorkBytes (%v)", + //tip.CumWork, BytesToBigint(minChainWorkBytes)) + return false + } } // Not current if the tip has a timestamp older than the maximum From 3f73bb6349fbde366a625b4abb14f0be8e2d1bc5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:21:41 -0400 Subject: [PATCH 041/131] fix txindex issue w/ genesis block --- lib/txindex.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/txindex.go b/lib/txindex.go index d3794d935..fa173e955 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -394,7 +394,9 @@ func (txi *TXIndex) Update() error { // // Only set a BitcoinManager if we have one. This makes some tests pass. utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { + if blockToAttach.Height > 0 && + blockToAttach.Header.PrevBlockHash != nil && + !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { From e20040765db199d1667afec195dcd6254003c21b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:34:47 -0400 Subject: [PATCH 042/131] fix process block pow logic for genesis block --- lib/blockchain.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index b8e20dde0..35735e4b5 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2622,7 +2622,6 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. - // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set @@ -2664,10 +2663,13 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - // TODO: validate that current height - 1 > 0 - parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) - if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { - return false, true, nil + var parentNode *BlockNode + if blockHeader.Height > 0 { + var parentNodeExists bool + parentNode, parentNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) + if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { + return false, true, nil + } } if nodeToValidate.Status.IsFullyProcessed() { @@ -2697,11 +2699,14 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reject the block if any of the following apply to the parent: // - Its header is nil. // - Its header or its block validation failed. - if parentNode.Header == nil || - (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { + if blockHeight > 0 { + if parentNode == nil || + parentNode.Header == nil || + (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) - return false, false, RuleErrorPreviousBlockInvalid + bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) + return false, false, RuleErrorPreviousBlockInvalid + } } // At this point, we know that we are processing a block we haven't seen From 3ad68be4113b5a050806cc9b919676e53e8b8637 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 11:41:56 -0400 Subject: [PATCH 043/131] fix - get next block after tip hash in update process of txindex --- lib/blockchain.go | 23 +++++++++-------------- lib/txindex.go | 11 ++++++----- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 35735e4b5..b8e20dde0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2622,6 +2622,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures bc.timer.Start("Blockchain.ProcessBlock: BlockNode") // See if a node for the block exists in our node index. + // TODO: validate that current height - 1 > 0 nodeToValidate, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, blockHeader.Height) // If no node exists for this block at all, then process the header // first before we do anything. This should create a node and set @@ -2663,13 +2664,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // In this case go ahead and return early. If its parents are truly legitimate then we // should re-request it and its parents from a node and reprocess it // once it is no longer an orphan. - var parentNode *BlockNode - if blockHeader.Height > 0 { - var parentNodeExists bool - parentNode, parentNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) - if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { - return false, true, nil - } + // TODO: validate that current height - 1 > 0 + parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) + if !parentNodeExists || (parentNode.Status&StatusBlockProcessed) == 0 { + return false, true, nil } if nodeToValidate.Status.IsFullyProcessed() { @@ -2699,14 +2697,11 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // Reject the block if any of the following apply to the parent: // - Its header is nil. // - Its header or its block validation failed. - if blockHeight > 0 { - if parentNode == nil || - parentNode.Header == nil || - (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { + if parentNode.Header == nil || + (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) - return false, false, RuleErrorPreviousBlockInvalid - } + bc.MarkBlockInvalid(nodeToValidate, RuleErrorPreviousBlockInvalid) + return false, false, RuleErrorPreviousBlockInvalid } // At this point, we know that we are processing a block we haven't seen diff --git a/lib/txindex.go b/lib/txindex.go index fa173e955..d43e5f6a2 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -369,8 +369,11 @@ func (txi *TXIndex) Update() error { // For each of the blocks we're adding, process them on our txindex chain // and add their mappings to our txn index. Compute any metadata that might // be useful. - blockToAttach := &BlockNode{} - *blockToAttach = *txindexTipNode + // Get the next block after the current txindex tip hash. we know we've already processed the txindex tip hash. + blockToAttach, exists, err := txi.CoreChain.GetBlockFromBestChainByHeight(uint64(txindexTipNode.Height+1), false) + if !exists || err != nil { + return fmt.Errorf("Update: Problem getting block at height %d: %v", txindexTipNode.Height+1, err) + } for !blockToAttach.Hash.IsEqual(blockTipNode.Hash) { if txi.killed { glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while attaching blocks")) @@ -394,9 +397,7 @@ func (txi *TXIndex) Update() error { // // Only set a BitcoinManager if we have one. This makes some tests pass. utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - if blockToAttach.Height > 0 && - blockToAttach.Header.PrevBlockHash != nil && - !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { + if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { From 78034d40d3571adb751c750daa0b6ec97e432d3e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 12:03:18 -0400 Subject: [PATCH 044/131] reduce direct access to BlockNode.Parent and get rid of heal orphan pointers since we don't need it --- lib/block_view.go | 1 + lib/block_view_lockups_test.go | 1 - lib/blockchain.go | 21 ++++++++---- lib/pos_blockchain.go | 63 +++++++++++++++++----------------- lib/postgres.go | 2 ++ 5 files changed, 49 insertions(+), 39 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 2cd959419..ab0e87285 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -5112,6 +5112,7 @@ func (bav *UtxoView) GetSpendableDeSoBalanceNanosForPublicKey(pkBytes []byte, // but we do have the header. As a result, this condition always evaluates to false and thus // we only process the block reward for the previous block instead of all immature block rewards // as defined by the params. + // NOTE: we are not using .GetParent here as it changes the meaning of this code. if blockNode.Parent != nil { nextBlockHash = blockNode.Parent.Hash } else { diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 41e53e74e..a43beecf4 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2461,7 +2461,6 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, err) require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) - // TODO: make sure this works? // Update the tip testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.Parent diff --git a/lib/blockchain.go b/lib/blockchain.go index b8e20dde0..dbcb96dc4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -390,6 +390,8 @@ func (nn *BlockNode) String() string { var parentHash *BlockHash if nn.Parent != nil { parentHash = nn.Parent.Hash + } else { + parentHash = nn.Header.PrevBlockHash } tstamp := uint32(0) if nn.Header != nil { @@ -429,6 +431,9 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode } node := nn + // NOTE: using .Parent here is okay b/c it explicitly set it + // if we don't already have it when we fetch the parent from + // the block index. for ; node != nil && node.Height != height; node = node.Parent { // Keep iterating node until the condition no longer holds. if node.Parent == nil { @@ -1478,7 +1483,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( // StatusBlockProcessed so this loop is guaranteed to terminate successfully. headerNodeStart = bc.headerTip() for headerNodeStart != nil && (headerNodeStart.Status&StatusBlockProcessed) == 0 { - headerNodeStart = headerNodeStart.Parent + headerNodeStart = headerNodeStart.GetParent(bc.blockIndex) } if headerNodeStart == nil { @@ -3006,11 +3011,13 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If we're syncing there's no risk of concurrency issues. Otherwise, we // need to make a copy in order to be save. - if bc.isSyncing() { - bc.blockIndex.setTip(nodeToValidate) - } else { - bc.blockIndex.setTip(nodeToValidate) - } + // We no longer need to worry about whether we're syncing or not. Just + // set the tip. + //if bc.isSyncing() { + // bc.blockIndex.setTip(nodeToValidate) + //} else { + bc.blockIndex.setTip(nodeToValidate) + //} // This node is on the main chain so set this variable. isMainChain = true @@ -3165,7 +3172,7 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // If the parent node has been marked as invalid then mark this node as // invalid as well. - if (attachNode.Parent.Status & StatusBlockValidateFailed) != 0 { + if (attachNode.GetParent(bc.blockIndex).Status & StatusBlockValidateFailed) != 0 { bc.MarkBlockInvalid(attachNode, RuleErrorPreviousBlockInvalid) continue } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff793e940..4b610f017 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -76,11 +76,11 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH } // Don't worry about healing orphan children when we're syncing. - if !bc.isSyncing() { - // Now that we know we have a valid header, we check the block index for it any orphan children for it - // and heal the parent pointers for all of them. - bc.healPointersForOrphanChildren(blockNode) - } + //if !bc.isSyncing() { + // // Now that we know we have a valid header, we check the block index for it any orphan children for it + // // and heal the parent pointers for all of them. + // bc.healPointersForOrphanChildren(blockNode) + //} // Exit early if the header is an orphan. if isOrphan { @@ -116,31 +116,31 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // later on, we not only need to store the parent in the block index but also need to update the // pointer from the orphan block's BlockNode to the parent. We do that dynamically here as we // process headers. -func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { - // Fetch all potential children of this blockNode from the block index. - blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) - exists := len(blockNodesAtNextHeight) > 0 - if !exists { - // No children of this blockNode exist in the block index. Exit early. - return - } - - // Iterate through all block nodes at the next block height and update their parent pointers. - for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { - // Check if it's a child of the parent block node. - if !blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) { - continue - } - - // Check if it has its parent pointer set. If it does, then we exit early. - if blockNodeAtNextHeight.Parent != nil { - continue - } - - // If the parent block node is not set, then we set it to the parent block node. - blockNodeAtNextHeight.Parent = blockNode - } -} +//func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { +// // Fetch all potential children of this blockNode from the block index. +// blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) +// exists := len(blockNodesAtNextHeight) > 0 +// if !exists { +// // No children of this blockNode exist in the block index. Exit early. +// return +// } +// +// // Iterate through all block nodes at the next block height and update their parent pointers. +// for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { +// // Check if it's a child of the parent block node. +// if !blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) { +// continue +// } +// +// // Check if it has its parent pointer set. If it does, then we exit early. +// if blockNodeAtNextHeight.Parent != nil { +// continue +// } +// +// // If the parent block node is not set, then we set it to the parent block node. +// blockNodeAtNextHeight.Parent = blockNode +// } +//} func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( _headerBlockNode *BlockNode, _isOrphan bool, _err error, @@ -1391,14 +1391,15 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock, ha } } blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) - prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) if blockNode != nil { // If the block node already exists, we should set its parent if it doesn't have one already. if blockNode.Parent == nil { + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) blockNode.Parent = prevBlockNode } return blockNode, nil } + prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) newBlockNode := NewBlockNode(prevBlockNode, hash, uint32(block.Header.Height), nil, nil, block.Header, StatusNone) bc.addNewBlockNodeToBlockIndex(newBlockNode) return newBlockNode, nil diff --git a/lib/postgres.go b/lib/postgres.go index e919c8e78..c4eef6bb0 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -1298,6 +1298,8 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { // The genesis block has a nil parent if blockNode.Parent != nil { block.ParentHash = blockNode.Parent.Hash + } else if !blockNode.Header.PrevBlockHash.IsEqual(GenesisBlockHash) { + block.ParentHash = blockNode.Header.PrevBlockHash } _, err := tx.Model(block).WherePK().OnConflict("(hash) DO UPDATE").Insert() From 02e9e15decd5d3abf8f61d6a30f3c4d8f27b8f3c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 9 Oct 2024 16:45:49 -0400 Subject: [PATCH 045/131] fix postgres get best chain logic --- lib/postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/postgres.go b/lib/postgres.go index c4eef6bb0..1a2136c0f 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -1341,7 +1341,7 @@ func (postgres *Postgres) GetBlockIndex() (*lru.Cache[BlockHash, *BlockNode], er parentHash := blockNode.Header.PrevBlockHash if parentHash != nil { parent, exists := blockMap.Get(*parentHash) - if !exists { + if !exists && blockNode.Height > 0 { glog.Fatal("Parent block not found in block map") } blockNode.Parent = parent From 2a82deaa2a57040e1f6b46b41f0a784a0d8d68fb Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:17:57 -0500 Subject: [PATCH 046/131] Ln/downgrade badger (#1430) * downgrade badger, ristretto, and klauspost/compress dependencies * go mod tidy --- cmd/node.go | 2 +- go.mod | 10 ++-- go.sum | 61 +++++++++++++++++---- integration_testing/tools.go | 2 +- lib/block_view.go | 2 +- lib/block_view_atomic_txns_test.go | 2 +- lib/block_view_bitcoin_test.go | 2 +- lib/block_view_creator_coin_test.go | 2 +- lib/block_view_dao_coin_limit_order_test.go | 2 +- lib/block_view_dao_coin_test.go | 2 +- lib/block_view_derived_key_test.go | 2 +- lib/block_view_flush.go | 2 +- lib/block_view_follow_test.go | 2 +- lib/block_view_like_test.go | 2 +- lib/block_view_lockups.go | 2 +- lib/block_view_lockups_test.go | 2 +- lib/block_view_message_test.go | 2 +- lib/block_view_nft_test.go | 2 +- lib/block_view_post.go | 2 +- lib/block_view_post_test.go | 2 +- lib/block_view_profile_test.go | 2 +- lib/block_view_stake.go | 2 +- lib/block_view_test.go | 2 +- lib/block_view_validator.go | 2 +- lib/blockchain.go | 2 +- lib/blockchain_test.go | 2 +- lib/db_adapter.go | 2 +- lib/db_utils.go | 2 +- lib/db_utils_test.go | 2 +- lib/legacy_mempool.go | 2 +- lib/notifier.go | 2 +- lib/pos_blockchain.go | 2 +- lib/pos_epoch.go | 2 +- lib/pos_mempool.go | 2 +- lib/pos_mempool_persister.go | 2 +- lib/pos_mempool_persister_test.go | 2 +- lib/pos_mempool_test.go | 2 +- lib/pos_random_seed.go | 2 +- lib/pos_snapshot_cache.go | 2 +- lib/pos_snapshot_entries.go | 2 +- lib/postgres.go | 2 +- lib/server.go | 3 +- lib/snapshot.go | 2 +- lib/txindex.go | 2 +- 44 files changed, 100 insertions(+), 56 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index 52bf98b70..b1501978b 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/deso-protocol/core/lib" "github.com/deso-protocol/core/migrate" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/golang/glog" migrations "github.com/robinjoseph08/go-pg-migrations/v3" diff --git a/go.mod b/go.mod index f4f3cb798..8ff891d0f 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/deso-protocol/go-deadlock v1.0.1 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 - github.com/dgraph-io/badger/v4 v4.3.1 + github.com/dgraph-io/badger/v3 v3.2103.5 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.14.11 github.com/fatih/color v1.17.0 @@ -36,7 +36,7 @@ require ( github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.1.0 github.com/unrolled/secure v1.16.0 @@ -69,11 +69,12 @@ require ( github.com/andygrunwald/go-jira v1.16.0 // indirect github.com/btcsuite/btclog v0.0.0-20241017175713-3428138b75c7 // indirect github.com/bwesterb/go-ristretto v1.2.3 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/dgraph-io/ristretto v1.0.0 // indirect + github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.0 // indirect @@ -85,6 +86,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20241009165004-a3522334989c // indirect @@ -98,7 +100,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/kyokomi/emoji/v2 v2.2.13 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index 29ce83362..c0af42e8b 100644 --- a/go.sum +++ b/go.sum @@ -33,9 +33,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -72,6 +75,8 @@ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -79,8 +84,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -104,10 +113,11 @@ github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+Ls github.com/deso-protocol/go-merkle-tree v1.0.0/go.mod h1:V/vbg/maaNv6G7zf9VVs645nLFx/jsO2L/awFB/S/ZU= github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cAHbdG6o= github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= -github.com/dgraph-io/badger/v4 v4.3.1 h1:7r5wKqmoRpGgSxqa0S/nGdpOpvvzuREGPLSua73C8tw= -github.com/dgraph-io/badger/v4 v4.3.1/go.mod h1:oObz97DImXpd6O/Dt8BqdKLLTDmEmarAimo72VV5whQ= -github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= -github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -154,12 +164,14 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -172,9 +184,11 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -184,6 +198,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -207,6 +222,7 @@ github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9 github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -218,6 +234,7 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -230,8 +247,9 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -243,6 +261,7 @@ github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -261,6 +280,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -287,6 +307,7 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= @@ -307,6 +328,7 @@ github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0 h1:EjexnDlSIZoK/gMfQmKIqB7tY github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0/go.mod h1:9yEG60N97UVFGD/UKQUXoGVZh/t8KXx3JxEpxhKFlKY= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -326,18 +348,25 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -345,6 +374,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -369,6 +399,7 @@ github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df h1:Y2l28Jr3 github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df/go.mod h1:pnyouUty/nBr/zm3GYwTIt+qFTLWbdjeLjZmJdzJOu8= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/unrolled/secure v1.16.0 h1:XgdAsS/Zl50ZfZPRJK6WpicFttfrsFYFd0+ONDBJubU= github.com/unrolled/secure v1.16.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= @@ -383,12 +414,14 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -400,6 +433,7 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -445,6 +479,7 @@ golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -455,9 +490,11 @@ golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -477,6 +514,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -527,15 +565,17 @@ gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -553,6 +593,7 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.69.0 h1:zSY6DDsFRMQDNQYKWCv/AEwJXoPpDf1FfMyw7 gopkg.in/DataDog/dd-trace-go.v1 v1.69.0/go.mod h1:U9AOeBHNAL95JXcd/SPf4a7O5GNeF/yD13sJtli/yaU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/integration_testing/tools.go b/integration_testing/tools.go index 26e322335..94617bd18 100644 --- a/integration_testing/tools.go +++ b/integration_testing/tools.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/lib/block_view.go b/lib/block_view.go index c9d3f590a..3a8eacb12 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_atomic_txns_test.go b/lib/block_view_atomic_txns_test.go index ead9a7049..53d71c7e1 100644 --- a/lib/block_view_atomic_txns_test.go +++ b/lib/block_view_atomic_txns_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_bitcoin_test.go b/lib/block_view_bitcoin_test.go index 99352cf84..c12b8af0a 100644 --- a/lib/block_view_bitcoin_test.go +++ b/lib/block_view_bitcoin_test.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/btcsuite/btcd/wire" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_creator_coin_test.go b/lib/block_view_creator_coin_test.go index 7e4a3a05b..e25af8ac9 100644 --- a/lib/block_view_creator_coin_test.go +++ b/lib/block_view_creator_coin_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_limit_order_test.go b/lib/block_view_dao_coin_limit_order_test.go index 98034b5e0..07d214725 100644 --- a/lib/block_view_dao_coin_limit_order_test.go +++ b/lib/block_view_dao_coin_limit_order_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_test.go b/lib/block_view_dao_coin_test.go index aa64707bd..b006c3d7d 100644 --- a/lib/block_view_dao_coin_test.go +++ b/lib/block_view_dao_coin_test.go @@ -6,7 +6,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_derived_key_test.go b/lib/block_view_derived_key_test.go index bc59dd3f9..e9d079786 100644 --- a/lib/block_view_derived_key_test.go +++ b/lib/block_view_derived_key_test.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_flush.go b/lib/block_view_flush.go index 52a210636..e65341b4f 100644 --- a/lib/block_view_flush.go +++ b/lib/block_view_flush.go @@ -5,7 +5,7 @@ import ( "reflect" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_follow_test.go b/lib/block_view_follow_test.go index ef3823f63..fedb91a03 100644 --- a/lib/block_view_follow_test.go +++ b/lib/block_view_follow_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_like_test.go b/lib/block_view_like_test.go index a757c7f70..01b715475 100644 --- a/lib/block_view_like_test.go +++ b/lib/block_view_like_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_lockups.go b/lib/block_view_lockups.go index c2c3e34ea..3a4a8d532 100644 --- a/lib/block_view_lockups.go +++ b/lib/block_view_lockups.go @@ -8,7 +8,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index af461cd5f..48f435363 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -7,7 +7,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_message_test.go b/lib/block_view_message_test.go index a47c3d85c..dbe204f99 100644 --- a/lib/block_view_message_test.go +++ b/lib/block_view_message_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_nft_test.go b/lib/block_view_nft_test.go index 2f4cfd64c..ba00d8ece 100644 --- a/lib/block_view_nft_test.go +++ b/lib/block_view_nft_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_post.go b/lib/block_view_post.go index e80d20daf..a6b9e6d30 100644 --- a/lib/block_view_post.go +++ b/lib/block_view_post.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/golang/glog" "github.com/pkg/errors" diff --git a/lib/block_view_post_test.go b/lib/block_view_post_test.go index 264df7d69..e84eaa1c9 100644 --- a/lib/block_view_post_test.go +++ b/lib/block_view_post_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_profile_test.go b/lib/block_view_profile_test.go index c59e37cd5..01101d005 100644 --- a/lib/block_view_profile_test.go +++ b/lib/block_view_profile_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_stake.go b/lib/block_view_stake.go index 10ad6ab7f..b83d5d7ac 100644 --- a/lib/block_view_stake.go +++ b/lib/block_view_stake.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_test.go b/lib/block_view_test.go index cb7a30fb8..8aecfb24f 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -13,7 +13,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" diff --git a/lib/block_view_validator.go b/lib/block_view_validator.go index 8cc00a14b..3b2560654 100644 --- a/lib/block_view_validator.go +++ b/lib/block_view_validator.go @@ -15,7 +15,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain.go b/lib/blockchain.go index 640b1cd71..90b20c2f2 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -30,7 +30,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 37930a11c..6ce522f7c 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -19,7 +19,7 @@ import ( chainlib "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/db_adapter.go b/lib/db_adapter.go index 8cff5d757..4eea4d0da 100644 --- a/lib/db_adapter.go +++ b/lib/db_adapter.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils.go b/lib/db_utils.go index 126cc703b..63f78fcbf 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -24,7 +24,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils_test.go b/lib/db_utils_test.go index 82a452df6..6a6a0f208 100644 --- a/lib/db_utils_test.go +++ b/lib/db_utils_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index 247092d1e..7e9dceddf 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -18,7 +18,7 @@ import ( "time" "github.com/btcsuite/btcd/btcutil" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/btcsuite/btcd/btcec/v2" diff --git a/lib/notifier.go b/lib/notifier.go index b6c88c37d..a42ffb3ab 100644 --- a/lib/notifier.go +++ b/lib/notifier.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/go-pg/pg/v10" diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 588df734a..4b610f017 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_epoch.go b/lib/pos_epoch.go index ef4e3c78d..fed62d90e 100644 --- a/lib/pos_epoch.go +++ b/lib/pos_epoch.go @@ -3,7 +3,7 @@ package lib import ( "bytes" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 6cfd630d2..5592553f0 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index ceeface31..05a1ef617 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister_test.go b/lib/pos_mempool_persister_test.go index 73887a01f..6cd1ef25d 100644 --- a/lib/pos_mempool_persister_test.go +++ b/lib/pos_mempool_persister_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_mempool_test.go b/lib/pos_mempool_test.go index 8dced4464..d04883919 100644 --- a/lib/pos_mempool_test.go +++ b/lib/pos_mempool_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_random_seed.go b/lib/pos_random_seed.go index 9c739c41d..6777de6e6 100644 --- a/lib/pos_random_seed.go +++ b/lib/pos_random_seed.go @@ -7,7 +7,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_cache.go b/lib/pos_snapshot_cache.go index f320e99b0..ec495633c 100644 --- a/lib/pos_snapshot_cache.go +++ b/lib/pos_snapshot_cache.go @@ -3,7 +3,7 @@ package lib import ( "sync" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_entries.go b/lib/pos_snapshot_entries.go index 7788d170c..b727c384a 100644 --- a/lib/pos_snapshot_entries.go +++ b/lib/pos_snapshot_entries.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/postgres.go b/lib/postgres.go index 6518d7a9b..9c9ba432f 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" "github.com/golang/glog" diff --git a/lib/server.go b/lib/server.go index 6165592f7..28f1947ff 100644 --- a/lib/server.go +++ b/lib/server.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "github.com/dgraph-io/badger/v4" "net" "path/filepath" "reflect" @@ -20,7 +21,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" diff --git a/lib/snapshot.go b/lib/snapshot.go index d10adc0f4..2d8ffb895 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -15,7 +15,7 @@ import ( "github.com/cloudflare/circl/group" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" diff --git a/lib/txindex.go b/lib/txindex.go index 2868245b6..29bc0336c 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" chainlib "github.com/btcsuite/btcd/blockchain" "github.com/golang/glog" From 35f7a6af30190c63659dc29dec94910b2d2a9dd6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:51:19 -0500 Subject: [PATCH 047/131] remove LocateBestBlockChainHeaders and replace with DH suggestion. flush headers in a batch at the end of each bundle, merge in latest main --- lib/blockchain.go | 72 ++++++++++++++++++--------------- lib/blockchain_test.go | 10 ++--- lib/pos_blockchain.go | 24 +++++------ lib/server.go | 91 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 143 insertions(+), 54 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 0937f7a61..653764400 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -624,6 +624,8 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { blocksAtHeight = []*BlockNode{} } else { // Make sure we don't add the same block node twice. + // TODO: we *could* make this more efficient by using a map, + // but generally we won't have many blocks at the same height. for ii, blockAtHeight := range blocksAtHeight { if blockAtHeight.Hash.IsEqual(blockNode.Hash) { blocksAtHeight[ii] = blockNode @@ -1191,6 +1193,7 @@ func fastLog2Floor(n uint32) uint8 { return rv } +// TODO: deprecate // locateInventory returns the node of the block after the first known block in // the locator along with the number of subsequent nodes needed to either reach // the provided stop hash or the provided max number of entries. @@ -1279,6 +1282,7 @@ func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, return startNode, total } +// TODO: deprecate // locateHeaders returns the headers of the blocks after the first known block // in the locator until the provided stop hash is reached, or up to the provided // max number of block headers. @@ -1319,6 +1323,7 @@ func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, m return headers } +// TODO: deprecate // LocateBestBlockChainHeaders returns the headers of the blocks after the first known block // in the locator until the provided stop hash is reached, or up to a max of // wire.MaxBlockHeadersPerMsg headers. Note that it returns the best headers @@ -1589,16 +1594,6 @@ func (bc *Blockchain) GetBlock(blockHash *BlockHash) *MsgDeSoBlock { return blk } -func (bc *Blockchain) GetBlockAtHeight(height uint32, isHeaderChain bool) (*MsgDeSoBlock, error) { - bn, bnExists, err := bc.GetBlockFromBestChainByHeight(uint64(height), isHeaderChain) - if !bnExists || err != nil { - glog.Errorf("Blockchain.GetBlockAtHeight: Problem getting block by height: %v", err) - return nil, err - } - - return bc.GetBlock(bn.Hash), nil -} - // GetBlockNodeWithHash looks for a block node in the bestChain list that matches the hash. func (bc *Blockchain) GetBlockNodeWithHash(hash *BlockHash) *BlockNode { if hash == nil { @@ -1936,6 +1931,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade } func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChain bool) (*BlockNode, bool, error) { + // TODO: figure out an optimization for header chain handling uncommitted state. if !useHeaderChain { committedTip, exists := bc.GetCommittedTip() if !exists { @@ -1944,9 +1940,6 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai if height >= uint64(committedTip.Height) { // For this, we can just loop back from the tip block. currentNode := bc.blockIndex.GetTip() - if useHeaderChain { - currentNode = bc.blockIndex.GetHeaderTip() - } for currentNode != nil { if uint64(currentNode.Height) == height { return currentNode, true, nil @@ -1964,10 +1957,15 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai return nil, false, nil } for _, blockNode := range blockNodes { - if !useHeaderChain && blockNode.IsCommitted() { + // If block node is committed, then we know it is + // in the best chain, whether we're looking at the header chain or not. + if blockNode.IsCommitted() { return blockNode, true, nil } - // TODO: this is crude and incorrect. + // TODO: this is crude and incorrect. We can have multiple headers + // at a specific height. It's possible that none of the blocks at + // this height are committed yet, but one of them is in the best chain. + // How can we figure it out? if useHeaderChain && blockNode.IsHeaderValidated() { return blockNode, true, nil } @@ -2340,10 +2338,13 @@ func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru.Cache } // Caller must acquire the ChainLock for writing prior to calling this. -func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *BlockHash) (_isMainChain bool, _isOrphan bool, _err error) { +func (bc *Blockchain) processHeaderPoW( + blockHeader *MsgDeSoHeader, + headerHash *BlockHash, +) (_blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error) { // Only accept the header if its height is below the PoS cutover height. if !bc.params.IsPoWBlockHeight(blockHeader.Height) { - return false, false, HeaderErrorBlockHeightAfterProofOfStakeCutover + return nil, false, false, HeaderErrorBlockHeightAfterProofOfStakeCutover } // Only accept headers if the best chain is still in PoW. Once the best chain reaches the final @@ -2351,16 +2352,16 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // headers past this point because they will un-commit blocks that are already committed to the PoS // chain. if bc.BlockTip().Header.Height >= bc.params.GetFinalPoWBlockHeight() { - return false, false, HeaderErrorBestChainIsAtProofOfStakeCutover + return nil, false, false, HeaderErrorBestChainIsAtProofOfStakeCutover } // Start by checking if the header already exists in our node // index. If it does, then return an error. We should generally // expect that processHeaderPoW will only be called on headers we // haven't seen before. - _, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) + blockNode, nodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, blockHeader.Height) if nodeExists { - return false, false, HeaderErrorDuplicateHeader + return blockNode, false, false, HeaderErrorDuplicateHeader } // If we're here then it means we're processing a header we haven't @@ -2373,7 +2374,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B "MaxTstampOffsetSeconds %d. blockHeader.TstampSecs=%d; adjustedTime=%d", tstampDiff, bc.params.MaxTstampOffsetSeconds, blockHeader.GetTstampSecs(), bc.timeSource.AdjustedTime().Unix()) - return false, false, HeaderErrorBlockTooFarInTheFuture + return nil, false, false, HeaderErrorBlockTooFarInTheFuture } // Try to find this header's parent in our block index. @@ -2381,13 +2382,13 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // can return early because we don't process unconnectedTxns. // TODO: Should we just return an error if the header is an orphan? if blockHeader.PrevBlockHash == nil { - return false, false, HeaderErrorNilPrevHash + return nil, false, false, HeaderErrorNilPrevHash } parentNode, parentNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHeader.PrevBlockHash, blockHeader.Height-1) if !parentNodeExists { // This block is an orphan if its parent doesn't exist and we don't // process unconnectedTxns. - return false, true, nil + return nil, false, true, nil } // If the parent node is invalid then this header is invalid as well. Note that @@ -2395,7 +2396,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B // ValidateFailed. parentHeader := parentNode.Header if parentHeader == nil || (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0 { - return false, false, errors.Wrapf( + return nil, false, false, errors.Wrapf( HeaderErrorInvalidParent, "Parent header: %v, Status check: %v, Parent node status: %v, Parent node header: %v", parentHeader, (parentNode.Status&(StatusHeaderValidateFailed|StatusBlockValidateFailed)) != 0, parentNode.Status, @@ -2407,7 +2408,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B if blockHeader.Height != prevHeight+1 { glog.Errorf("processHeaderPoW: Height of block (=%d) is not equal to one greater "+ "than the parent height (=%d)", blockHeader.Height, prevHeight) - return false, false, HeaderErrorHeightInvalid + return nil, false, false, HeaderErrorHeightInvalid } // Make sure the block timestamp is greater than the previous block's timestamp. @@ -2438,7 +2439,7 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B "before timestamp of previous block %v", time.Unix(int64(blockHeader.GetTstampSecs()), 0), time.Unix(int64(parentHeader.GetTstampSecs()), 0)) - return false, false, HeaderErrorTimestampTooEarly + return nil, false, false, HeaderErrorTimestampTooEarly } // Check that the proof of work beats the difficulty as calculated from @@ -2448,14 +2449,14 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B diffTarget, err := bc.CalcNextDifficultyTarget( parentNode, blockHeader.Version) if err != nil { - return false, false, errors.Wrapf(err, + return nil, false, false, errors.Wrapf(err, "ProcessBlock: Problem computing difficulty "+ "target from parent block %s", hex.EncodeToString(parentNode.Hash[:])) } diffTargetBigint := HashToBigint(diffTarget) blockHashBigint := HashToBigint(headerHash) if diffTargetBigint.Cmp(blockHashBigint) < 0 { - return false, false, + return nil, false, false, errors.Wrapf(HeaderErrorBlockDifficultyAboveTarget, "Target: %v, Actual: %v", diffTarget, headerHash) } @@ -2515,17 +2516,21 @@ func (bc *Blockchain) processHeaderPoW(blockHeader *MsgDeSoHeader, headerHash *B bc.blockIndex.setHeaderTip(newNode) } - return isMainChain, false, nil + return blockNode, isMainChain, false, nil } // ProcessHeader is a wrapper around processHeaderPoW and processHeaderPoS, which do the leg-work. -func (bc *Blockchain) ProcessHeader(blockHeader *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) (_isMainChain bool, _isOrphan bool, _err error) { +func (bc *Blockchain) ProcessHeader( + blockHeader *MsgDeSoHeader, + headerHash *BlockHash, + verifySignatures bool, +) (_blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error) { bc.ChainLock.Lock() defer bc.ChainLock.Unlock() if blockHeader == nil { // If the header is nil then we return an error. Nothing we can do here. - return false, false, fmt.Errorf("ProcessHeader: Header is nil") + return nil, false, false, fmt.Errorf("ProcessHeader: Header is nil") } // If the header's height is after the PoS cut-over fork height, then we use the PoS header processing logic. @@ -2644,7 +2649,10 @@ func (bc *Blockchain) processBlockPoW(desoBlock *MsgDeSoBlock, verifySignatures // first before we do anything. This should create a node and set // the header validation status for it. if !nodeExists { - _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) + // Note: it's okay that we don't write the block node for the header + // to the db here as it happens below when we call + // PutHeightHashToNodeInfo + _, _, isOrphan, err := bc.processHeaderPoW(blockHeader, blockHash) if err != nil { // If an error occurred processing the header, then the header // should be marked as invalid, which should be sufficient. diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 6ce522f7c..be4ffed0e 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -707,7 +707,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockA1.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockA1.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockA1.Header, headerHash, false) require.NoError(err) require.True(isMainChain) require.False(isOrphan) @@ -723,7 +723,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockA2.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockA2.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockA2.Header, headerHash, false) require.NoError(err) require.True(isMainChain) require.False(isOrphan) @@ -739,7 +739,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB1.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB1.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB1.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.False(isMainChain) @@ -756,7 +756,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB2.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB2.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB2.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.False(isMainChain) @@ -773,7 +773,7 @@ func TestProcessHeaderskReorgBlocks(t *testing.T) { require.Equal(uint64(1), GetUtxoNumEntries(db, chain.snapshot)) headerHash, err := blockB3.Header.Hash() require.NoError(err) - isMainChain, isOrphan, err := chain.ProcessHeader(blockB3.Header, headerHash, false) + _, isMainChain, isOrphan, err := chain.ProcessHeader(blockB3.Header, headerHash, false) require.NoError(err) // Should not be main chain yet require.True(isMainChain) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index dcfc6e3bf..8e412e50b 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -39,10 +39,10 @@ import ( // 5. Exit early if the's view is less than the current header chain's tip. // 6. Reorg the best header chain if the header's view is higher than the current tip. func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( - _isMainChain bool, _isOrphan bool, _err error, + _blockNode *BlockNode, _isMainChain bool, _isOrphan bool, _err error, ) { if !bc.params.IsPoSBlockHeight(header.Height) { - return false, false, errors.Errorf( + return nil, false, false, errors.Errorf( "processHeaderPoS: Header height %d is less than the ProofOfStake2ConsensusCutoverBlockHeight %d", header.Height, bc.params.GetFirstPoSBlockHeight(), ) @@ -53,26 +53,26 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - // TODO: make sure this is ok or do we need to explicitly check the block index's cache? - _, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") + return nil, false, false, + errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") } if isInBestHeaderChain { - return true, false, nil + return blockNode, true, false, nil } // If the incoming header is part of a reorg that uncommits the committed tip from the best chain, // then we exit early. Such headers are invalid and should not be synced. committedBlockchainTip, _ := bc.GetCommittedTip() if committedBlockchainTip != nil && committedBlockchainTip.Header.Height >= header.Height { - return false, false, errors.New("processHeaderPoS: Header conflicts with committed tip") + return nil, false, false, errors.New("processHeaderPoS: Header conflicts with committed tip") } // Validate the header and index it in the block index. blockNode, isOrphan, err := bc.validateAndIndexHeaderPoS(header, headerHash, verifySignatures) if err != nil { - return false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") + return blockNode, false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") } // Don't worry about healing orphan children when we're syncing. @@ -84,14 +84,14 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Exit early if the header is an orphan. if isOrphan { - return false, true, nil + return blockNode, false, true, nil } // Exit early if the header's view is less than the current header chain's tip. The header is not // the new tip for the best header chain. currentTip := bc.headerTip() if header.ProposedInView <= currentTip.Header.ProposedInView { - return false, false, nil + return blockNode, false, false, nil } bc.blockIndex.setHeaderTip(blockNode) @@ -107,7 +107,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH //) // Success. The header is at the tip of the best header chain. - return true, false, nil + return blockNode, true, false, nil } // healPointersForOrphanChildren fixes an inconsistency in the block index that may have @@ -364,7 +364,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // header and applying it to the header chain will result in the two chains being out of // sync. The header chain is less critical and mutations to it are reversible. So we attempt // to mutate it first before attempting to mutate the block chain. - if _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { + if _, _, _, err = bc.processHeaderPoS(block.Header, blockHash, verifySignatures); err != nil { return false, false, nil, errors.Wrap(err, "processBlockPoS: Problem processing header") } diff --git a/lib/server.go b/lib/server.go index bcf9fa521..c5f25c75c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -364,19 +364,20 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { blockIndexMigrationFileName := filepath.Join(db.Opts().Dir, BlockIndexMigrationFileName) - glog.V(0).Info("FileName: ", blockIndexMigrationFileName) + glog.V(2).Info("FileName: ", blockIndexMigrationFileName) hasRunMigration, err := ReadBoolFromFile(blockIndexMigrationFileName) if err == nil && hasRunMigration { - glog.V(0).Info("Block index migration has already been run") + glog.V(2).Info("Block index migration has already been run") return nil } - glog.V(0).Info("Running block index migration") + glog.V(2).Info("Running block index migration") if err = RunBlockIndexMigration(db, nil, nil, params); err != nil { return errors.Wrapf(err, "Problem running block index migration") } if err = SaveBoolToFile(blockIndexMigrationFileName, true); err != nil { return errors.Wrapf(err, "Problem saving block index migration file") } + glog.V(2).Info("Block index migration complete") return nil } @@ -771,7 +772,21 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { if pp.NegotiatedProtocolVersion >= ProtocolVersion2 { maxHeadersPerMsg = MaxHeadersPerMsgPos } - headers := srv.blockchain.LocateBestBlockChainHeaders(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + // FIXME: We can eliminate the call to LocateBestBlockChainHeaders and do a much + // simpler "shortcut" version that doesn't require complicated tree-traversal bs. + // The shortcut would be to just return all headers starting from msg.BlockLocator[0] + // up to msg.StopHash or maxHeadersPerMsg, whichever comes first. This would allow + // other nodes to sync from us and *keep* in sync with us, while allowing us to delete + // ALL of the complicated logic around locators and the best header chain. This all works + // because msg.BlockLocator[0] is the requesting-node's tip hash. The rest of the + // hashes, and all of the locator bs, are only needed to resolve forks, which can't + // happen with PoS anymore. + //headers := srv.blockchain.LocateBestBlockChainHeaders(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + + headers, err := srv.GetHeadersForLocatorAndStopHash(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + if err != nil { + glog.Errorf("Server._handleGetHeadersMessage: Error getting headers: %v", err) + } // Send found headers to the requesting peer. blockTip := srv.blockchain.blockTip() @@ -785,6 +800,51 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { headers, blockTip.Hash, blockTip.Height, pp) } +func (srv *Server) GetHeadersForLocatorAndStopHash( + locator []*BlockHash, + stopHash *BlockHash, + maxHeadersPerMsg uint32, +) ([]*MsgDeSoHeader, error) { + var headers []*MsgDeSoHeader + + stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHash(stopHash, true) + // Special case when there is no block locator provided but only a stop hash. + if len(locator) == 0 { + if stopNodeError != nil || !stopNodeExists || stopNode == nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Stop hash provided but no stop node found") + } + return []*MsgDeSoHeader{stopNode.Header}, nil + } + startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHash(locator[0], true) + if startNodeError != nil || !startNodeExists || startNode == nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") + } + nextNodeHeight := startNode.Header.Height + 1 + nextNode, nextNodeExists, nextNodeError := srv.blockchain.GetBlockFromBestChainByHeight(nextNodeHeight, true) + if nextNodeError != nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Error getting start node by height: %v", startNodeError) + } + if !nextNodeExists || nextNode == nil { + return nil, nil + } + for ii := uint32(0); ii < maxHeadersPerMsg; ii++ { + headers = append(headers, nextNode.Header) + if stopNode != nil && nextNode.Hash.IsEqual(stopNode.Hash) { + break + } + nextNode, nextNodeExists, nextNodeError = srv.blockchain.GetBlockFromBestChainByHeight( + nextNode.Header.Height+1, true) + if nextNodeError != nil { + glog.Errorf("Server._handleGetHeadersMessage: Error getting next node by height: %v", nextNodeError) + break + } + if !nextNodeExists || nextNode == nil { + break + } + } + return headers, nil +} + // GetSnapshot is used for sending MsgDeSoGetSnapshot messages to peers. We will // check if the passed peer has been assigned to an in-progress prefix and if so, // we will request a snapshot data chunk from them. Otherwise, we will assign a @@ -1118,6 +1178,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // Start by processing all the headers given to us. They should start // right after the tip of our header chain ideally. While going through them // tally up the number that we actually process. + var blockNodeBatch []*BlockNode for ii, headerReceived := range msg.Headers { // If we've set a maximum height for node sync and we've reached it, // then we will not process any more headers. @@ -1169,7 +1230,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // Process the header, as we haven't seen it before, set verifySignatures to false // if we're in the process of syncing. - _, isOrphan, err := srv.blockchain.ProcessHeader(headerReceived, headerHash, verifySignatures) + blockNode, _, isOrphan, err := srv.blockchain.ProcessHeader(headerReceived, headerHash, verifySignatures) numLogHeaders := 2000 if ii%numLogHeaders == 0 { @@ -1189,9 +1250,29 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { pp, srv.blockchain.chainState(), err, isOrphan) pp.Disconnect("Error processing header") + // Just to be safe, we flush all the headers we just got even tho we have a header. + currTime := time.Now() + if err := PutHeightHashToNodeInfoBatch( + srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) + return + } + glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) return } + + // Append the block node to the block node batch. + if blockNode != nil { + blockNodeBatch = append(blockNodeBatch, blockNode) + } + } + currTime := time.Now() + if err := PutHeightHashToNodeInfoBatch( + srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { + glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) + return } + glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) // After processing all the headers this will check to see if we are fully current // and send a request to our Peer to start a Mempool sync if so. From 69e18251aa255e1cf95a5e7c996fe2c470f7de60 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:55:24 -0500 Subject: [PATCH 048/131] make GetUtxoViewAndUtxoOpsAtBlockHash public --- lib/pos_blockchain.go | 12 ++++++------ lib/pos_blockchain_test.go | 2 +- lib/pos_consensus.go | 4 ++-- lib/state_change_syncer.go | 2 +- lib/txindex.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 8e412e50b..7c8e79f4f 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -320,7 +320,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v // We expect the utxoView for the parent block to be valid because we check that all ancestor blocks have // been validated. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) + parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -824,7 +824,7 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( return nil, errors.Wrapf(err, "validatePreviouslyIndexedBlockPoS: Problem fetching block from DB") } // Build utxoView for the block's parent. - parentUtxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) + parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { // This should never happen. If the parent is validated and extends from the tip, then we should // be able to build a UtxoView for it. This failure can only happen due to transient or badger issues. @@ -1755,7 +1755,7 @@ func (bc *Blockchain) commitBlockPoS(blockHash *BlockHash, blockHeight uint64, v return errors.Errorf("commitBlockPoS: Block %v is already committed", blockHash.String()) } // Connect a view up to block we are committing. - utxoViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) + utxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*blockHash, uint64(blockNode.Height)) if err != nil { return errors.Wrapf(err, "commitBlockPoS: Problem initializing UtxoView: ") } @@ -1931,7 +1931,7 @@ func (viewAndUtxoOps *BlockViewAndUtxoOps) Copy() *BlockViewAndUtxoOps { func (bc *Blockchain) GetUncommittedTipView() (*UtxoView, error) { // Connect the uncommitted blocks to the tip so that we can validate subsequent blocks blockTip := bc.BlockTip() - blockViewAndUtxoOps, err := bc.getUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) + blockViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*blockTip.Hash, uint64(blockTip.Height)) if err != nil { return nil, errors.Wrapf(err, "GetUncommittedTipView: Problem getting UtxoView at block hash") } @@ -1949,7 +1949,7 @@ func (bc *Blockchain) getCachedBlockViewAndUtxoOps(blockHash BlockHash) (*BlockV // all uncommitted ancestors of this block. Then it checks the block view cache to see if we have already // computed this view. If not, connecting the uncommitted ancestor blocks and saving to the cache. The // returned UtxoOps and FullBlock should NOT be modified. -func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( +func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, blockHeight uint64) ( *BlockViewAndUtxoOps, error) { // Always fetch the lineage from the committed tip to the block provided first to // ensure that a valid UtxoView is returned. @@ -1962,7 +1962,7 @@ func (bc *Blockchain) getUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc highestCommittedBlock, exists := bc.GetCommittedTip() glog.Infof("Highest committed block: %v", highestCommittedBlock) if !exists || highestCommittedBlock == nil { - return nil, errors.Errorf("getUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") + return nil, errors.Errorf("GetUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") } // If the provided block is committed, we need to make sure it's the committed tip. // Otherwise, we return an error. diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 5c32df53a..75ef9beea 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -2486,7 +2486,7 @@ func _generateRealBlockWithFailingTxn(testMeta *TestMeta, blockHeight uint64, vi prevBlock, exists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) require.True(testMeta.t, exists) // Always update the testMeta latestBlockView - latestBlockViewAndUtxoOps, err := testMeta.chain.getUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) + latestBlockViewAndUtxoOps, err := testMeta.chain.GetUtxoViewAndUtxoOpsAtBlockHash(*prevBlockHash, blockHeight-1) require.NoError(testMeta.t, err) latestBlockView := latestBlockViewAndUtxoOps.UtxoView latestBlockNode, latestBlockNodeExists := testMeta.chain.blockIndex.GetBlockNodeByHashAndHeight(prevBlockHash, blockHeight-1) diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index a33983240..ab7a73190 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -701,7 +701,7 @@ func (fc *FastHotStuffConsensus) tryProcessBlockAsNewTip(block *MsgDeSoBlock) ([ return nil, errors.Errorf("Error hashing tip block: %v", err) } - utxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) + utxoViewAndUtxoOps, err := fc.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*tipBlockHash, tipBlock.Height) if err != nil { return nil, errors.Errorf("Error fetching UtxoView for tip block: %v", err) } @@ -750,7 +750,7 @@ func (fc *FastHotStuffConsensus) produceUnsignedBlockForBlockProposalEvent( } // Build a UtxoView at the parent block - parentUtxoViewAndUtxoOps, err := fc.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) + parentUtxoViewAndUtxoOps, err := fc.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*parentBlockHash, uint64(parentBlock.Height)) if err != nil { // This should never happen as long as the parent block is a descendant of the committed tip. return nil, errors.Errorf("Error fetching UtxoView for parent block: %v", parentBlockHash) diff --git a/lib/state_change_syncer.go b/lib/state_change_syncer.go index 358d86294..f81bcf421 100644 --- a/lib/state_change_syncer.go +++ b/lib/state_change_syncer.go @@ -847,7 +847,7 @@ func (stateChangeSyncer *StateChangeSyncer) SyncMempoolToStateSyncer(server *Ser // TODO: Have Z look at if we need to do some caching in the uncommitted blocks logic. // First connect the uncommitted blocks to the mempool view. for _, uncommittedBlock := range uncommittedBlocks { - utxoViewAndOpsAtBlockHash, err := server.blockchain.getUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) + utxoViewAndOpsAtBlockHash, err := server.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*uncommittedBlock.Hash, uint64(uncommittedBlock.Height)) if err != nil { mempoolUtxoView.EventManager.stateSyncerFlushed(&StateSyncerFlushedEvent{ FlushId: originalCommittedFlushId, diff --git a/lib/txindex.go b/lib/txindex.go index 29bc0336c..1f5158c1b 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -399,7 +399,7 @@ func (txi *TXIndex) Update() error { utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) if blockToAttach.Header.PrevBlockHash != nil && !utxoView.TipHash.IsEqual(blockToAttach.Header.PrevBlockHash) { var utxoViewAndUtxoOps *BlockViewAndUtxoOps - utxoViewAndUtxoOps, err = txi.TXIndexChain.getUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) + utxoViewAndUtxoOps, err = txi.TXIndexChain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockToAttach.Header.PrevBlockHash, blockToAttach.Header.Height-1) if err != nil { return fmt.Errorf("Update: Problem getting UtxoView at block hash %v: %v", blockToAttach.Header.PrevBlockHash, err) From a859c72e50639e87291bdefce91ab7471cc7734a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 22:59:18 -0500 Subject: [PATCH 049/131] cleanup --- lib/blockchain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 653764400..fde5a0940 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1770,7 +1770,6 @@ func (bc *Blockchain) checkArchivalMode() bool { } firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight - _ = firstSnapshotHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? // We take a sampling of blocks to determine if we've downloaded all the blocks up to the first snapshot height. From 01cd500ab7eb573136c2c7e9af2ece2d396acbce Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 4 Dec 2024 23:52:52 -0500 Subject: [PATCH 050/131] change GetBlockFromBestChainByHash to GetBlockFromBestChainByHashAndOptionalHeight to improve performance when height is available --- lib/blockchain.go | 27 +++++++++++++++++++-------- lib/pos_blockchain.go | 2 +- lib/pos_blockchain_test.go | 22 +++++++++++----------- lib/server.go | 20 ++++++-------------- 4 files changed, 37 insertions(+), 34 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index fde5a0940..1f2a30ea8 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -454,9 +454,9 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode // height minus provided distance. // // This function is safe for concurrent access. -func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { - return nn.Ancestor(nn.Height-distance, blockIndex) -} +//func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { +// return nn.Ancestor(nn.Height-distance, blockIndex) +//} // CalcNextDifficultyTarget computes the difficulty target expected of the // next block. @@ -1215,7 +1215,7 @@ func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, // There are no block locators so a specific block is being requested // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHash(stopHash, true) + stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { // TODO: what should we really do here? @@ -1884,8 +1884,19 @@ func (bc *Blockchain) BestChain() []*BlockNode { panic("BestChain not supported.") } -func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeaderChain bool) (*BlockNode, bool, error) { - bn, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) +func (bc *Blockchain) GetBlockFromBestChainByHashAndOptionalHeight( + blockHash *BlockHash, + optionalHeight *uint64, + useHeaderChain bool, +) (*BlockNode, bool, error) { + var bn *BlockNode + var exists bool + var err error + if optionalHeight != nil { + bn, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, *optionalHeight) + } else { + bn, exists, err = bc.blockIndex.GetBlockNodeByHashOnly(blockHash) + } if err != nil { return nil, false, err } @@ -1906,11 +1917,11 @@ func (bc *Blockchain) GetBlockFromBestChainByHash(blockHash *BlockHash, useHeade blockTip = bc.HeaderTip() } if blockTip == nil { - return nil, false, fmt.Errorf("GetBlockFromBestChainByHash: Block tip not found: use header chain: %v", useHeaderChain) + return nil, false, fmt.Errorf("GetBlockFromBestChainByHashAndOptionalHeight: Block tip not found: use header chain: %v", useHeaderChain) } committedTip, exists := bc.GetCommittedTip() if !exists { - return nil, false, errors.New("GetBlockFromBestChainByHash: Committed tip not found") + return nil, false, errors.New("GetBlockFromBestChainByHashAndOptionalHeight: Committed tip not found") } if uint64(bn.Height) > uint64(blockTip.Height) || uint64(bn.Height) < uint64(committedTip.Height) { return nil, false, nil diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 7c8e79f4f..1c959dde5 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -53,7 +53,7 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly // updated. This is necessary because the block index may have been updated with the header but the // bestHeaderChain.ChainMap may not have been updated yet. - blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHash(headerHash, true) + blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(headerHash, &header.Height, true) if err != nil { return nil, false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 75ef9beea..d3ee5ac1c 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1338,7 +1338,7 @@ func TestTryApplyNewTip(t *testing.T) { ancestors, _, err := bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) checkBestChainForHash := func(hash *BlockHash) bool { - _, exists, err := bc.GetBlockFromBestChainByHash(hash, false) + _, exists, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash, nil, false) require.NoError(t, err) return exists } @@ -1353,7 +1353,7 @@ func TestTryApplyNewTip(t *testing.T) { require.NoError(t, err) require.True(t, appliedNewTip) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash3, false) + _, hash3ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash3, nil, false) require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) @@ -1361,14 +1361,14 @@ func TestTryApplyNewTip(t *testing.T) { require.Len(t, disconnectedBlockHashes, 1) // newBlock should be in the best chain and the best chain map and should be the tip. - _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(newBlockHash, false) + _, newBlockExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(newBlockHash, nil, false) require.NoError(t, err) require.True(t, newBlockExistsInBestChainMap) require.True(t, checkBestChainForHash(newBlockHash)) require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // Make sure block 2 and block 1 are still in the best chain. - _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash2, false) + _, hash2ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash2, nil, false) require.NoError(t, err) require.True(t, hash2ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash2)) @@ -1428,22 +1428,22 @@ func TestTryApplyNewTip(t *testing.T) { // newBlockHash should be tip. require.True(t, bc.BlockTip().Hash.IsEqual(newBlockHash)) // hash 3 should no longer be in the best chain or best chain map - _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash3, false) + _, hash3ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHashAndOptionalHeight(hash3, nil, false) require.NoError(t, err) require.False(t, hash3ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash3)) // hash 2 should no longer be in the best chain or best chain map - _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHash(hash2, false) + _, hash2ExistsInBestChainMap, err = bc.GetBlockFromBestChainByHashAndOptionalHeight(hash2, nil, false) require.NoError(t, err) require.False(t, hash2ExistsInBestChainMap) require.False(t, checkBestChainForHash(hash2)) // hash 4 should be in the best chain and the best chain map - _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash4, false) + _, hash4ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash4, nil, false) require.NoError(t, err) require.True(t, hash4ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash4)) // hash 5 should be in the best chain and the best chain map - _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHash(hash5, false) + _, hash5ExistsInBestChainMap, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(hash5, nil, false) require.NoError(t, err) require.True(t, hash5ExistsInBestChainMap) require.True(t, checkBestChainForHash(hash5)) @@ -1670,7 +1670,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, committedHash := range committedBlocks { // Okay so let's make sure the block is committed. - blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(committedHash, false) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(committedHash, nil, false) require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.True(testMeta.t, blockNode.IsCommitted()) @@ -1696,7 +1696,7 @@ func _verifyCommitRuleHelper(testMeta *TestMeta, committedBlocks []*BlockHash, u } for _, uncommittedBlockHash := range uncommittedBlocks { // Okay so let's make sure the block is uncommitted. - blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHash(uncommittedBlockHash, false) + blockNode, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(uncommittedBlockHash, nil, false) require.NoError(testMeta.t, err) require.True(testMeta.t, exists) require.False(testMeta.t, blockNode.IsCommitted()) @@ -1889,7 +1889,7 @@ func testProcessBlockPoS(t *testing.T, testMeta *TestMeta) { // Timeout block will no longer be in best chain, and will still be in an uncommitted state in the block index _verifyCommitRuleHelper(testMeta, []*BlockHash{blockHash1, blockHash2}, []*BlockHash{blockHash3, reorgBlockHash}, blockHash2) _verifyRandomSeedHashHelper(testMeta, reorgBlock) - _, exists, err := testMeta.chain.GetBlockFromBestChainByHash(timeoutBlockHash, false) + _, exists, err := testMeta.chain.GetBlockFromBestChainByHashAndOptionalHeight(timeoutBlockHash, nil, false) require.NoError(t, err) require.False(t, exists) diff --git a/lib/server.go b/lib/server.go index c5f25c75c..179660029 100644 --- a/lib/server.go +++ b/lib/server.go @@ -807,7 +807,7 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( ) ([]*MsgDeSoHeader, error) { var headers []*MsgDeSoHeader - stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHash(stopHash, true) + stopNode, stopNodeExists, stopNodeError := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) // Special case when there is no block locator provided but only a stop hash. if len(locator) == 0 { if stopNodeError != nil || !stopNodeExists || stopNode == nil { @@ -815,7 +815,7 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( } return []*MsgDeSoHeader{stopNode.Header}, nil } - startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHash(locator[0], true) + startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(locator[0], nil, true) if startNodeError != nil || !startNodeExists || startNode == nil { return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") } @@ -1108,18 +1108,10 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b if checkpointBlockInfo == nil { return true, false } - var hasSeenCheckpointBlockHash bool - var checkpointBlockNode *BlockNode - var err error srv.blockchain.ChainLock.RLock() defer srv.blockchain.ChainLock.RUnlock() - if isHeaderChain { - checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, true) - } else { - checkpointBlockNode, hasSeenCheckpointBlockHash, err = srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, false) - } + checkpointBlockNode, hasSeenCheckpointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( + checkpointBlockInfo.Hash, &checkpointBlockInfo.Height, isHeaderChain) if err != nil { glog.Fatalf("shouldVerifySignatures: Problem getting checkpoint block node from best chain: %v", err) } @@ -1149,8 +1141,8 @@ func (srv *Server) getCheckpointSyncingStatus(isHeaders bool) string { if checkpointBlockInfo == nil { return "" } - _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHash( - checkpointBlockInfo.Hash, isHeaders) + _, hasSeenCheckPointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( + checkpointBlockInfo.Hash, &checkpointBlockInfo.Height, isHeaders) if err != nil { glog.Fatalf("getCheckpointSyncingStatus: Problem getting checkpoint block node from best chain: %v", err) From 09091df5e447224ad222f8703fce30b6157f17ae Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:01:35 -0500 Subject: [PATCH 051/131] use GetBlockFromBestChainByHashAndOptionalHeight in GetBlockNodesToFetch --- lib/blockchain.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 1f2a30ea8..40d958929 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1482,7 +1482,9 @@ func (bc *Blockchain) GetBlockNodesToFetch( // If the tip of the best block chain is in the main header chain, make that // the start point for our fetch. - headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHeight(uint64(bestBlockTip.Height), true) + bestBlockTipHeight := uint64(bestBlockTip.Height) + headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight( + bestBlockTip.Hash, &bestBlockTipHeight, true) if err != nil { glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) return nil From 77a1f57e8fe20c656ca547c2327ba53251106b47 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:32:47 -0500 Subject: [PATCH 052/131] nuke locator and inventory logic, just return a slice of the tip hash --- lib/blockchain.go | 287 +--------------------------------------------- lib/server.go | 15 +-- 2 files changed, 8 insertions(+), 294 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 40d958929..6cc75255b 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1174,289 +1174,8 @@ func NewBlockchain( return bc, nil } -// log2FloorMasks defines the masks to use when quickly calculating -// floor(log2(x)) in a constant log2(32) = 5 steps, where x is a uint32, using -// shifts. They are derived from (2^(2^x) - 1) * (2^(2^x)), for x in 4..0. -var log2FloorMasks = []uint32{0xffff0000, 0xff00, 0xf0, 0xc, 0x2} - -// fastLog2Floor calculates and returns floor(log2(x)) in a constant 5 steps. -func fastLog2Floor(n uint32) uint8 { - rv := uint8(0) - exponent := uint8(16) - for i := 0; i < 5; i++ { - if n&log2FloorMasks[i] != 0 { - rv += exponent - n >>= exponent - } - exponent >>= 1 - } - return rv -} - -// TODO: deprecate -// locateInventory returns the node of the block after the first known block in -// the locator along with the number of subsequent nodes needed to either reach -// the provided stop hash or the provided max number of entries. -// -// In addition, there are two special cases: -// -// - When no locators are provided, the stop hash is treated as a request for -// that block, so it will either return the node associated with the stop hash -// if it is known, or nil if it is unknown -// - When locators are provided, but none of them are known, nodes starting -// after the genesis block will be returned -// -// This is primarily a helper function for the locateBlocks and locateHeaders -// functions. -// -// This function MUST be called with the chain state lock held (for reads). -// TODO: this function needs a whole bunch of work. -func (bc *Blockchain) locateInventory(locator []*BlockHash, stopHash *BlockHash, maxEntries uint32) (*BlockNode, uint32) { - - // There are no block locators so a specific block is being requested - // as identified by the stop hash. - stopNode, stopNodeExists, stopNodeError := bc.GetBlockFromBestChainByHashAndOptionalHeight(stopHash, nil, true) - if len(locator) == 0 { - if stopNodeError != nil || !stopNodeExists || stopNode == nil { - // TODO: what should we really do here? - glog.Errorf("locateInventory: Block %v is not known", stopHash) - // No blocks with the stop hash were found so there is - // nothing to do. - return nil, 0 - } - return stopNode, 1 - } - - // Find the most recent locator block hash in the main chain. In the - // case none of the hashes in the locator are in the main chain, fall - // back to the genesis block. - startNode, startNodeExists, err := bc.GetBlockFromBestChainByHeight(0, true) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by height: %v", err) - return nil, 0 - } - if !startNodeExists { - glog.Errorf("locateInventory: Genesis block not found") - return nil, 0 - } - for _, hash := range locator { - // TODO: replace w/ read-through cache call. - node := bc.GetBlockNodeWithHash(hash) - if node != nil { - startNode = node - break - } - } - - // Start at the block after the most recently known block. When there - // is no next block it means the most recently known block is the tip of - // the best chain, so there is nothing more to do. - nextNodeHeight := uint32(startNode.Header.Height) + 1 - startNode, startNodeExists, err = bc.GetBlockFromBestChainByHeight(uint64(nextNodeHeight), true) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by height: %v", err) - return nil, 0 - } - if !startNodeExists { - return nil, 0 - } - - // Calculate how many entries are needed. - total := (bc.blockIndex.GetTip().Height - startNode.Height) + 1 - if stopNodeError != nil && stopNodeExists && stopNode != nil && - stopNode.Header.Height >= startNode.Header.Height { - - _, bestChainContainsStopNode, err := bc.blockIndex.GetBlockNodeByHashOnly(stopNode.Hash) - if err != nil { - glog.Errorf("locateInventory: Problem getting block by hash: %v", err) - return nil, 0 - } - if bestChainContainsStopNode { - total = uint32((stopNode.Header.Height - startNode.Header.Height) + 1) - } - } - if total > maxEntries { - total = maxEntries - } - - return startNode, total -} - -// TODO: deprecate -// locateHeaders returns the headers of the blocks after the first known block -// in the locator until the provided stop hash is reached, or up to the provided -// max number of block headers. -// -// See the comment on the exported function for more details on special cases. -// -// This function MUST be called with the ChainLock held (for reads). -func (bc *Blockchain) locateHeaders(locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { - - // Find the node after the first known block in the locator and the - // total number of nodes after it needed while respecting the stop hash - // and max entries. - node, total := bc.locateInventory(locator, stopHash, maxHeaders) - if total == 0 { - return nil - } - - // Populate and return the found headers. - headers, err := SafeMakeSliceWithLengthAndCapacity[*MsgDeSoHeader](0, uint64(total)) - if err != nil { - // TODO: do we really want to introduce an error here? - } - for ii := uint32(0); ii < total; ii++ { - headers = append(headers, node.Header) - if uint32(len(headers)) == total { - break - } - var nodeExists bool - node, nodeExists, err = bc.GetBlockFromBestChainByHeight(node.Header.Height+1, true) - if err != nil { - glog.Errorf("locateHeaders: Problem getting block by height: %v", err) - break - } - if !nodeExists { - break - } - } - return headers -} - -// TODO: deprecate -// LocateBestBlockChainHeaders returns the headers of the blocks after the first known block -// in the locator until the provided stop hash is reached, or up to a max of -// wire.MaxBlockHeadersPerMsg headers. Note that it returns the best headers -// considering only headers for which we have blocks (that is, it considers the -// best *block* chain we have rather than the best *header* chain). This is -// the correct thing to do because in general this function is called in order -// to serve a response to a peer's GetHeaders request. -// -// In addition, there are two special cases: -// -// - When no locators are provided, the stop hash is treated as a request for -// that header, so it will either return the header for the stop hash itself -// if it is known, or nil if it is unknown -// - When locators are provided, but none of them are known, headers starting -// after the genesis block will be returned -// -// This function is safe for concurrent access. -func (bc *Blockchain) LocateBestBlockChainHeaders( - locator []*BlockHash, stopHash *BlockHash, maxHeaders uint32) []*MsgDeSoHeader { - - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - headers := bc.locateHeaders(locator, stopHash, maxHeaders) - - return headers -} - -// LatestLocator returns a block locator for the passed block node. The passed -// node can be nil in which case the block locator for the current tip -// associated with the view will be returned. -// -// BlockLocator is used to help locate a specific block. The algorithm for -// building the block locator is to add the hashes in reverse order until -// the genesis block is reached. In order to keep the list of locator hashes -// to a reasonable number of entries, first the most recent previous 12 block -// hashes are added, then the step is doubled each loop iteration to -// exponentially decrease the number of hashes as a function of the distance -// from the block being located. -// -// For example, assume a block chain with a side chain as depicted below: -// -// genesis -> 1 -> 2 -> ... -> 15 -> 16 -> 17 -> 18 -// \-> 16a -> 17a -// -// The block locator for block 17a would be the hashes of blocks: -// [17a 16a 15 14 13 12 11 10 9 8 7 6 4 genesis] -// -// Caller is responsible for acquiring the ChainLock before calling this function. -func (bc *Blockchain) LatestLocator(tip *BlockNode) []*BlockHash { - - // Calculate the max number of entries that will ultimately be in the - // block locator. See the description of the algorithm for how these - // numbers are derived. - var maxEntries uint8 - if tip.Header.Height <= 12 { - maxEntries = uint8(tip.Header.Height) + 1 - } else { - // Requested hash itself + previous 10 entries + genesis block. - // Then floor(log2(height-10)) entries for the skip portion. - adjustedHeight := uint32(tip.Header.Height) - 10 - maxEntries = 12 + fastLog2Floor(adjustedHeight) - } - locator := make([]*BlockHash, 0, maxEntries) - - step := int32(1) - for tip != nil { - locator = append(locator, tip.Hash) - - // Nothing more to add once the genesis block has been added. - if tip.Header.Height == 0 { - break - } - - // Calculate height of previous node to include ensuring the - // final node is the genesis block. - height := int32(tip.Header.Height) - step - if height < 0 { - height = 0 - } - - // When the node is in the current chain view, all of its - // ancestors must be too, so use a much faster O(1) lookup in - // that case. Otherwise, fall back to walking backwards through - // the nodes of the other chain to the correct ancestor. - _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(tip.Hash) - if err != nil { - glog.Errorf("LatestLocator: Problem getting block by hash: %v", err) - exists = false - } - if exists { - var innerExists bool - tip, innerExists, err = bc.GetBlockFromBestChainByHeight(uint64(height), true) - if err != nil { - glog.Errorf("LatestLocator: Problem getting block by height: %v", err) - break - } - if !innerExists { - glog.Errorf("LatestLocator: Block %v not found in best header chain", height) - break - } - } else { - tip = tip.Ancestor(uint32(height), bc.blockIndex) - } - - // Once 11 entries have been included, start doubling the - // distance between included hashes. - if len(locator) > 10 { - step *= 2 - } - } - - return locator -} - -func (bc *Blockchain) HeaderLocatorWithNodeHash(blockHash *BlockHash) ([]*BlockHash, error) { - // We can acquire the ChainLock because the only place this is called currently is from - // _handleHeaderBundle, which doesn't have the lock. - // If we do not acquire the lock, we may hit a concurrent map read write error which causes panic. - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - node, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(blockHash) - if err != nil { - return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Problem getting node for hash %v: %v", blockHash, err) - } - if !exists || node == nil { - return nil, fmt.Errorf("Blockchain.HeaderLocatorWithNodeHash: Node for hash %v is not in our blockIndexByHash", blockHash) - } - - return bc.LatestLocator(node), nil -} - -// LatestHeaderLocator calls LatestLocator in order to fetch a locator -// for the best header chain. +// LatestHeaderLocator calls returns a block locator for the current tip of the +// header chain. func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { // We can acquire the ChainLock here because all calls to this function happen in peer.go // and server.go, which don't hold the lock. @@ -1465,7 +1184,7 @@ func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { defer bc.ChainLock.RUnlock() headerTip := bc.headerTip() - return bc.LatestLocator(headerTip) + return []*BlockHash{headerTip.Hash} } func (bc *Blockchain) GetBlockNodesToFetch( diff --git a/lib/server.go b/lib/server.go index 179660029..11154b265 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1124,6 +1124,9 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } return false, false } + // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid + // checking if we have the checkpoint block node if the header we're processing is below the height. + // This will save us 17-18% of the time it takes to process headers. // If the current header has a height below the checkpoint block height, we should skip signature verification // even if we've seen the checkpoint block hash. if header.Height < checkpointBlockInfo.Height { @@ -1210,6 +1213,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // If we get here then we have a header we haven't seen before. // check if we need to verify signatures + // TODO: we can add some logic into should verify signatures to avoid trying to get the checkpoint block node. verifySignatures, shouldDisconnect := srv.shouldVerifySignatures(headerReceived, true) if shouldDisconnect { glog.Errorf("Server._handleHeaderBundle: Disconnecting peer %v in state %s because a mismatch was "+ @@ -1476,18 +1480,9 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // even if the peer has a long fork with more work than our current header // chain. lastHash, _ := msg.Headers[len(msg.Headers)-1].Hash() - locator, err := srv.blockchain.HeaderLocatorWithNodeHash(lastHash) - if err != nil { - glog.Warningf("Server._handleHeaderBundle: Disconnecting peer %v because "+ - "she indicated that she has more headers but the last hash %v in "+ - "the header bundle does not correspond to a block in our index.", - pp, lastHash) - pp.Disconnect("Last hash in header bundle not in our index") - return - } pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, - BlockLocator: locator, + BlockLocator: []*BlockHash{lastHash}, }, false) headerTip := srv.blockchain.headerTip() glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ From 0585ebc1a7464c87181093953a18d5e4d8ef747e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:35:11 -0500 Subject: [PATCH 053/131] move check in should verify signatures up in function to reduce amount of times we need to fetch a block node we clearly know isn't there --- lib/server.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/server.go b/lib/server.go index 11154b265..69d376591 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1108,6 +1108,14 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b if checkpointBlockInfo == nil { return true, false } + // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid + // checking if we have the checkpoint block node if the header we're processing is below the height. + // This will save us 17-18% of the time it takes to process headers. + // If the current header has a height below the checkpoint block height, we should skip signature verification + // even if we've seen the checkpoint block hash. + if header.Height < checkpointBlockInfo.Height { + return false, false + } srv.blockchain.ChainLock.RLock() defer srv.blockchain.ChainLock.RUnlock() checkpointBlockNode, hasSeenCheckpointBlockHash, err := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( @@ -1124,14 +1132,6 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b } return false, false } - // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid - // checking if we have the checkpoint block node if the header we're processing is below the height. - // This will save us 17-18% of the time it takes to process headers. - // If the current header has a height below the checkpoint block height, we should skip signature verification - // even if we've seen the checkpoint block hash. - if header.Height < checkpointBlockInfo.Height { - return false, false - } // Make sure that the header in the best chain map has the correct height, otherwise we need to disconnect this peer. if uint64(checkpointBlockNode.Height) != checkpointBlockInfo.Height { return true, true From 139fb2ca0da00c0daf7f0e0e1bfe2f9596404617 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 00:41:02 -0500 Subject: [PATCH 054/131] reduce size of block index cache --- lib/blockchain.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 6cc75255b..6e9457ae9 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -59,7 +59,9 @@ const ( // have room for multiple forks each an entire history's length with this value). If // each node takes up 100 bytes of space this amounts to around 500MB, which also seems // like a reasonable size. - MaxBlockIndexNodes = 50000000 // TODO: trim this down somehow... + // UPDATE: now that we don't keep everything in memory, we reduced this value from + // 50000000 to 1000000 + MaxBlockIndexNodes = 1000000 // TODO: trim this down somehow... ) type BlockStatus uint32 From 4259c393b406419bc7be0cdef2f5a5a60e96b70c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 11:57:49 -0500 Subject: [PATCH 055/131] use header chain for get blocks to store --- lib/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/server.go b/lib/server.go index 69d376591..9b2a4b592 100644 --- a/lib/server.go +++ b/lib/server.go @@ -941,7 +941,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { - blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), false) + blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return @@ -1248,7 +1248,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { pp.Disconnect("Error processing header") // Just to be safe, we flush all the headers we just got even tho we have a header. currTime := time.Now() - if err := PutHeightHashToNodeInfoBatch( + if err = PutHeightHashToNodeInfoBatch( srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager); err != nil { glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) return From aea51a8c129b60640beb3a14e3aee65a723ecb3a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 14:08:23 -0500 Subject: [PATCH 056/131] fix return value from processHeaderPoW --- lib/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 6e9457ae9..231580b33 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2249,7 +2249,7 @@ func (bc *Blockchain) processHeaderPoW( bc.blockIndex.setHeaderTip(newNode) } - return blockNode, isMainChain, false, nil + return newNode, isMainChain, false, nil } // ProcessHeader is a wrapper around processHeaderPoW and processHeaderPoS, which do the leg-work. From fc63a94af39875b92f98149f758d94ce741be5a7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 15:04:28 -0500 Subject: [PATCH 057/131] disable deadlock detection around putting block nodes into db after handling snapshot --- lib/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/server.go b/lib/server.go index 9b2a4b592..6099dc5b7 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1826,6 +1826,13 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { // being too large and possibly causing an error in badger. glog.V(0).Infof("Server._handleSnapshot: Updating snapshot block nodes in the database") var blockNodeBatch []*BlockNode + flushBlockNodeStartTime := time.Now() + // Disable deadlock detection, as the process of flushing entries to file can take a long time and + // if it takes longer than the deadlock detection timeout interval, it will cause an error to be thrown. + deadlock.Opts.Disable = true + defer func() { + deadlock.Opts.Disable = false + }() // acquire the chain lock while we update the best chain and best chain map. srv.blockchain.ChainLock.Lock() for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { @@ -1861,6 +1868,8 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { glog.Errorf("Server._handleSnapshot: Problem updating snapshot block nodes, error: (%v)", err) } } + glog.V(0).Infof("Time to store %v block nodes in the database: %v", + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, time.Since(flushBlockNodeStartTime)) err = PutBestHash(srv.blockchain.db, srv.snapshot, msg.SnapshotMetadata.CurrentEpochBlockHash, ChainTypeDeSoBlock, srv.eventManager) if err != nil { From 90b5372b68f1d0ae81d0c6b432f2bd8543f65ba6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 15:17:39 -0500 Subject: [PATCH 058/131] add more logging as we're processing headers after finishing downloading snapshot --- lib/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/server.go b/lib/server.go index 6099dc5b7..35449a593 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1852,6 +1852,14 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) + if currentNode.Height%100000 == 0 { + glog.V(0).Infof("Time to process %v of %v block nodes in %v", + currentNode.Height, + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, + time.Since(flushBlockNodeStartTime), + ) + } + // TODO: should we adjust this value for batch sizes? if len(blockNodeBatch) < 10000 { continue } From f9a4f3e3b5ae34008e70cede9e59b955546dc39a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 5 Dec 2024 16:20:01 -0500 Subject: [PATCH 059/131] iterate backwards to update status of block nodes after finishing hypersync --- lib/server.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/lib/server.go b/lib/server.go index 35449a593..399ef4d23 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1835,16 +1835,19 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { }() // acquire the chain lock while we update the best chain and best chain map. srv.blockchain.ChainLock.Lock() - for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { - currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(ii, true) - if err != nil { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) - break - } - if !currentNodeExists { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") - break - } + // TODO: we should iterate in reverse so we can use GetBlockFromBestChainByHashAndOptionalHeight + // by doing currentNode.Height - 1 and currentNode.Header.PrevBlockHash. + currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, true) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + // TODO: should we return here? + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + // TODO: should we return here? + } + for currentNode.Height > 0 { + //for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { // Do not set the StatusBlockStored flag, because we still need to download the past blocks. currentNode.Status |= StatusBlockProcessed currentNode.Status |= StatusBlockValidated @@ -1852,13 +1855,24 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) - if currentNode.Height%100000 == 0 { + if (srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height))%100000 == 0 { glog.V(0).Infof("Time to process %v of %v block nodes in %v", - currentNode.Height, + srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height), srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, time.Since(flushBlockNodeStartTime), ) } + + prevNodeHeight := uint64(currentNode.Height) - 1 + currentNode, currentNodeExists, err = srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(currentNode.Header.PrevBlockHash, &prevNodeHeight, true) + if err != nil { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) + break + } + if !currentNodeExists { + glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") + break + } // TODO: should we adjust this value for batch sizes? if len(blockNodeBatch) < 10000 { continue From f425317c89e173057c9c4d70ccb4403290b46614 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 6 Dec 2024 15:03:46 -0500 Subject: [PATCH 060/131] fix set tip call since we reverse backwards through the block nodes at the end of hypersync --- lib/server.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/server.go b/lib/server.go index 399ef4d23..0c913fd95 100644 --- a/lib/server.go +++ b/lib/server.go @@ -972,7 +972,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. - currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, false) + currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) return @@ -1846,6 +1846,8 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") // TODO: should we return here? } + // Set the block tip to the snapshot height block node. + srv.blockchain.blockIndex.setTip(currentNode) for currentNode.Height > 0 { //for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { // Do not set the StatusBlockStored flag, because we still need to download the past blocks. @@ -1853,7 +1855,6 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { currentNode.Status |= StatusBlockValidated currentNode.Status |= StatusBlockCommitted srv.blockchain.addNewBlockNodeToBlockIndex(currentNode) - srv.blockchain.blockIndex.setTip(currentNode) blockNodeBatch = append(blockNodeBatch, currentNode) if (srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight-uint64(currentNode.Height))%100000 == 0 { glog.V(0).Infof("Time to process %v of %v block nodes in %v", From 231a6dc398e8bf47bb7c8de7800bdb2387c6a581 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 12:09:17 -0500 Subject: [PATCH 061/131] small clean up, add some todos --- cmd/config.go | 1 - lib/server.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/config.go b/cmd/config.go index fe04cbac7..7a5278900 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -266,7 +266,6 @@ func (config *Config) Print() { glog.Infof("MaxSyncBlockHeight: %v", config.MaxSyncBlockHeight) } - glog.Infof("Connect IPs: %s", config.ConnectIPs) if len(config.ConnectIPs) > 0 { glog.Infof("Connect IPs: %s", config.ConnectIPs) } diff --git a/lib/server.go b/lib/server.go index 0c913fd95..2332d0f89 100644 --- a/lib/server.go +++ b/lib/server.go @@ -941,6 +941,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { // Go through the block nodes in the blockchain and download the blocks if they're not stored. for ii := uint32(srv.blockchain.lowestBlockNotStored); ii <= srv.blockchain.blockTip().Height; ii++ { + // TODO: this may be really slow. blockNode, exists, err := srv.blockchain.GetBlockFromBestChainByHeight(uint64(ii), true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) @@ -972,6 +973,7 @@ func (srv *Server) GetBlocksToStore(pp *Peer) { } // Get the current hash and increment the height. Genesis has height 0, so currentHeight corresponds to // the array index. + // TODO: this may be really slow. currentNode, currNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(currentHeight, true) if err != nil { glog.Errorf("GetBlocksToStore: Error getting block from best chain by height: %v", err) From ca70c8a5c125a75b01a1f05321eec4576572e05b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 17:42:10 -0500 Subject: [PATCH 062/131] add more comments and TODOs and clean up for all files except blockchain.go, pos_blockchain.go, pos_blockchain_test.go, and server.go --- cmd/config.go | 5 +- collections/lru.go | 75 +++++++++++++++++++++++ lib/block_view_test.go | 4 +- lib/blockchain.go | 53 +++++----------- lib/connection_manager.go | 6 +- lib/constants.go | 3 + lib/db_utils.go | 121 +++++++++++++++++++++++++------------ lib/miner.go | 1 - lib/network_manager.go | 11 ++-- lib/peer.go | 12 ++-- lib/pos_blockchain.go | 12 +--- lib/pos_blockchain_test.go | 14 ++--- lib/pos_mempool.go | 16 ++--- lib/postgres.go | 9 +-- lib/server.go | 11 ++-- lib/snapshot.go | 12 ++-- lib/txindex.go | 87 -------------------------- 17 files changed, 228 insertions(+), 224 deletions(-) create mode 100644 collections/lru.go diff --git a/cmd/config.go b/cmd/config.go index 7a5278900..31f003960 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -94,8 +94,8 @@ type Config struct { func GetStringSliceWorkaround(flagName string) []string { value := viper.GetString(flagName) if value == "" || value == " " { - values := viper.GetStringSlice(flagName) - return values + // Backwards compatibility for n0 scripts. + return viper.GetStringSlice(flagName) } return strings.Split(value, ",") } @@ -148,6 +148,7 @@ func LoadConfig() *Config { // Peers config.ConnectIPs = GetStringSliceWorkaround("connect-ips") + glog.V(2).Infof("Connect IPs read in: %v", config.ConnectIPs) config.AddIPs = GetStringSliceWorkaround("add-ips") config.AddSeeds = GetStringSliceWorkaround("add-seeds") config.TargetOutboundPeers = viper.GetUint32("target-outbound-peers") diff --git a/collections/lru.go b/collections/lru.go new file mode 100644 index 000000000..7033541b3 --- /dev/null +++ b/collections/lru.go @@ -0,0 +1,75 @@ +package collections + +import lru "github.com/hashicorp/golang-lru/v2" + +// We implement our own LRU cache as a wrapper around a dependency. This allows us to +// easily change the underlying implementation in the future if needed with minimal +// changes outside of this file. + +type LruCache[K comparable, V any] struct { + underlyingCache *lru.Cache[K, V] +} + +func NewLruCache[K comparable, V any](maxSize int) (*LruCache[K, V], error) { + underlyingCache, err := lru.New[K, V](maxSize) + if err != nil { + return nil, err + } + return &LruCache[K, V]{underlyingCache}, nil +} + +func (lruCache *LruCache[K, V]) Put(key K, value V) { + lruCache.underlyingCache.Add(key, value) +} + +func (lruCache *LruCache[K, V]) Get(key K) (V, bool) { + return lruCache.underlyingCache.Get(key) +} + +func (lruCache *LruCache[K, V]) Contains(key K) bool { + return lruCache.underlyingCache.Contains(key) +} + +func (lruCache *LruCache[K, V]) Delete(key K) { + lruCache.underlyingCache.Remove(key) +} + +func (lruCache *LruCache[K, V]) Purge() { + lruCache.underlyingCache.Purge() +} + +func (lruCache *LruCache[K, V]) Keys() []K { + return lruCache.underlyingCache.Keys() +} + +type LruSet[K comparable] struct { + underlyingCache *lru.Cache[K, struct{}] +} + +func NewLruSet[K comparable](maxSize int) (*LruSet[K], error) { + underlyingCache, err := lru.New[K, struct{}](maxSize) + if err != nil { + return nil, err + } + return &LruSet[K]{underlyingCache}, nil +} + +func (lruSet *LruSet[K]) Put(key K) { + lruSet.underlyingCache.Add(key, struct{}{}) +} + +func (lruSet *LruSet[K]) Contains(key K) bool { + return lruSet.underlyingCache.Contains(key) +} + +func (lruSet *LruSet[K]) Delete(key K) { + lruSet.underlyingCache.Remove(key) +} + +func (lruSet *LruSet[K]) Purge() { + lruSet.underlyingCache.Purge() +} + +func (lruSet *LruSet[K]) Items() []K { + return lruSet.underlyingCache.Keys() +} diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 852404510..0f835cc02 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" - lru "github.com/hashicorp/golang-lru/v2" + "github.com/deso-protocol/core/collections" "math" _ "net/http/pprof" "reflect" @@ -797,7 +797,7 @@ func (tes *transactionTestSuite) testDisconnectBlock(tm *transactionTestMeta, te // We don't pass the chain's snapshot above to prevent certain concurrency issues. As a // result, we need to reset the snapshot's db cache to get rid of stale data. if tm.chain.snapshot != nil { - tm.chain.snapshot.DatabaseCache, err = lru.New[string, []byte](int(DatabaseCacheSize)) + tm.chain.snapshot.DatabaseCache, err = collections.NewLruCache[string, []byte](int(DatabaseCacheSize)) require.NoError(err) } diff --git a/lib/blockchain.go b/lib/blockchain.go index 231580b33..a1b4f12d0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -17,8 +17,6 @@ import ( "sync" "time" - "github.com/hashicorp/golang-lru/v2" - "github.com/deso-protocol/core/collections" "github.com/deso-protocol/uint256" @@ -572,15 +570,15 @@ type CheckpointBlockInfoAndError struct { type BlockIndex struct { db *badger.DB snapshot *Snapshot - blockIndexByHash *lru.Cache[BlockHash, *BlockNode] - blockIndexByHeight *lru.Cache[uint64, []*BlockNode] + blockIndexByHash *collections.LruCache[BlockHash, *BlockNode] + blockIndexByHeight *collections.LruCache[uint64, []*BlockNode] tip *BlockNode headerTip *BlockNode } func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { - blockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? - blockIndexByHeight, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHeight, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ db: db, snapshot: snapshot, @@ -591,8 +589,8 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) - newHeightToBlockNodeMap, _ := lru.New[uint64, []*BlockNode](MaxBlockIndexNodes) + newHashToBlockNodeMap, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + newHeightToBlockNodeMap, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? bi.blockIndexByHash = newHashToBlockNodeMap bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { @@ -620,7 +618,7 @@ func (bi *BlockIndex) setTip(tip *BlockNode) { } func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { - bi.blockIndexByHash.Add(*blockNode.Hash, blockNode) + bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) if !exists { blocksAtHeight = []*BlockNode{} @@ -635,7 +633,7 @@ func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { } } } - bi.blockIndexByHeight.Add(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) + bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) } func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { @@ -747,7 +745,7 @@ type Blockchain struct { blockView *UtxoView // cache block view for each block - blockViewCache *lru.Cache[BlockHash, *BlockViewAndUtxoOps] + blockViewCache *collections.LruCache[BlockHash, *BlockViewAndUtxoOps] // snapshot cache snapshotCache *SnapshotCache @@ -879,12 +877,12 @@ func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { } func (bc *Blockchain) CopyBlockIndexes() ( - _blockIndexByHash *lru.Cache[BlockHash, *BlockNode], + _blockIndexByHash *collections.LruCache[BlockHash, *BlockNode], ) { - newBlockIndexByHash, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + newBlockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) - newBlockIndexByHash.Add(key, val) + newBlockIndexByHash.Put(key, val) } return newBlockIndexByHash } @@ -1032,6 +1030,7 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } + // @diamondhands - we could reduce this to just the last hour if we want. // Walk back the last 24 hours of blocks. currBlockCounter := 1 for currBlockCounter < 3600*24 && tipNode.Header.PrevBlockHash != nil { @@ -1131,7 +1130,7 @@ func NewBlockchain( timer := &Timer{} timer.Initialize() - blockViewCache, _ := lru.New[BlockHash, *BlockViewAndUtxoOps](100) // TODO: parameterize + blockViewCache, _ := collections.NewLruCache[BlockHash, *BlockViewAndUtxoOps](100) // TODO: parameterize bc := &Blockchain{ db: db, postgres: postgres, @@ -1719,7 +1718,7 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { } func (bc *Blockchain) setBestChainMap( - blockIndexByHash *lru.Cache[BlockHash, *BlockNode], + blockIndexByHash *collections.LruCache[BlockHash, *BlockNode], tipNode *BlockNode, ) { bc.blockIndex.blockIndexByHash = blockIndexByHash @@ -2048,28 +2047,6 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( return commonAncestor, detachBlocks, attachBlocks } -func updateBestChainInMemory(mainChainList []*BlockNode, mainChainMap *lru.Cache[BlockHash, *BlockNode], detachBlocks []*BlockNode, attachBlocks []*BlockNode) ( - chainList []*BlockNode, chainMap *lru.Cache[BlockHash, *BlockNode]) { - - // Remove the nodes we detached from the end of the best chain node list. - tipIndex := len(mainChainList) - 1 - for blockOffset := 0; blockOffset < len(detachBlocks); blockOffset++ { - blockIndex := tipIndex - blockOffset - mainChainMap.Remove(*mainChainList[blockIndex].Hash) - } - mainChainList = mainChainList[:len(mainChainList)-len(detachBlocks)] - - // Add the nodes we attached to the end of the list. Note that this loop iterates - // forward because attachBlocks has the node right after the common ancestor - // first, with the new tip at the end. - for _, attachNode := range attachBlocks { - mainChainList = append(mainChainList, attachNode) - mainChainMap.Add(*attachNode.Hash, attachNode) - } - - return mainChainList, mainChainMap -} - // Caller must acquire the ChainLock for writing prior to calling this. func (bc *Blockchain) processHeaderPoW( blockHeader *MsgDeSoHeader, diff --git a/lib/connection_manager.go b/lib/connection_manager.go index 858b803de..7183355eb 100644 --- a/lib/connection_manager.go +++ b/lib/connection_manager.go @@ -2,6 +2,7 @@ package lib import ( "fmt" + "github.com/deso-protocol/core/collections" "math" "net" "sync" @@ -11,7 +12,6 @@ import ( "github.com/btcsuite/btcd/addrmgr" "github.com/btcsuite/btcd/wire" "github.com/golang/glog" - "github.com/hashicorp/golang-lru/v2" ) // connection_manager.go contains most of the logic for creating and managing @@ -53,7 +53,7 @@ type ConnectionManager struct { // TODO: seems like we don't use this. // Keep track of the nonces we've sent in our version messages so // we can prevent connections to ourselves. - sentNonces *lru.Cache[uint64, struct{}] + sentNonces *collections.LruSet[uint64] // This section defines the data structures for storing all the // peers we're aware of. @@ -126,7 +126,7 @@ func NewConnectionManager( ValidateHyperSyncFlags(_hyperSync, _syncType) - sentNoncesCache, _ := lru.New[uint64, struct{}](1000) + sentNoncesCache, _ := collections.NewLruSet[uint64](1000) return &ConnectionManager{ srv: _srv, params: _params, diff --git a/lib/constants.go b/lib/constants.go index 56348de8c..9a76bf346 100644 --- a/lib/constants.go +++ b/lib/constants.go @@ -1956,6 +1956,9 @@ const DefaultTestnetCheckpointProvider = "https://test.deso.org" const RoutePathGetCommittedTipBlockInfo = "/api/v0/get-committed-tip-block-info" +// BlockIndexMigrationFileName is the name of the file that contains a boolean value +// that indicates whether the block index migration has been run. See RunBlockIndexMigrationOnce +// for more information. const BlockIndexMigrationFileName = "block_index_migration.txt" // BtcecPubKeyBytesLenUncompressed is a constant that was removed from newer version of Btcec diff --git a/lib/db_utils.go b/lib/db_utils.go index 63f78fcbf..e1bcc248b 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -606,7 +606,11 @@ type DBPrefixes struct { // PrefixHashToHeight is used to store the height of a block given its hash. // This helps us map a block hash to its height so we can look up the full info - // in PrefixHeightHashToNodeInfo. + // in PrefixHeightHashToNodeInfo. Note that the block index migration will run + // to populate this index when upgrading the node to the new version that + // introduces this index. Before the introduction of this index, the only way + // to find a block node given its hash was to do a full scan of + // PrefixHeightHashToNodeInfo. PrefixHashToHeight []byte `prefix_id:"[98]"` // NEXT_TAG: 99 } @@ -1142,7 +1146,7 @@ func DBSetWithTxn(txn *badger.Txn, snap *Snapshot, key []byte, value []byte, eve return errors.Wrapf(err, "DBSetWithTxn: Problem preparing ancestral record") } // Now save the newest record to cache. - snap.DatabaseCache.Add(keyString, value) + snap.DatabaseCache.Put(keyString, value) if !snap.disableChecksum { // We have to remove the previous value from the state checksum. @@ -1241,7 +1245,7 @@ func DBDeleteWithTxn(txn *badger.Txn, snap *Snapshot, key []byte, eventManager * return errors.Wrapf(err, "DBDeleteWithTxn: Problem preparing ancestral record") } // Now delete the past record from the cache. - snap.DatabaseCache.Remove(keyString) + snap.DatabaseCache.Delete(keyString) // We have to remove the previous value from the state checksum. // Because checksum is commutative, we can safely remove the past value here. if !snap.disableChecksum { @@ -5192,6 +5196,8 @@ func _heightHashToNodeIndexPrefix(bitcoinNodes bool) []byte { return prefix } +// _heightHashToNodePrefixByHeight returns the prefix for the height hash to node index +// for a given height. This is useful to find all blocks at a given height. func _heightHashToNodePrefixByHeight(height uint32, bitcoinNodes bool) []byte { prefix := _heightHashToNodeIndexPrefix(bitcoinNodes) heightBytes := make([]byte, 4) @@ -5210,6 +5216,7 @@ func _heightHashToNodeIndexKey(height uint32, hash *BlockHash, bitcoinNodes bool return key } +// _hashToHeightIndexKey returns the key for the hash to height index. func _hashToHeightIndexKey(hash *BlockHash) []byte { key := append([]byte{}, Prefixes.PrefixHashToHeight...) key = append(key, hash[:]...) @@ -5252,10 +5259,12 @@ func PutHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, return errors.Wrapf(err, "PutHeightHashToNodeInfoWithTxn: Problem serializing node") } + // Store the full block node in the hash height to block node index. if err := DBSetWithTxn(txn, snap, key, serializedNode, eventManager); err != nil { return err } + // Also store the height to hash mapping. hashToHeightKey := _hashToHeightIndexKey(node.Hash) if err = DBSetWithTxn(txn, snap, hashToHeightKey, UintToBuf(uint64(node.Height)), eventManager); err != nil { return err @@ -5264,6 +5273,8 @@ func PutHeightHashToNodeInfoWithTxn(txn *badger.Txn, snap *Snapshot, return nil } +// PutHashToHeightBatch puts a map of block hashes to heights in the db in the hash to height index. +// This is only used for the block index migration. func PutHashToHeightBatch(handle *badger.DB, snap *Snapshot, hashToHeight map[BlockHash]uint32, eventManager *EventManager) error { return handle.Update(func(txn *badger.Txn) error { for hash, height := range hashToHeight { @@ -5276,6 +5287,7 @@ func PutHashToHeightBatch(handle *badger.DB, snap *Snapshot, hashToHeight map[Bl }) } +// GetHeightForHash returns the height for a given block hash by using the hash to height index. func GetHeightForHash(db *badger.DB, snap *Snapshot, hash *BlockHash) (uint64, error) { var height uint64 err := db.View(func(txn *badger.Txn) error { @@ -5342,7 +5354,7 @@ func DbBulkDeleteHeightHashToNodeInfo(handle *badger.DB, snap *Snapshot, nodes [ return nil } -// InitDbWithGenesisBlock initializes the database to contain only the genesis +// InitDbWithDeSoGenesisBlock initializes the database to contain only the genesis // block. func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB, eventManager *EventManager, snap *Snapshot, postgres *Postgres) error { @@ -5596,7 +5608,11 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } +// LoadBlockIndexFromHeight loads the block index from the database starting at +// a given height. This is only used by initChain, where we want to load all blocks +// that could be descendents of the current tip block in the database. func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams) error { + // Get the prefix for the provided height. prefix := _heightHashToNodePrefixByHeight(height, false) return bi.db.View(func(txn *badger.Txn) error { @@ -5612,9 +5628,6 @@ func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams err := item.Value(func(blockNodeBytes []byte) error { // Deserialize the block node. var err error - // TODO: There is room for optimization here by pre-allocating a - // contiguous list of block nodes and then populating that list - // rather than having each blockNode be a stand-alone allocation. blockNode, err = DeserializeBlockNode(blockNodeBytes) if err != nil { return err @@ -5630,48 +5643,54 @@ func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams bi.addNewBlockNodeToBlockIndex(blockNode) // Find the parent of this block, which should already have been read - // in and connect it. Skip the genesis block, which has height 0. Also - // skip the block if its PrevBlockHash is empty, which will be true for - // the BitcoinStartBlockNode. - // - // TODO: There is room for optimization here by keeping a reference to - // the last node we've iterated over and checking if that node is the - // parent. Doing this would avoid an expensive hashmap check to get - // the parent by its block hash. + // in and connect it. Skip the genesis block, which has height 0. if blockNode.Height == 0 || (*blockNode.Header.PrevBlockHash == BlockHash{}) { continue } if parent, ok := bi.GetBlockNodeByHashAndHeight(blockNode.Header.PrevBlockHash, uint64(blockNode.Height)); ok { // We found the parent node so connect it. blockNode.Parent = parent - } else { - // If we're syncing a DeSo node and we hit a PoS block, we expect there to - // be orphan blocks in the block index. In this case, we don't throw an error. - if params.IsPoSBlockHeight(uint64(blockNode.Height)) { - continue - } - // In this case we didn't find the parent so error. There shouldn't - // be any unconnectedTxns in our block index. - return fmt.Errorf("GetBlockIndex: Could not find parent for blockNode: %+v", blockNode) + continue + } + // If we're syncing a DeSo node and we hit a PoS block, we expect there to + // be orphan blocks in the block index. In this case, we don't throw an error. + if params.IsPoSBlockHeight(uint64(blockNode.Height)) { + continue } + // In this case we didn't find the parent so error. There shouldn't + // be any unconnectedTxns in our block index. + return fmt.Errorf("GetBlockIndex: Could not find parent for blockNode: %+v", blockNode) } return nil }) } +// RunBlockIndexMigration runs a migration to populate the hash to height index from the height hash to +// block node index. We can't use the encoder migrations to handle this situation since it's a new index +// and not a modification of the existing entry type stored. This migration simply iterates over the keys in the +// height hash to block node index, extracts the height and hash, and puts a key of the hash and a value of the +// height in the hash to height prefix. func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager *EventManager, params *DeSoParams) error { + // @diamondhands - if we want to migrate from a uint32 -> uint64 for height in the height hash to node index, + // this would be a good time to do it. It's not necessary, but it's a bit annoying that we use uint64 in some + // places and uint32 in others. Specifically, we don't always validate that we have a uint32 when we go to get + // the block from teh DB. return handle.Update(func(txn *badger.Txn) error { + // Get the prefix for the height hash to node index. prefix := _heightHashToNodeIndexPrefix(false) opts := badger.DefaultIteratorOptions opts.Prefix = prefix - // We don't need values for this migration. + // We don't need values for this migration since the height and hash are in the key. opts.PrefetchValues = false nodeIterator := txn.NewIterator(opts) defer nodeIterator.Close() + // Initialize a map to store the hash to height mappings. hashToHeightMap := make(map[BlockHash]uint32) // Just in case we need it, get the height of the best hash. bestHash := DbGetBestHash(handle, snapshot, ChainTypeDeSoBlock) var bestHashHeight uint32 + // Iterate over all the keys in the height hash to node index, extract the height and hash, + // and batch write every 10k entries to the hash to height index. for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { item := nodeIterator.Item().Key() @@ -5680,18 +5699,23 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager hash := BlockHash{} copy(hash[:], item[5:]) hashToHeightMap[hash] = height + // If we have a best hash, we want to store the height of the best hash. if bestHash != nil && bestHash.IsEqual(&hash) { bestHashHeight = height } + // If we have fewer than 10K entries, continue. if len(hashToHeightMap) < 10000 { continue } + // If we have more than 10K entries, batch write the entries to the hash to height index + // and reset the map. innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) if innerErr != nil { return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } hashToHeightMap = make(map[BlockHash]uint32) } + // If we have any entries left in the map, batch write them to the hash to height index. if len(hashToHeightMap) > 0 { innerErr := PutHashToHeightBatch(handle, snapshot, hashToHeightMap, eventManager) if innerErr != nil { @@ -5702,7 +5726,18 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager if bestHash == nil { return nil } - // TODO: get best chain up to PoS Cutover height and set all blocks in that chain to committed. + + // We want to mark all PoW blocks as committed, so we'll get the first pos block + // and iterate backwards marking all blocks as committed. If the PoS cutover hasn't + // happened yet, then we'll just the current best hash from the DB to determine the + // current tip to iterate back from. This allows us to track the best chain even if + // we have multiple proof-of-work blocks at the same height in the DB. Before the + // change to not keep the entire best chain in memory, we would know if a block was + // in the best chain by looking it up in the best chain map. Now that we no longer + // have this, we rely on the IsCommitted function to determine if blocks are in the + // best chain. Without this, nodes upgrading to this code that have been running for + // a long time and experienced PoW forks will have issues determining which blocks + // are in the best chain for PoW block heights. firstPoSBlockHeight := params.GetFirstPoSBlockHeight() // Look up blocks at cutover height. prefixKey := _heightHashToNodePrefixByHeight(uint32(firstPoSBlockHeight), false) @@ -5710,17 +5745,20 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") } + // There should be 0 or 1 blocks at the cutover height. if len(valsFound) > 1 { return fmt.Errorf("RunBlockIndexMigration: More than one block found at PoS cutover height") } var blockNode *BlockNode - // In this case, we need to find pull the best hash from the DB and iterate backwards. + // In this case we have not reached the cutover, we need to find pull the best hash + // from the DB and iterate backwards. if len(valsFound) == 0 { blockNode = GetHeightHashToNodeInfoWithTxn(txn, snapshot, bestHashHeight, bestHash, false) if blockNode == nil { return fmt.Errorf("RunBlockIndexMigration: block with Best hash (%v) and height (%v) not found", bestHash, bestHashHeight) } } else { + // If we have 1 block at the cutover height, we'll use that block to iterate backwards. blockNode, err = DeserializeBlockNode(valsFound[0]) if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") @@ -5728,34 +5766,43 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager } var blockNodeBatch []*BlockNode for blockNode != nil { + // If the block is not committed, mark it as committed. if !blockNode.IsCommitted() { blockNode.Status |= StatusBlockCommitted } - // TODO: make sure I don't need a copy. + // Add it to the batch. blockNodeBatch = append(blockNodeBatch, blockNode) + // Find the parent of this block. + parentBlockNode := GetHeightHashToNodeInfoWithTxn(txn, snapshot, blockNode.Height, blockNode.Hash, false /*bitcoinNodes*/) + if blockNode.Height > 0 && parentBlockNode == nil { + return errors.New("RunBlockIndexMigration: Parent block node not found") + } + // Jump up to the parent block node. + blockNode = parentBlockNode + // If we have fewer than 10K entries, continue. if len(blockNodeBatch) < 10000 { continue } + // If we have more than 10K entries, write the batch and reset the slice. err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") } - parentBlockNode := GetHeightHashToNodeInfoWithTxn(txn, snapshot, blockNode.Height, blockNode.Hash, false /*bitcoinNodes*/) - if blockNode.Height > 0 && parentBlockNode == nil { - return errors.New("RunBlockIndexMigration: Parent block node not found") - } - blockNode = parentBlockNode + blockNodeBatch = []*BlockNode{} } - err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) - if err != nil { - return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + // If any entries are left in the batch, write them to the DB. + if len(blockNodeBatch) > 0 { + err = PutHeightHashToNodeInfoBatch(handle, snapshot, blockNodeBatch, false /*bitcoinNodes*/, eventManager) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem putting block node batch") + } } return nil }) } // TODO: refactor to actually get the whole best chain if that's -// what someone wants. It'll take a while, but whatever. +// what someone wants. It'll take a while and a lot of memory. func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. diff --git a/lib/miner.go b/lib/miner.go index d912f5858..67ed8976a 100644 --- a/lib/miner.go +++ b/lib/miner.go @@ -208,7 +208,6 @@ func (desoMiner *DeSoMiner) MineAndProcessSingleBlock(threadIndex uint32, mempoo desoMiner.BlockProducer.chain.blockTip().Header) scs := spew.ConfigState{DisableMethods: true, Indent: " ", DisablePointerAddresses: true} glog.V(1).Infof(scs.Sdump(blockToMine)) - // Sanitize the block for the comparison we're about to do. We need to do // this because the comparison function below will think they're different // if one has nil and one has an empty list. Annoying, but this solves the diff --git a/lib/network_manager.go b/lib/network_manager.go index a77974800..8361b1a5b 100644 --- a/lib/network_manager.go +++ b/lib/network_manager.go @@ -15,7 +15,6 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" "github.com/golang/glog" - "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -69,7 +68,7 @@ type NetworkManager struct { NonValidatorInboundIndex *collections.ConcurrentMap[RemoteNodeId, *RemoteNode] // Cache of nonces used during handshake. - usedNonces *lru.Cache[uint64, struct{}] + usedNonces *collections.LruSet[uint64] // The address manager keeps track of peer addresses we're aware of. When // we need to connect to a new outbound peer, it chooses one of the addresses @@ -121,7 +120,7 @@ func NewNetworkManager( minTxFeeRateNanosPerKB uint64, nodeServices ServiceFlag, ) *NetworkManager { - usedNoncesCache, _ := lru.New[uint64, struct{}](1000) + usedNoncesCache, _ := collections.NewLruSet[uint64](1000) return &NetworkManager{ params: params, srv: srv, @@ -261,7 +260,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag // If we've seen this nonce before then return an error since this is a connection from ourselves. msgNonce := verMsg.Nonce if nm.usedNonces.Contains(msgNonce) { - nm.usedNonces.Remove(msgNonce) + nm.usedNonces.Delete(msgNonce) glog.Errorf("NetworkManager.handleVersionMessage: Disconnecting RemoteNode with id: (%v) "+ "nonce collision, nonce (%v)", origin.ID, msgNonce) nm.Disconnect(rn, "nonce collision") @@ -277,7 +276,7 @@ func (nm *NetworkManager) _handleVersionMessage(origin *Peer, desoMsg DeSoMessag return } - nm.usedNonces.Add(responseNonce, struct{}{}) + nm.usedNonces.Put(responseNonce) } // _handleVerackMessage is called when a new verack message is received. @@ -1248,7 +1247,7 @@ func (nm *NetworkManager) InitiateHandshake(rn *RemoteNode) { glog.Errorf("NetworkManager.InitiateHandshake: Error initiating handshake: %v", err) nm.Disconnect(rn, fmt.Sprintf("error initiating handshake: %v", err)) } - nm.usedNonces.Add(nonce, struct{}{}) + nm.usedNonces.Put(nonce) } // handleHandshakeComplete is called on a completed handshake with a RemoteNodes. diff --git a/lib/peer.go b/lib/peer.go index 27b56a8b5..1e32d8bf2 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -2,6 +2,7 @@ package lib import ( "fmt" + "github.com/deso-protocol/core/collections" "github.com/deso-protocol/go-deadlock" "net" "sort" @@ -11,7 +12,6 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/golang/glog" - "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -110,7 +110,7 @@ type Peer struct { // Inventory stuff. // The inventory that we know the peer already has. - knownInventory *lru.Cache[InvVect, struct{}] + knownInventory *collections.LruSet[InvVect] // Whether the peer is ready to receive INV messages. For a peer that // still needs a mempool download, this is false. @@ -291,7 +291,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { for _, invVect := range msg.InvList { // No matter what, add the inv to the peer's known inventory. - pp.knownInventory.Add(*invVect, struct{}{}) + pp.knownInventory.Put(*invVect) // If this is a hash we are currently processing, no need to do anything. // This check serves to fill the gap between the time when we've decided @@ -343,7 +343,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { // If we made it here, it means the inventory was added to one of the // lists so mark it as processed on the Server. - pp.srv.inventoryBeingProcessed.Add(*invVect, struct{}{}) + pp.srv.inventoryBeingProcessed.Put(*invVect) } // If there were any transactions we don't yet have, request them using @@ -644,7 +644,7 @@ func NewPeer(_id uint64, _conn net.Conn, _isOutbound bool, _netAddr *wire.NetAdd _syncType NodeSyncType, peerDisconnectedChan chan *Peer) *Peer { - knownInventoryCache, _ := lru.New[InvVect, struct{}](maxKnownInventory) + knownInventoryCache, _ := collections.NewLruSet[InvVect](maxKnownInventory) pp := Peer{ ID: _id, @@ -984,7 +984,7 @@ out: // Add the new inventory to the peer's knownInventory. for _, invVect := range invMsg.InvList { - pp.knownInventory.Add(*invVect, struct{}{}) + pp.knownInventory.Put(*invVect) } } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 1c959dde5..636b8b916 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -96,16 +96,6 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH bc.blockIndex.setHeaderTip(blockNode) - // The header is not an orphan and has a higher view than the current tip. We reorg the header chain - // and apply the incoming header as the new tip. - //_, blocksToDetach, blocksToAttach := bc.GetReorgBlocks(currentTip, blockNode) - //bc.bestHeaderChain.Chain, bc.bestHeaderChain.ChainMap = updateBestChainInMemory( - // bc.bestHeaderChain.Chain, - // bc.bestHeaderChain.ChainMap, - // blocksToDetach, - // blocksToAttach, - //) - // Success. The header is at the tip of the best header chain. return blockNode, true, false, nil } @@ -2023,7 +2013,7 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc utxoView.TipHash = &blockHash // Save a copy of the UtxoView to the cache. copiedView := utxoView.CopyUtxoView() - bc.blockViewCache.Add(blockHash, &BlockViewAndUtxoOps{ + bc.blockViewCache.Put(blockHash, &BlockViewAndUtxoOps{ UtxoView: copiedView, UtxoOps: utxoOps, Block: fullBlock, diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index d3ee5ac1c..2f8904cf0 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -253,7 +253,7 @@ func TestHasValidBlockHeight(t *testing.T) { ValidatorsTimeoutAggregateQC: nil, }, StatusBlockStored|StatusBlockValidated) bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{*genesisBlock.Hash: genesisBlock}) - bc.blockIndex.blockIndexByHash.Add(*genesisBlock.Hash, genesisBlock) + bc.blockIndex.blockIndexByHash.Put(*genesisBlock.Hash, genesisBlock) // Create a block with a valid header. randomPayload := RandomBytes(256) randomBLSPrivateKey := _generateRandomBLSPrivateKey(t) @@ -842,7 +842,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { PrevBlockHash: hash1, }, StatusBlockStored|StatusBlockValidated|StatusBlockCommitted) bc.blockIndex.setTip(block2) - bc.blockIndex.blockIndexByHash.Add(*hash2, block2) + bc.blockIndex.blockIndexByHash.Put(*hash2, block2) ancestors, missingBlockHashes, err = bc.getStoredLineageFromCommittedTip(block.Header) require.Error(t, err) require.Equal(t, err, RuleErrorDoesNotExtendCommittedTip) @@ -1320,9 +1320,9 @@ func TestTryApplyNewTip(t *testing.T) { bc.addTipBlockToBestChain(bn1) bc.addTipBlockToBestChain(bn2) bc.addTipBlockToBestChain(bn3) - bc.blockIndex.blockIndexByHash.Add(*hash1, bn1) - bc.blockIndex.blockIndexByHash.Add(*hash2, bn2) - bc.blockIndex.blockIndexByHash.Add(*hash3, bn3) + bc.blockIndex.blockIndexByHash.Put(*hash1, bn1) + bc.blockIndex.blockIndexByHash.Put(*hash2, bn2) + bc.blockIndex.blockIndexByHash.Put(*hash3, bn3) // Simple reorg. Just replacing the uncommitted tip. newBlock := &MsgDeSoBlock{ @@ -1407,8 +1407,8 @@ func TestTryApplyNewTip(t *testing.T) { Height: 6, }, } - bc.blockIndex.blockIndexByHash.Add(*hash4, bn4) - bc.blockIndex.blockIndexByHash.Add(*hash5, bn5) + bc.blockIndex.blockIndexByHash.Put(*hash4, bn4) + bc.blockIndex.blockIndexByHash.Put(*hash5, bn5) // Set new block's parent to hash5 newBlockNode.Header.PrevBlockHash = hash5 diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 5592553f0..7a17864c8 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -3,6 +3,7 @@ package lib import ( "bytes" "fmt" + "github.com/deso-protocol/core/collections" "path/filepath" "sync" "sync/atomic" @@ -10,7 +11,6 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/golang/glog" - "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -182,11 +182,11 @@ type PosMempool struct { // recentBlockTxnCache is an LRU KV cache used to track the transaction that have been included in blocks. // This cache is used to power logic that waits for a transaction to either be validated in the mempool // or be included in a block. - recentBlockTxnCache *lru.Cache[BlockHash, struct{}] + recentBlockTxnCache *collections.LruSet[BlockHash] // recentRejectedTxnCache is a cache to store the txns that were recently rejected so that we can return better // errors for them. - recentRejectedTxnCache *lru.Cache[BlockHash, error] + recentRejectedTxnCache *collections.LruCache[BlockHash, error] } func NewPosMempool() *PosMempool { @@ -233,8 +233,8 @@ func (mp *PosMempool) Init( mp.mempoolBackupIntervalMillis = mempoolBackupIntervalMillis mp.maxValidationViewConnects = maxValidationViewConnects mp.transactionValidationRefreshIntervalMillis = transactionValidationRefreshIntervalMillis - mp.recentBlockTxnCache, _ = lru.New[BlockHash, struct{}](100000) // cache 100K latest txns from blocks. - mp.recentRejectedTxnCache, _ = lru.New[BlockHash, error](100000) // cache 100K rejected txns. + mp.recentBlockTxnCache, _ = collections.NewLruSet[BlockHash](100000) // cache 100K latest txns from blocks. + mp.recentRejectedTxnCache, _ = collections.NewLruCache[BlockHash, error](100000) // cache 100K rejected txns. // Recreate and initialize the transaction register and the nonce tracker. mp.txnRegister = NewTransactionRegister() @@ -485,11 +485,11 @@ func (mp *PosMempool) AddTransaction(txn *MsgDeSoTxn, txnTimestamp time.Time) er } func (mp *PosMempool) addTxnHashToRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Add(txnHash, struct{}{}) + mp.recentBlockTxnCache.Put(txnHash) } func (mp *PosMempool) deleteTxnHashFromRecentBlockCache(txnHash BlockHash) { - mp.recentBlockTxnCache.Remove(txnHash) + mp.recentBlockTxnCache.Delete(txnHash) } func (mp *PosMempool) isTxnHashInRecentBlockCache(txnHash BlockHash) bool { return mp.recentBlockTxnCache.Contains(txnHash) @@ -846,7 +846,7 @@ func (mp *PosMempool) validateTransactions() error { // Mark the txn as invalid and add an error to the cache so we can return it to the user if they // try to resubmit it. txn.SetValidated(false) - mp.recentRejectedTxnCache.Add(*txn.Hash, err) + mp.recentRejectedTxnCache.Put(*txn.Hash, err) // Try to remove the transaction with a lock. mp.removeTransaction(txn, true) diff --git a/lib/postgres.go b/lib/postgres.go index 9c9ba432f..088752833 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/hashicorp/golang-lru/v2" + "github.com/deso-protocol/core/collections" "net/url" "regexp" "strings" @@ -1299,6 +1299,7 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { if blockNode.Parent != nil { block.ParentHash = blockNode.Parent.Hash } else if !blockNode.Header.PrevBlockHash.IsEqual(GenesisBlockHash) { + // TODO: LN - why did I need to add this? block.ParentHash = blockNode.Header.PrevBlockHash } @@ -1307,16 +1308,16 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // GetBlockIndex gets all the PGBlocks and creates a map of BlockHash to BlockNode as needed by blockchain.go -func (postgres *Postgres) GetBlockIndex() (*lru.Cache[BlockHash, *BlockNode], error) { +func (postgres *Postgres) GetBlockIndex() (*collections.LruCache[BlockHash, *BlockNode], error) { var blocks []PGBlock err := postgres.db.Model(&blocks).Select() if err != nil { return nil, err } - blockMap, _ := lru.New[BlockHash, *BlockNode](MaxBlockIndexNodes) + blockMap, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) for _, block := range blocks { - blockMap.Add(*block.Hash, &BlockNode{ + blockMap.Put(*block.Hash, &BlockNode{ Hash: block.Hash, Height: uint32(block.Height), DifficultyTarget: block.DifficultyTarget, diff --git a/lib/server.go b/lib/server.go index 2332d0f89..3458b42f0 100644 --- a/lib/server.go +++ b/lib/server.go @@ -22,7 +22,6 @@ import ( "github.com/deso-protocol/core/consensus" "github.com/deso-protocol/go-deadlock" "github.com/golang/glog" - "github.com/hashicorp/golang-lru/v2" "github.com/pkg/errors" ) @@ -88,7 +87,7 @@ type Server struct { // adding it to this map and checking this map before replying will make it // so that we only send a reply to the first peer that sent us the inv, which // is more efficient. - inventoryBeingProcessed *lru.Cache[InvVect, struct{}] + inventoryBeingProcessed *collections.LruSet[InvVect] // hasRequestedSync indicates whether we've bootstrapped our mempool // by requesting all mempool transactions from a @@ -227,7 +226,7 @@ func (srv *Server) _removeRequest(hash *BlockHash) { Type: InvTypeTx, Hash: *hash, } - srv.inventoryBeingProcessed.Remove(*invVect) + srv.inventoryBeingProcessed.Delete(*invVect) } // dataLock must be acquired for writing before calling this function. @@ -705,7 +704,7 @@ func NewServer( srv.blockProducer = _blockProducer srv.incomingMessages = _incomingMessages // Make this hold a multiple of what we hold for individual peers. - srv.inventoryBeingProcessed, _ = lru.New[InvVect, struct{}](maxKnownInventory) + srv.inventoryBeingProcessed, _ = collections.NewLruSet[InvVect](maxKnownInventory) srv.requestTimeoutSeconds = 10 @@ -1902,7 +1901,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { } // We also reset the in-memory snapshot cache, because it is populated with stale records after // we've initialized the chain with seed transactions. - srv.snapshot.DatabaseCache, _ = lru.New[string, []byte](int(DatabaseCacheSize)) + srv.snapshot.DatabaseCache, _ = collections.NewLruCache[string, []byte](int(DatabaseCacheSize)) // If we got here then we finished the snapshot sync so set appropriate flags. srv.blockchain.syncingState = false @@ -2192,7 +2191,7 @@ func (srv *Server) _relayTransactions() { // Add the transaction to the peer's known inventory. We do // it here when we enqueue the message to the peers outgoing // message queue so that we don't have to remember to do it later. - pp.knownInventory.Add(*invVect, struct{}{}) + pp.knownInventory.Put(*invVect) invMsg.InvList = append(invMsg.InvList, invVect) } if len(invMsg.InvList) > 0 { diff --git a/lib/snapshot.go b/lib/snapshot.go index 0c0c731ee..8276f03d3 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -5,6 +5,7 @@ import ( "context" "encoding/hex" "fmt" + "github.com/deso-protocol/core/collections" "math" "reflect" "runtime" @@ -18,7 +19,6 @@ import ( "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" - "github.com/hashicorp/golang-lru/v2" "github.com/oleiade/lane" "github.com/pkg/errors" "golang.org/x/sync/semaphore" @@ -313,7 +313,7 @@ type Snapshot struct { // DatabaseCache is used to store most recent DB records that we've read/written. // This is a low-level optimization for ancestral records that // saves us read time when we're writing to the DB during UtxoView flush. - DatabaseCache *lru.Cache[string, []byte] + DatabaseCache *collections.LruCache[string, []byte] // AncestralFlushCounter is used to offset ancestral records flush to occur only after x blocks. AncestralFlushCounter uint64 @@ -483,7 +483,7 @@ func NewSnapshot( "This may lead to unexpected behavior.") } - databaseCache, _ := lru.New[string, []byte](int(DatabaseCacheSize)) + databaseCache, _ := collections.NewLruCache[string, []byte](int(DatabaseCacheSize)) // Set the snapshot. snap := &Snapshot{ @@ -1409,7 +1409,7 @@ type StateChecksum struct { ctx context.Context // hashToCurveCache is a cache of computed hashToCurve mappings - hashToCurveCache *lru.Cache[string, group.Element] + hashToCurveCache *collections.LruCache[string, group.Element] // When we want to add a database record to the state checksum, we will first have to // map the record to the Ristretto255 curve using the hash_to_curve. We will then add the @@ -1437,7 +1437,7 @@ func (sc *StateChecksum) Initialize(mainDb *badger.DB, snapshotDbMutex *sync.Mut sc.maxWorkers = int64(runtime.GOMAXPROCS(0)) // Set the hashToCurveCache - sc.hashToCurveCache, _ = lru.New[string, group.Element](int(HashToCurveCache)) + sc.hashToCurveCache, _ = collections.NewLruCache[string, group.Element](int(HashToCurveCache)) // Set the worker pool semaphore and context. sc.semaphore = semaphore.NewWeighted(sc.maxWorkers) @@ -1508,7 +1508,7 @@ func (sc *StateChecksum) HashToCurve(bytes []byte) group.Element { // Compute the hash_to_curve primitive, mapping the bytes to an elliptic curve point. hashElement = sc.curve.HashToElement(bytes, sc.dst) // Also add to the hashToCurveCache - sc.hashToCurveCache.Add(bytesStr, hashElement) + sc.hashToCurveCache.Put(bytesStr, hashElement) } return hashElement diff --git a/lib/txindex.go b/lib/txindex.go index 1f5158c1b..cd3916f95 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -278,93 +278,6 @@ func (txi *TXIndex) Update() error { // For each of the blocks we're removing, delete the transactions from // the transaction index. - // TODO: delete - we're simplifying the txindex logic to only use committed state. - //for _, blockToDetach := range detachBlocks { - // if txi.killed { - // glog.Infof(CLog(Yellow, "TxIndex: Update: Killed while detaching blocks")) - // break - // } - // // Go through each txn in the block and delete its mappings from our - // // txindex. - // glog.V(1).Infof("Update: Detaching block (height: %d, hash: %v)", - // blockToDetach.Height, blockToDetach.Hash) - // blockMsg, err := GetBlock(blockToDetach.Hash, txi.TXIndexChain.DB(), nil) - // if err != nil { - // return fmt.Errorf("Update: Problem fetching detach block "+ - // "with hash %v: %v", blockToDetach.Hash, err) - // } - // blockHeight := uint64(txi.CoreChain.blockTip().Height) - // err = txi.TXIndexChain.DB().Update(func(dbTxn *badger.Txn) error { - // for _, txn := range blockMsg.Txns { - // if err := DbDeleteTxindexTransactionMappingsWithTxn(dbTxn, nil, - // blockHeight, txn, txi.Params, txi.CoreChain.eventManager, true); err != nil { - // - // return fmt.Errorf("Update: Problem deleting "+ - // "transaction mappings for transaction %v: %v", txn.Hash(), err) - // } - // } - // return nil - // }) - // if err != nil { - // return err - // } - // - // // Now that all the transactions have been deleted from our txindex, - // // it's safe to disconnect the block from our txindex chain. - // utxoView := NewUtxoView(txi.TXIndexChain.DB(), txi.Params, nil, nil, txi.CoreChain.eventManager) - // utxoOps, err := GetUtxoOperationsForBlock( - // txi.TXIndexChain.DB(), nil, blockToDetach.Hash) - // if err != nil { - // return fmt.Errorf( - // "Update: Error getting UtxoOps for block %v: %v", blockToDetach, err) - // } - // // Compute the hashes for all the transactions. - // txHashes, err := ComputeTransactionHashes(blockMsg.Txns) - // if err != nil { - // return fmt.Errorf( - // "Update: Error computing tx hashes for block %v: %v", - // blockToDetach, err) - // } - // if err := utxoView.DisconnectBlock(blockMsg, txHashes, utxoOps, blockHeight); err != nil { - // return fmt.Errorf("Update: Error detaching block "+ - // "%v from UtxoView: %v", blockToDetach, err) - // } - // if err := utxoView.FlushToDb(blockHeight); err != nil { - // return fmt.Errorf("Update: Error flushing view to db for block "+ - // "%v: %v", blockToDetach, err) - // } - // // We have to flush a couple of extra things that the view doesn't flush... - // if err := PutBestHash(txi.TXIndexChain.DB(), nil, utxoView.TipHash, ChainTypeDeSoBlock, txi.CoreChain.eventManager); err != nil { - // return fmt.Errorf("Update: Error putting best hash for block "+ - // "%v: %v", blockToDetach, err) - // } - // err = txi.TXIndexChain.DB().Update(func(txn *badger.Txn) error { - // if err := DeleteUtxoOperationsForBlockWithTxn(txn, nil, blockToDetach.Hash, txi.TXIndexChain.eventManager, true); err != nil { - // return fmt.Errorf("Update: Error deleting UtxoOperations 1 for block %v, %v", blockToDetach.Hash, err) - // } - // if err := txn.Delete(BlockHashToBlockKey(blockToDetach.Hash)); err != nil { - // return fmt.Errorf("Update: Error deleting UtxoOperations 2 for block %v %v", blockToDetach.Hash, err) - // } - // return nil - // }) - // - // if err != nil { - // return fmt.Errorf("Update: Error updating badgger: %v", err) - // } - // // Delete this block from the chain db so we don't get duplicate block errors. - // - // // Remove this block from our bestChain data structures. - // newBlockIndex := txi.TXIndexChain.CopyBlockIndexes() - // newTip := blockToDetach.GetParent(txi.TXIndexChain.blockIndex) - // if newTip == nil { - // return fmt.Errorf("Update: Error getting parent of block %v", blockToDetach) - // } - // - // txi.TXIndexChain.SetBestChainMap(newBlockIndex, newTip) - // - // // At this point the entries for the block should have been removed - // // from both our Txindex chain and our transaction index mappings. - //} // For each of the blocks we're adding, process them on our txindex chain // and add their mappings to our txn index. Compute any metadata that might From a9b549a3cdf3ef2ec5966e89c0d53f39d126bee2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 17:49:14 -0500 Subject: [PATCH 063/131] contains -> exists for lru cache --- collections/lru.go | 2 +- lib/db_utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collections/lru.go b/collections/lru.go index 7033541b3..891ef6bda 100644 --- a/collections/lru.go +++ b/collections/lru.go @@ -26,7 +26,7 @@ func (lruCache *LruCache[K, V]) Get(key K) (V, bool) { return lruCache.underlyingCache.Get(key) } -func (lruCache *LruCache[K, V]) Contains(key K) bool { +func (lruCache *LruCache[K, V]) Exists(key K) bool { return lruCache.underlyingCache.Contains(key) } diff --git a/lib/db_utils.go b/lib/db_utils.go index e1bcc248b..69048a8fd 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -2814,7 +2814,7 @@ func DBGetAccessGroupExistenceByAccessGroupIdWithTxn(txn *badger.Txn, snap *Snap // Lookup the snapshot cache and check if we've already stored a value there. if isState { - if exists := snap.DatabaseCache.Contains(keyString); exists { + if exists := snap.DatabaseCache.Exists(keyString); exists { return true, nil } } From 4caabaae4c3ee07fca1565189c5f21066d8f62e5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 19:27:51 -0500 Subject: [PATCH 064/131] add comments in blockchain.go --- lib/blockchain.go | 268 ++++++++++++++++++++++++++++++---------------- 1 file changed, 173 insertions(+), 95 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index a1b4f12d0..86ae5240b 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -121,12 +121,17 @@ func (nn *BlockNode) IsValidateFailed() bool { } // IsCommitted returns true if a BlockNode has passed all validations, and it has been committed to -// the Blockchain according to the Fast HotStuff commit rule. +// the Blockchain according to the Fast HotStuff commit rule. This function previously considered +// as PoW blocks to be committed regardless of status, but since we no longer keep the entire best +// chain in-memory, we needed to update all PoW blocks in the best chain to be Committed such that +// we could distinguish them from forks that occurred in PoW and still determine if a node is in the +// best chain or not. func (nn *BlockNode) IsCommitted() bool { - //return nn.Status&StatusBlockCommitted != 0 || !blockNodeProofOfStakeCutoverMigrationTriggered(nn.Height) return nn.Status&StatusBlockCommitted != 0 } +// ClearCommittedStatus is used to clear the committed status from a BlockNode. This is useful +// only for PoW blocks when handling forks. func (nn *BlockNode) ClearCommittedStatus() { nn.Status &= BlockStatus(^uint32(StatusBlockCommitted)) } @@ -245,7 +250,9 @@ func (nn *BlockNode) GetParent(blockIndex *BlockIndex) *BlockNode { if nn.Parent != nil { return nn.Parent } - // If we don't have a parent, try to get it from the block index. + // If we don't have a parent, try to get it from the block index. We + // no longer have a guarantee that we have set the parent node since + // we no longer keep the entire block index in memory. parentNode, exists := blockIndex.GetBlockNodeByHashAndHeight(nn.Header.PrevBlockHash, uint64(nn.Height-1)) if !exists { return nil @@ -449,15 +456,6 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode return node } -// RelativeAncestor returns the ancestor block node a relative 'distance' blocks -// before this node. This is equivalent to calling Ancestor with the node's -// height minus provided distance. -// -// This function is safe for concurrent access. -//func (nn *BlockNode) RelativeAncestor(distance uint32, blockIndex *BlockIndex) *BlockNode { -// return nn.Ancestor(nn.Height-distance, blockIndex) -//} - // CalcNextDifficultyTarget computes the difficulty target expected of the // next block. func (bc *Blockchain) CalcNextDifficultyTarget( @@ -497,9 +495,6 @@ func (bc *Blockchain) CalcNextDifficultyTarget( maxRetargetTimeSecs := targetSecs * bc.params.MaxDifficultyRetargetFactor firstNodeHeight := lastNode.Height - blocksPerRetarget - // TODO: we need to write the migration to only have committed blocks from PoW. - // This code is dead for PoS. - // TODO: do we need to do something if we need to get this from the header chain? firstNode, exists, err := bc.GetBlockFromBestChainByHeight(uint64(firstNodeHeight), true) if err != nil { return nil, errors.Wrapf(err, "CalcNextDifficultyTarget: Problem getting block at "+ @@ -567,6 +562,11 @@ type CheckpointBlockInfoAndError struct { Error error } +// BlockIndex operates as a read-through cache for block nodes. It is used to +// look up any block node we know by its height or hash (but is faster if you +// provide both!). It also is used to determine which block nodes are in the +// best chain using the provided functions. Additionally, it always tracks +// the block tip and header tip. type BlockIndex struct { db *badger.DB snapshot *Snapshot @@ -576,6 +576,7 @@ type BlockIndex struct { headerTip *BlockNode } +// NewBlockIndex creates a new BlockIndex with the provided snapshot and tip node. func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { blockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? blockIndexByHeight, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? @@ -588,6 +589,8 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block } } +// setBlockIndexFromMap is a helper function only used in tests. It constructs the +// block index from the provided map of block hashes to block nodes. func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { newHashToBlockNodeMap, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? newHeightToBlockNodeMap, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? @@ -605,42 +608,55 @@ func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { } } +// setHeaderTip sets the header tip of the block index to the provided block node. +// This must be called whenever we have a new tip added to the best header chain. func (bi *BlockIndex) setHeaderTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.headerTip = tip } +// setTip sets the tip of the block index to the provided block node. This must be +// called whenever we have a new tip added to the best chain. func (bi *BlockIndex) setTip(tip *BlockNode) { // Just to be safe, we also add it to the block index. bi.addNewBlockNodeToBlockIndex(tip) bi.tip = tip } +// addNewBlockNodeToBlockIndex adds a new block node to the block index. This adds +// it to both the block index by hash and the block index by height. func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) - if !exists { + if !exists || blocksAtHeight == nil { blocksAtHeight = []*BlockNode{} } else { - // Make sure we don't add the same block node twice. // TODO: we *could* make this more efficient by using a map, // but generally we won't have many blocks at the same height. - for ii, blockAtHeight := range blocksAtHeight { + // Make sure we don't add the same block node twice. + for _, blockAtHeight := range blocksAtHeight { if blockAtHeight.Hash.IsEqual(blockNode.Hash) { - blocksAtHeight[ii] = blockNode - break + // If we've already seen this block node, just + // return early. there's nothing left to do. + return } } } bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) } +// GetBlockNodeByHashOnly retrieves a block node from the block index by its hash. +// Note that this is the least efficient method of getting a block node from the block +// index if we don't have it in the cache as we need to first get the block height +// from the database and then get the block node from the database. func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, bool, error) { + // If we have it in the cache, just return it! val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true, nil } + // If we don't have it in the cache, we need to get the height from the database. height, err := GetHeightForHash(bi.db, bi.snapshot, blockHash) if err != nil { if errors.Is(err, badger.ErrKeyNotFound) { @@ -648,61 +664,82 @@ func (bi *BlockIndex) GetBlockNodeByHashOnly(blockHash *BlockHash) (*BlockNode, } return nil, false, errors.Wrapf(err, "GetBlockNodeByHashOnly: Problem getting height for hash") } + // Then we can get the block node from the database. blockNode := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if blockNode == nil { return nil, false, nil } + // Add it to the cache and return it. bi.addNewBlockNodeToBlockIndex(blockNode) return blockNode, true, nil } +// GetBlockNodeByHashAndHeight retrieves a block node from the block index by its hash +// height. This is the most efficient method of getting a block node from the block +// index as we do not need to get the block height from the database. func (bi *BlockIndex) GetBlockNodeByHashAndHeight(blockHash *BlockHash, height uint64) (*BlockNode, bool) { + // If we have it in the cache, just return it! val, exists := bi.blockIndexByHash.Get(*blockHash) if exists { return val, true } + // If the height exceeds an uint32, we have a problem as the height hash to node + // prefix only supports uint32 heights. if height > math.MaxUint32 { glog.Fatalf("GetBlockNodeByHashAndHeight: Height %d is greater than math.MaxUint32", height) } + // If we don't have it in the cache, we need to get the block node from the database. bn := GetHeightHashToNodeInfo(bi.db, bi.snapshot, uint32(height), blockHash, false) if bn == nil { return nil, false } + // Add it to the cache and return it. bi.addNewBlockNodeToBlockIndex(bn) return bn, true } +// GetBlockNodesByHeight retrieves all block nodes at a given height from the block index. +// We may have many block nodes at the same height, so the caller must determine which they +// want to use. func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { + // If the height exceeds an uint32, we have a problem as the height hash to node + // prefix only supports uint32 heights. if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } - //if height > bi.maxHeightSeen { - // return []*BlockNode{} - //} + // If we have it in the cache, just return it! + // TODO: maybe this isn't necessarily correct. We probably need to check the DB + // to be safe... blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) if exists { return blockNodesAtHeight } - // TODO: cache current height to exit early? + // If we don't have it in the cache, we need to get the block nodes from the database. + // We need to get the prefix key for the height hash to node prefix. prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) + // Enumerate all block nodes for the prefix. _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) blockNodes := []*BlockNode{} for _, val := range valsFound { + // Deserialize all block nodes. blockNode, err := DeserializeBlockNode(val) if err != nil { glog.Errorf("GetBlockNodesByHeight: Problem deserializing block node: %v", err) continue } + // Add them to the block index and append them to our result. bi.addNewBlockNodeToBlockIndex(blockNode) blockNodes = append(blockNodes, blockNode) } return blockNodes } +// GetTip retrieves the tip of the best chain. func (bi *BlockIndex) GetTip() *BlockNode { return bi.tip } +// GetHeaderTip retrieves the tip of the header chain. func (bi *BlockIndex) GetHeaderTip() *BlockNode { return bi.headerTip } @@ -733,7 +770,15 @@ type Blockchain struct { ChainLock deadlock.RWMutex // These should only be accessed after acquiring the ChainLock. - blockIndex *BlockIndex + // + // blockIndex is a read-through cache for block nodes. It is used to look up any block node we know + // by its height or hash (but is faster if you provide both!). It also is used to determine which + // block nodes are in the best chain using the provided functions. Additionally, it always tracks + // the block tip and header tip. + blockIndex *BlockIndex + // lowestBlockNotStored tracks the lowest known block height that is not stored in the database. + // This is only used for hypersync-archival syncing to determine at which height need to start + // requesting blocks in GetBlocksToStore. lowestBlockNotStored uint64 // We keep track of orphan blocks with the following data structures. Orphans @@ -872,14 +917,19 @@ func getCheckpointBlockInfoFromProviderHelper(provider string) *CheckpointBlockI } } +// addNewBlockNodeToBlockIndex is a pass-through to calling the same function on the +// BlockIndex of a Blockchain. func (bc *Blockchain) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bc.blockIndex.addNewBlockNodeToBlockIndex(blockNode) } +// CopyBlockIndexes returns a copy of the BlockIndex of a Blockchain. func (bc *Blockchain) CopyBlockIndexes() ( _blockIndexByHash *collections.LruCache[BlockHash, *BlockNode], ) { + // Create a new lru cache. newBlockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) + // Iterate over the keys of the block index and copy them to the new lru cache. for _, key := range bc.blockIndex.blockIndexByHash.Keys() { val, _ := bc.blockIndex.blockIndexByHash.Get(key) newBlockIndexByHash.Put(key, val) @@ -887,16 +937,22 @@ func (bc *Blockchain) CopyBlockIndexes() ( return newBlockIndexByHash } +// GetBlockIndex is a helper to return the BlockIndex of a Blockchain. +// This is used by the backend. func (bc *Blockchain) GetBlockIndex() *BlockIndex { return bc.blockIndex } -// TODO: read through to DB. +// getAllBlockNodesIndexedAtHeight returns all block nodes at a given height from the block index. func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { return bc.blockIndex.GetBlockNodesByHeight(blockHeight) } +// hasBlockNodesIndexedAtHeight returns whether or not there are block nodes at a given height in the block index. func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { + // TODO: there's an optimization we can do here where we just check if the + // cache has any block nodes at the height instead of loading everything + // from the database as well. blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) return len(blockNodes) > 0 } @@ -905,42 +961,54 @@ func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. chainState := bc.ChainState() - if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && - bc.headerTip().Height-bc.blockTip().Height < 10) { - // Get a sampling of blocks from the best chain and check if they are fully stored. - // We only need to check a few blocks to determine if the chain is fully stored. - blockTipHeight := uint64(bc.BlockTip().Height) - increment := blockTipHeight / 20 - if increment == 0 { - increment = 1 - } - blockHeights := []uint64{} - for ii := uint64(0); ii < blockTipHeight; ii += increment { + if chainState != SyncStateFullyCurrent && !(chainState == SyncStateNeedBlocksss && + bc.headerTip().Height-bc.blockTip().Height >= 10) { + return false + } + // Get a sampling of blocks from the best chain and check if they are fully stored. + // We only need to check a few blocks to determine if the chain is fully stored. + blockTipHeight := uint64(bc.BlockTip().Height) + increment := blockTipHeight / 20 + if increment == 0 { + increment = 1 + } + blockHeights := []uint64{} + // Sample 20 blocks. If we have 200 blocks, we'll check the following blocks: + // 0, 10, 20, 30, ..., 180, 190, 200. + for ii := uint64(0); ii < blockTipHeight; ii += increment { + blockHeights = append(blockHeights, ii) + } + // Check the most recent 20 blocks if we're past block 100. + if blockTipHeight > 100 { + // Continuing the above example, if we have 200 blocks, we'll add the following blocks: + // 180, 181, 182, 183, ..., 198, 199, 200. + // Note that it's not a big deal that we may have the same block twice, we dedupe them + // when we convert to a set. + for ii := blockTipHeight - 20; ii < blockTipHeight; ii++ { blockHeights = append(blockHeights, ii) } - if blockTipHeight > 100 { - for ii := blockTipHeight - 20; ii < blockTipHeight; ii++ { - blockHeights = append(blockHeights, ii) - } + } + // Always add the tip height to the sample. + blockHeights = append(blockHeights, blockTipHeight) + // Convert all block heights to a set to dedupe them. + blockHeightSet := NewSet(blockHeights) + for _, blockHeight := range blockHeightSet.ToSlice() { + // Get the block node at the height from the best blockchain chain. + blockNode, exists, err := bc.GetBlockFromBestChainByHeight(blockHeight, false) + if err != nil { + glog.Errorf("IsFullyStored: Problem getting block at height %d: %v", blockHeight, err) + return false } - blockHeights = append(blockHeights, blockTipHeight) - blockHeightSet := NewSet(blockHeights) - for _, blockHeight := range blockHeightSet.ToSlice() { - blockNode, exists, err := bc.GetBlockFromBestChainByHeight(blockHeight, false) - if err != nil { - glog.Errorf("IsFullyStored: Problem getting block at height %d: %v", blockHeight, err) - return false - } - if !exists { - return false - } - if !blockNode.Status.IsFullyProcessed() { - return false - } + // If the block node doesn't exist, the chain is not fully stored. + if !exists { + return false + } + // If the block node is not fully processed, the chain is not fully stored. + if !blockNode.Status.IsFullyProcessed() { + return false } - return true } - return false + return true } // _initChain initializes the in-memory data structures for the Blockchain object @@ -995,18 +1063,15 @@ func (bc *Blockchain) _initChain() error { // to previous blocks we've read in and error if they don't. This works because // reading blocks in height order as we do here ensures that we'll always // add a block's parents, if they exist, before adding the block itself. - //var err error - //if bc.postgres != nil { - // bc.blockIndexByHash, err = bc.postgres.GetBlockIndex() - //} else { - // bc.blockIndexByHash, err = GetBlockIndex(bc.db, false /*bitcoinNodes*/, bc.params) - //} - //if err != nil { - // return errors.Wrapf(err, "_initChain: Problem reading block index from db") - //} - //bc.blockIndexByHeight = bc.constructBlockIndexByHeight() - // For postgres, we still load the entire block index into memory. This is because + // Update: we no longer track the entire block index in memory. After the + // migration to Proof-of-Stake, we are producing blocks ~300x faster than before. + // It has become too memory intensive to keep the entire best chain and block index + // in memory. We now only keep track of the tip of the best header chain and best + // block chain. We still keep a cache of recently seen block nodes in order to speed + // up processing, but we use the BlockIndex struct as a read-through cache to look up + // blocks from the cache or database as needed. + var err error var tipNode *BlockNode if bc.postgres != nil { @@ -1019,7 +1084,7 @@ func (bc *Blockchain) _initChain() error { } else { var tipNodeExists bool // For badger, we only need the tip block to get started. - // Weird hack required for the genesis block. + // Special case for looking up the genesis block. if bestBlockHash.IsEqual(GenesisBlockHash) { tipNode, tipNodeExists = bc.blockIndex.GetBlockNodeByHashAndHeight(bestBlockHash, 0) } else { @@ -1038,6 +1103,8 @@ func (bc *Blockchain) _initChain() error { currBlockCounter++ } } + // Load all blocks greater than the tip node into the block index. This allows us to + // have quick access to all uncommitted blocks and orphans. if err = bc.blockIndex.LoadBlockIndexFromHeight(tipNode.Height, bc.params); err != nil { return errors.Wrapf(err, "_initChain: Problem loading block index from db") } @@ -1091,6 +1158,7 @@ func (bc *Blockchain) _applyUncommittedBlocksToBestChain() error { return errors.Wrapf(err, "_applyUncommittedBlocksToBestChain: ") } + // Set the tip of the best header chain and best block chain to the uncommitted tip. bc.blockIndex.setTip(uncommittedTipBlockNode) bc.blockIndex.setHeaderTip(uncommittedTipBlockNode) return nil @@ -1143,7 +1211,7 @@ func NewBlockchain( eventManager: eventManager, archivalMode: archivalMode, - blockIndex: NewBlockIndex(db, snapshot, nil), // TODO: replace with actual tip. + blockIndex: NewBlockIndex(db, snapshot, nil), // This tip will be set in _initChain. blockViewCache: blockViewCache, snapshotCache: NewSnapshotCache(), @@ -1276,27 +1344,19 @@ func (bc *Blockchain) GetBlockNodesToFetch( return blockNodesToFetch } +// HasHeader checks if a header exists in the block index given its hash. func (bc *Blockchain) HasHeader(headerHash *BlockHash) (bool, error) { _, exists, err := bc.blockIndex.GetBlockNodeByHashOnly(headerHash) return exists, errors.Wrap(err, "Blockchain.HasHeader: ") } +// HasHeaderByHashAndHeight checks if a header exists in the block index given its hash and height. func (bc *Blockchain) HasHeaderByHashAndHeight(headerHash *BlockHash, height uint64) bool { - if height > uint64(bc.headerTip().Height) { - return false - } _, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(headerHash, height) return exists } -// TODO: delete me? -func (bc *Blockchain) HeaderAtHeight(blockHeight uint32) (*BlockNode, bool, error) { - if blockHeight >= bc.blockIndex.GetHeaderTip().Height { - return nil, false, nil - } - return bc.GetBlockFromBestChainByHeight(uint64(blockHeight), true) -} - +// HasBlockInBlockIndex checks if a block exists in the block index given its hash. func (bc *Blockchain) HasBlockInBlockIndex(blockHash *BlockHash) (bool, error) { bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() @@ -1494,25 +1554,40 @@ func (bc *Blockchain) checkArchivalMode() bool { firstSnapshotHeight := bc.snapshot.CurrentEpochSnapshotMetadata.FirstSnapshotBlockHeight // @diamondhands - can we spot check just a few blocks such as firstSnapshotHeight - 1, // firstSnapshotHeight / 2 - 1, and firstSnapshotHeight / 4 - 1 to see if they are stored? + // And then we can do check some intervals between firstSnapshotHeight and the tip of the best chain. // We take a sampling of blocks to determine if we've downloaded all the blocks up to the first snapshot height. blockHeights := []uint64{} increment := firstSnapshotHeight / 10 + if increment == 0 { + increment = 1 + } + // We'll sample 10 blocks. If we have 200 as the first snapshot block height, + // we'll check the following blocks: + // 0, 20, 40, 60, ..., 180, 200. for ii := uint64(0); ii < firstSnapshotHeight; ii += increment { blockHeights = append(blockHeights, ii) } - for ii := firstSnapshotHeight - 10; ii < firstSnapshotHeight; ii++ { - blockHeights = append(blockHeights, ii) + // Check the most recent 10 blocks if we're past block 10. + if firstSnapshotHeight > 10 { + for ii := firstSnapshotHeight - 10; ii < firstSnapshotHeight; ii++ { + blockHeights = append(blockHeights, ii) + } } - blockHeights = append(blockHeights, firstSnapshotHeight) - for _, height := range blockHeights { + + // Convert to set to de-dupe and check the block node for each height. + for _, height := range NewSet(append(blockHeights, firstSnapshotHeight)).ToSlice() { blockNode, exists, err := bc.GetBlockFromBestChainByHeight(height, false) if err != nil { glog.Errorf("checkArchivalMode: Problem getting block by height: %v", err) return false } + // If the block node doesn't exist, we're not in for archival mode. if !exists { return false } + // If the block node is greater than the first snapshot height, + // we're not ready for archival mode. This shouldn't happen since + // the move to not having the entire best chain in memory. if uint64(blockNode.Height) > firstSnapshotHeight { return false } @@ -1606,6 +1681,9 @@ func (bc *Blockchain) BestChain() []*BlockNode { panic("BestChain not supported.") } +// GetBlockFromBestChainByHashAndOptionalHeight returns a block node from the best chain +// given a block hash and an optional height. Providing the height is optional, but it can +// greatly improve the performance of calls to this function. func (bc *Blockchain) GetBlockFromBestChainByHashAndOptionalHeight( blockHash *BlockHash, optionalHeight *uint64, @@ -1626,7 +1704,8 @@ func (bc *Blockchain) GetBlockFromBestChainByHashAndOptionalHeight( return nil, false, nil } if bn.IsCommitted() { - return bn, true, nil // TODO: what do we do about header chain? they're not committed so we're going to + return bn, true, nil + // TODO: what do we do about header chain? they're not committed so we're going to // have to get a bunch of parents in order to be sure it is part of the best header chain. I guess we could // have a map, but kinda defeats the purpose of this refactor. } @@ -1662,9 +1741,14 @@ func (bc *Blockchain) GetBlockFromBestChainByHashAndOptionalHeight( return nil, false, nil } +// GetBlockFromBestChainByHeight returns a block node from the best chain given a height. func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChain bool) (*BlockNode, bool, error) { // TODO: figure out an optimization for header chain handling uncommitted state. if !useHeaderChain { + // If we're looking for a block from the best block chain, we first + // check if the height is greater than the committed tip. If it is, we + // know it's an uncommitted block and will need to trace it back from + // the best block tip. committedTip, exists := bc.GetCommittedTip() if !exists { return nil, false, nil @@ -1684,6 +1768,7 @@ func (bc *Blockchain) GetBlockFromBestChainByHeight(height uint64, useHeaderChai return nil, false, nil } } + // Otherwise, get the block nodes by height and find the one in the best chain. blockNodes := bc.blockIndex.GetBlockNodesByHeight(height) if len(blockNodes) == 0 { return nil, false, nil @@ -1717,14 +1802,6 @@ func (bc *Blockchain) SetBestChain(bestChain []*BlockNode) { } } -func (bc *Blockchain) setBestChainMap( - blockIndexByHash *collections.LruCache[BlockHash, *BlockNode], - tipNode *BlockNode, -) { - bc.blockIndex.blockIndexByHash = blockIndexByHash - bc.blockIndex.setTip(tipNode) -} - func (bc *Blockchain) _validateOrphanBlockPoW(desoBlock *MsgDeSoBlock) error { // Error if the block is missing a parent hash or header. if desoBlock.Header == nil { @@ -1867,7 +1944,8 @@ func (bc *Blockchain) MarkBlockInvalid(node *BlockNode, errOccurred RuleError) { //} } -// Note: we make some assumptions that we only care about ancestors in the best chain. +// Note: we make some assumptions that we only care about ancestors in the best chain. This +// is only used for processBlockPoW so it's not a *huge* deal. func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *BlockNode { if node1 == nil || node2 == nil { // If either node is nil then there can't be a common ancestor. From 8a0bf0f5b56a099429a90f4af23d62edbddff6d2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 22:05:54 -0500 Subject: [PATCH 065/131] add more comments and cleanup. remove block index by height, just go to db for clarity --- lib/blockchain.go | 65 ++++++++------------------------------ lib/db_utils.go | 14 ++++++++ lib/pos_blockchain.go | 65 ++++++++++---------------------------- lib/pos_blockchain_test.go | 3 -- lib/server.go | 34 +++++--------------- 5 files changed, 52 insertions(+), 129 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 86ae5240b..b55bf5e3e 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -568,34 +568,29 @@ type CheckpointBlockInfoAndError struct { // best chain using the provided functions. Additionally, it always tracks // the block tip and header tip. type BlockIndex struct { - db *badger.DB - snapshot *Snapshot - blockIndexByHash *collections.LruCache[BlockHash, *BlockNode] - blockIndexByHeight *collections.LruCache[uint64, []*BlockNode] - tip *BlockNode - headerTip *BlockNode + db *badger.DB + snapshot *Snapshot + blockIndexByHash *collections.LruCache[BlockHash, *BlockNode] + tip *BlockNode + headerTip *BlockNode } // NewBlockIndex creates a new BlockIndex with the provided snapshot and tip node. func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *BlockIndex { - blockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? - blockIndexByHeight, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + blockIndexByHash, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? return &BlockIndex{ - db: db, - snapshot: snapshot, - blockIndexByHash: blockIndexByHash, - blockIndexByHeight: blockIndexByHeight, - tip: tipNode, + db: db, + snapshot: snapshot, + blockIndexByHash: blockIndexByHash, + tip: tipNode, } } // setBlockIndexFromMap is a helper function only used in tests. It constructs the // block index from the provided map of block hashes to block nodes. func (bi *BlockIndex) setBlockIndexFromMap(input map[BlockHash]*BlockNode) { - newHashToBlockNodeMap, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? - newHeightToBlockNodeMap, _ := collections.NewLruCache[uint64, []*BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? + newHashToBlockNodeMap, _ := collections.NewLruCache[BlockHash, *BlockNode](MaxBlockIndexNodes) // TODO: parameterize this? bi.blockIndexByHash = newHashToBlockNodeMap - bi.blockIndexByHeight = newHeightToBlockNodeMap for _, val := range input { bi.addNewBlockNodeToBlockIndex(val) // This function is always used for tests. @@ -628,22 +623,6 @@ func (bi *BlockIndex) setTip(tip *BlockNode) { // it to both the block index by hash and the block index by height. func (bi *BlockIndex) addNewBlockNodeToBlockIndex(blockNode *BlockNode) { bi.blockIndexByHash.Put(*blockNode.Hash, blockNode) - blocksAtHeight, exists := bi.blockIndexByHeight.Get(uint64(blockNode.Height)) - if !exists || blocksAtHeight == nil { - blocksAtHeight = []*BlockNode{} - } else { - // TODO: we *could* make this more efficient by using a map, - // but generally we won't have many blocks at the same height. - // Make sure we don't add the same block node twice. - for _, blockAtHeight := range blocksAtHeight { - if blockAtHeight.Hash.IsEqual(blockNode.Hash) { - // If we've already seen this block node, just - // return early. there's nothing left to do. - return - } - } - } - bi.blockIndexByHeight.Put(uint64(blockNode.Height), append(blocksAtHeight, blockNode)) } // GetBlockNodeByHashOnly retrieves a block node from the block index by its hash. @@ -707,15 +686,6 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { if height > math.MaxUint32 { glog.Fatalf("GetBlockNodesByHeight: Height %d is greater than math.MaxUint32", height) } - // If we have it in the cache, just return it! - // TODO: maybe this isn't necessarily correct. We probably need to check the DB - // to be safe... - blockNodesAtHeight, exists := bi.blockIndexByHeight.Get(height) - if exists { - return blockNodesAtHeight - } - // If we don't have it in the cache, we need to get the block nodes from the database. - // We need to get the prefix key for the height hash to node prefix. prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) // Enumerate all block nodes for the prefix. _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) @@ -943,18 +913,11 @@ func (bc *Blockchain) GetBlockIndex() *BlockIndex { return bc.blockIndex } -// getAllBlockNodesIndexedAtHeight returns all block nodes at a given height from the block index. -func (bc *Blockchain) getAllBlockNodesIndexedAtHeight(blockHeight uint64) []*BlockNode { - return bc.blockIndex.GetBlockNodesByHeight(blockHeight) -} - // hasBlockNodesIndexedAtHeight returns whether or not there are block nodes at a given height in the block index. func (bc *Blockchain) hasBlockNodesIndexedAtHeight(blockHeight uint64) bool { - // TODO: there's an optimization we can do here where we just check if the - // cache has any block nodes at the height instead of loading everything - // from the database as well. - blockNodes := bc.blockIndex.GetBlockNodesByHeight(blockHeight) - return len(blockNodes) > 0 + prefix := _heightHashToNodePrefixByHeight(uint32(blockHeight), false) + keysFound := EnumeratePaginatedLimitedKeysForPrefix(bc.db, prefix, prefix, 1) + return len(keysFound) > 0 } // IsFullyStored determines if there are block nodes that haven't been fully stored or processed in the best block chain. diff --git a/lib/db_utils.go b/lib/db_utils.go index 69048a8fd..680d1fedc 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -1585,6 +1585,20 @@ func _enumerateKeysOnlyForPrefixWithTxn(txn *badger.Txn, dbPrefix []byte) (_keys return _enumeratePaginatedLimitedKeysForPrefixWithTxn(txn, dbPrefix, dbPrefix, math.MaxUint32) } +func EnumeratePaginatedLimitedKeysForPrefix( + db *badger.DB, + dbPrefix []byte, + startKey []byte, + limit uint32, +) (_keysFound [][]byte) { + var keysFound [][]byte + _ = db.View(func(txn *badger.Txn) error { + keysFound = _enumeratePaginatedLimitedKeysForPrefixWithTxn(txn, dbPrefix, startKey, limit) + return nil + }) + return keysFound +} + // _enumeratePaginatedLimitedKeysForPrefixWithTxn will look for keys in the db that are GREATER OR EQUAL to the startKey // and satisfy the dbPrefix prefix. The total number of entries fetched will be EQUAL OR SMALLER than provided limit. func _enumeratePaginatedLimitedKeysForPrefixWithTxn(txn *badger.Txn, dbPrefix []byte, startKey []byte, limit uint32) (_keysFound [][]byte) { diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 636b8b916..fd963e259 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -50,10 +50,8 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH // If the incoming header is already part of the best header chain, then we can exit early. // The header is not part of a fork, and is already an ancestor of the current header chain tip. - // Here we explicitly check the bestHeaderChain.ChainMap to make sure the in-memory struct is properly - // updated. This is necessary because the block index may have been updated with the header but the - // bestHeaderChain.ChainMap may not have been updated yet. - blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight(headerHash, &header.Height, true) + blockNode, isInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight( + headerHash, &header.Height, true) if err != nil { return nil, false, false, errors.Wrapf(err, "processHeaderPoS: Problem getting block from best chain by hash: ") @@ -75,13 +73,6 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH return blockNode, false, false, errors.Wrapf(err, "processHeaderPoS: Problem validating and indexing header: ") } - // Don't worry about healing orphan children when we're syncing. - //if !bc.isSyncing() { - // // Now that we know we have a valid header, we check the block index for it any orphan children for it - // // and heal the parent pointers for all of them. - // bc.healPointersForOrphanChildren(blockNode) - //} - // Exit early if the header is an orphan. if isOrphan { return blockNode, false, true, nil @@ -100,38 +91,6 @@ func (bc *Blockchain) processHeaderPoS(header *MsgDeSoHeader, headerHash *BlockH return blockNode, true, false, nil } -// healPointersForOrphanChildren fixes an inconsistency in the block index that may have -// occurred as a result of a node restart. In cases where we have an orphan node that we store in the -// DB, then on restart, that node's parent will not be in the block index. When processing the parent -// later on, we not only need to store the parent in the block index but also need to update the -// pointer from the orphan block's BlockNode to the parent. We do that dynamically here as we -// process headers. -//func (bc *Blockchain) healPointersForOrphanChildren(blockNode *BlockNode) { -// // Fetch all potential children of this blockNode from the block index. -// blockNodesAtNextHeight := bc.blockIndex.GetBlockNodesByHeight(blockNode.Header.Height + 1) -// exists := len(blockNodesAtNextHeight) > 0 -// if !exists { -// // No children of this blockNode exist in the block index. Exit early. -// return -// } -// -// // Iterate through all block nodes at the next block height and update their parent pointers. -// for _, blockNodeAtNextHeight := range blockNodesAtNextHeight { -// // Check if it's a child of the parent block node. -// if !blockNodeAtNextHeight.Header.PrevBlockHash.IsEqual(blockNode.Hash) { -// continue -// } -// -// // Check if it has its parent pointer set. If it does, then we exit early. -// if blockNodeAtNextHeight.Parent != nil { -// continue -// } -// -// // If the parent block node is not set, then we set it to the parent block node. -// blockNodeAtNextHeight.Parent = blockNode -// } -//} - func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHash *BlockHash, verifySignatures bool) ( _headerBlockNode *BlockNode, _isOrphan bool, _err error, ) { @@ -155,8 +114,11 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas } // The header is an orphan. No need to store it in the block index. Exit early. - // TODO: validate that height - 1 > 0 - parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight(header.PrevBlockHash, header.Height-1) + if header.Height < 1 { + return nil, false, errors.New("validateAndIndexHeaderPoS: Header height is less than 1 - no valid parent height") + } + parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight( + header.PrevBlockHash, header.Height-1) if !parentBlockNodeExists { return nil, true, nil } @@ -308,6 +270,9 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v "processBlockPoS: Unexpected problem getting lineage from committed tip: ") } + if block.Header.Height < 1 { + return false, false, nil, errors.New("processBlockPoS: Block height is less than 1 - no valid parent height") + } // We expect the utxoView for the parent block to be valid because we check that all ancestor blocks have // been validated. parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) @@ -685,7 +650,6 @@ func (bc *Blockchain) validateAndIndexBlockPoS( ) (*BlockNode, error) { // Base case - Check if the block is validated or validate failed. If so, we can return early. - // TODO: validate height doesn't overflow uint32 blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) if exists && (blockNode.IsValidateFailed() || blockNode.IsValidated()) { return blockNode, nil @@ -813,6 +777,9 @@ func (bc *Blockchain) validatePreviouslyIndexedBlockPoS( // provided the block was cached in the block index and stored in the DB first. return nil, errors.Wrapf(err, "validatePreviouslyIndexedBlockPoS: Problem fetching block from DB") } + if block.Header.Height < 1 { + return nil, errors.New("processBlockPoS: Block height is less than 1 - no valid parent height") + } // Build utxoView for the block's parent. parentUtxoViewAndUtxoOps, err := bc.GetUtxoViewAndUtxoOpsAtBlockHash(*block.Header.PrevBlockHash, block.Header.Height-1) if err != nil { @@ -1659,7 +1626,6 @@ func (bc *Blockchain) addTipBlockToBestChain(blockNode *BlockNode) { func (bc *Blockchain) removeTipBlockFromBestChain() *BlockNode { // Remove the last block from the best chain. lastBlock := bc.blockIndex.GetTip() - // Uhhh what happens if we don't have the parent set up!? bc.blockIndex.setTip(lastBlock.GetParent(bc.blockIndex)) return lastBlock } @@ -2068,12 +2034,13 @@ func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { safeBlocks := []*BlockNode{committedTip} maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) for ii := uint64(committedTip.Height + 1); ii < maxHeightWithSafeBlocks+1; ii++ { + blockNodes := bc.blockIndex.GetBlockNodesByHeight(ii) // If we don't have any blocks at this height, we know that any blocks at a later height are not safe blocks. - if !bc.hasBlockNodesIndexedAtHeight(ii) { + if len(blockNodes) == 0 { break } hasSeenValidatedBlockAtThisHeight := false - blockNodes := bc.getAllBlockNodesIndexedAtHeight(ii) + for _, blockNode := range blockNodes { // TODO: Are there other conditions we should consider? if blockNode.IsValidated() { diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 2f8904cf0..81bf502aa 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -390,7 +390,6 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.Len(t, byHeightBlockNodes, 1) require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) - require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) // Check the DB for the block uncommittedBlock, err := GetBlock(newHash, bc.db, bc.snapshot) require.NoError(t, err) @@ -414,7 +413,6 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash)) require.True(t, byHeightBlockNodes[0].IsValidated()) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) - require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 1) // Okay now we'll put in another block at the same height. // Update the random seed hash so we have a new hash for the block. @@ -441,7 +439,6 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { require.True(t, byHeightBlockNodes[0].Hash.IsEqual(newHash) || byHeightBlockNodes[1].Hash.IsEqual(newHash)) require.True(t, byHeightBlockNodes[0].Hash.IsEqual(updatedBlockHash) || byHeightBlockNodes[1].Hash.IsEqual(updatedBlockHash)) require.True(t, bc.hasBlockNodesIndexedAtHeight(2)) - require.Len(t, bc.getAllBlockNodesIndexedAtHeight(2), 2) // If we're missing a field in the header, we should get an error // as we can't compute the hash. diff --git a/lib/server.go b/lib/server.go index 3458b42f0..1a010ee3c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -361,6 +361,8 @@ func ValidateHyperSyncFlags(isHypersync bool, syncType NodeSyncType) { } } +// RunBlockIndexMigrationOnce runs the block index migration once and saves a file to +// indicate that it has been run. func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { blockIndexMigrationFileName := filepath.Join(db.Opts().Dir, BlockIndexMigrationFileName) glog.V(2).Info("FileName: ", blockIndexMigrationFileName) @@ -771,16 +773,6 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { if pp.NegotiatedProtocolVersion >= ProtocolVersion2 { maxHeadersPerMsg = MaxHeadersPerMsgPos } - // FIXME: We can eliminate the call to LocateBestBlockChainHeaders and do a much - // simpler "shortcut" version that doesn't require complicated tree-traversal bs. - // The shortcut would be to just return all headers starting from msg.BlockLocator[0] - // up to msg.StopHash or maxHeadersPerMsg, whichever comes first. This would allow - // other nodes to sync from us and *keep* in sync with us, while allowing us to delete - // ALL of the complicated logic around locators and the best header chain. This all works - // because msg.BlockLocator[0] is the requesting-node's tip hash. The rest of the - // hashes, and all of the locator bs, are only needed to resolve forks, which can't - // happen with PoS anymore. - //headers := srv.blockchain.LocateBestBlockChainHeaders(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) headers, err := srv.GetHeadersForLocatorAndStopHash(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) if err != nil { @@ -799,6 +791,8 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { headers, blockTip.Hash, blockTip.Height, pp) } +// GetHeadersForLocatorAndStopHash returns a list of headers given a list of locator block hashes +// and a stop hash. Note that this may be slow if the block nodes requested are not in the cache. func (srv *Server) GetHeadersForLocatorAndStopHash( locator []*BlockHash, stopHash *BlockHash, @@ -1214,7 +1208,6 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // If we get here then we have a header we haven't seen before. // check if we need to verify signatures - // TODO: we can add some logic into should verify signatures to avoid trying to get the checkpoint block node. verifySignatures, shouldDisconnect := srv.shouldVerifySignatures(headerReceived, true) if shouldDisconnect { glog.Errorf("Server._handleHeaderBundle: Disconnecting peer %v in state %s because a mismatch was "+ @@ -1489,7 +1482,6 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) - // TODO: this may be wrong? glog.V(0).Infof("Server._handleHeaderBundle: Num Headers in header chain: (header tip height: %v) ", srv.blockchain.blockIndex.GetHeaderTip()) } @@ -1781,7 +1773,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { } if !exists { glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist: (%v)", msg.SnapshotMetadata.SnapshotBlockHeight) - //return + return } else { glog.Infof(CLog(Yellow, fmt.Sprintf("Best header chain %v best block chain %v", blockNode, srv.blockchain.blockIndex.GetTip()))) @@ -1836,21 +1828,11 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { }() // acquire the chain lock while we update the best chain and best chain map. srv.blockchain.ChainLock.Lock() - // TODO: we should iterate in reverse so we can use GetBlockFromBestChainByHashAndOptionalHeight - // by doing currentNode.Height - 1 and currentNode.Header.PrevBlockHash. - currentNode, currentNodeExists, err := srv.blockchain.GetBlockFromBestChainByHeight(srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight, true) - if err != nil { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, error: (%v)", err) - // TODO: should we return here? - } - if !currentNodeExists { - glog.Errorf("Server._handleSnapshot: Problem getting block node by height, block node does not exist") - // TODO: should we return here? - } + currentNode := blockNode + currentNodeExists := true // Set the block tip to the snapshot height block node. srv.blockchain.blockIndex.setTip(currentNode) for currentNode.Height > 0 { - //for ii := uint64(1); ii <= srv.HyperSyncProgress.SnapshotMetadata.SnapshotBlockHeight; ii++ { // Do not set the StatusBlockStored flag, because we still need to download the past blocks. currentNode.Status |= StatusBlockProcessed currentNode.Status |= StatusBlockValidated @@ -1876,7 +1858,7 @@ func (srv *Server) _handleSnapshot(pp *Peer, msg *MsgDeSoSnapshotData) { break } // TODO: should we adjust this value for batch sizes? - if len(blockNodeBatch) < 10000 { + if len(blockNodeBatch) < 25000 { continue } err = PutHeightHashToNodeInfoBatch(srv.blockchain.db, srv.snapshot, blockNodeBatch, false /*bitcoinNodes*/, srv.eventManager) From 7b311fd9cda0884aa84d660cb4571cbb78e949b5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 22:35:18 -0500 Subject: [PATCH 066/131] Fix tests and always flush PoW headers in process header pow so calc difficulty logic works --- lib/blockchain.go | 5 +++++ lib/blockchain_test.go | 28 ++++++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index b55bf5e3e..817540414 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2267,6 +2267,11 @@ func (bc *Blockchain) processHeaderPoW( bc.blockIndex.setHeaderTip(newNode) } + // Put the height hash to node info in the db. + if err = PutHeightHashToNodeInfo(bc.db, bc.snapshot, newNode, false /*bitcoinNodes*/, bc.eventManager); err != nil { + return nil, false, false, errors.Wrapf(err, "ProcessHeader: Problem calling PutHeightHashToNodeInfo") + } + return newNode, isMainChain, false, nil } diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index be4ffed0e..ef2a7d6e3 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1247,7 +1247,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), @@ -1255,7 +1255,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) - + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1285,7 +1285,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 4 second, which is 2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), @@ -1293,7 +1293,7 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) - + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1349,7 +1349,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), @@ -1357,7 +1357,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) - + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1387,7 +1387,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 8 second, which is >2x too slow. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 4)), @@ -1395,7 +1395,7 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) - + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1451,7 +1451,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating all at once. TstampNanoSecs: SecondsToNanoSeconds(0), @@ -1459,6 +1459,7 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1509,7 +1510,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 2 second, which is under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii * 2)), @@ -1517,6 +1518,7 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1568,7 +1570,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 1 second, which is 2x too fast. TstampNanoSecs: SecondsToNanoSeconds(int64(ii)), @@ -1576,6 +1578,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } @@ -1605,7 +1608,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, - nil, + big.NewInt(1), &MsgDeSoHeader{ // Blocks generating every 3 seconds, which is slow but under the limit. TstampNanoSecs: SecondsToNanoSeconds(int64(ii) * 3), @@ -1613,6 +1616,7 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { StatusHeaderValidated, )) bc.blockIndex.setHeaderTip(nodes[len(nodes)-1]) + require.NoError(PutHeightHashToNodeInfo(bc.db, nil, nodes[len(nodes)-1], false, nil)) diffsAsInts = append(diffsAsInts, HashToBigint(nextDiff).Int64()) } From 01abf3d98451daab827a4004c500601a0bc70681 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 9 Dec 2024 22:47:40 -0500 Subject: [PATCH 067/131] add comment about slowness --- lib/blockchain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/blockchain.go b/lib/blockchain.go index 817540414..442032132 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2193,6 +2193,8 @@ func (bc *Blockchain) processHeaderPoW( return nil, false, false, HeaderErrorTimestampTooEarly } + // @diamondhands - this check is significantly slower because we need to go + // to the DB to get a block at a specific height without knowledge of its hash. // Check that the proof of work beats the difficulty as calculated from // the parent block. Note that if the parent block is in the block index // then it has necessarily had its difficulty validated, and so using it to From 091ad57daff69c2b03abbc89633d1352ae2ea598 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 10 Dec 2024 13:26:40 -0500 Subject: [PATCH 068/131] write faster GetHeadersForLocatorAndStopHash --- lib/server.go | 55 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/lib/server.go b/lib/server.go index 1a010ee3c..81369349b 100644 --- a/lib/server.go +++ b/lib/server.go @@ -774,7 +774,7 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { maxHeadersPerMsg = MaxHeadersPerMsgPos } - headers, err := srv.GetHeadersForLocatorAndStopHash(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + headers, err := srv.GetHeadersForLocatorAndStopHash2(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) if err != nil { glog.Errorf("Server._handleGetHeadersMessage: Error getting headers: %v", err) } @@ -812,26 +812,49 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( if startNodeError != nil || !startNodeExists || startNode == nil { return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") } - nextNodeHeight := startNode.Header.Height + 1 - nextNode, nextNodeExists, nextNodeError := srv.blockchain.GetBlockFromBestChainByHeight(nextNodeHeight, true) - if nextNodeError != nil { - return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Error getting start node by height: %v", startNodeError) - } - if !nextNodeExists || nextNode == nil { - return nil, nil + var backtrackingNode *BlockNode + var backtrackingNodeExists bool + var backtrackingNodeError error + // If the stop node isn't provided and the max header msgs would put us past the header tip, + // we use the header tip to start back tracking. + if stopNode == nil && srv.blockchain.HeaderTip().Height < startNode.Height+maxHeadersPerMsg { + backtrackingNode = srv.blockchain.HeaderTip() + backtrackingNodeExists = true + } else if stopNode == nil || stopNode.Height > startNode.Height+maxHeadersPerMsg { + // If the stop node isn't provided or the stop node is more than maxHeadersPerMsg away + // from the start node, we compute the height of the last header expected and start + // back tracking from there. + backtrackingNode, backtrackingNodeExists, backtrackingNodeError = srv.blockchain.GetBlockFromBestChainByHeight( + uint64(startNode.Height+maxHeadersPerMsg), true) + if backtrackingNodeError != nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Error getting backtracking node by height: %v", backtrackingNodeError) + } + if !backtrackingNodeExists || backtrackingNode == nil { + return nil, errors.New("GetHeadersForLocatorAndStopHash: Backtracking node not found") + } + } else { + // Otherwise, the stop node is provided and we start back tracking from the stop node. + backtrackingNode = stopNode } for ii := uint32(0); ii < maxHeadersPerMsg; ii++ { - headers = append(headers, nextNode.Header) - if stopNode != nil && nextNode.Hash.IsEqual(stopNode.Hash) { + // If we've back tracked all the way to the start node, exit. + if backtrackingNode.Hash.IsEqual(startNode.Hash) { + break + } + headers = append([]*MsgDeSoHeader{backtrackingNode.Header}, headers...) + // Avoid underflow. + if backtrackingNode.Height < 1 { break } - nextNode, nextNodeExists, nextNodeError = srv.blockchain.GetBlockFromBestChainByHeight( - nextNode.Header.Height+1, true) - if nextNodeError != nil { - glog.Errorf("Server._handleGetHeadersMessage: Error getting next node by height: %v", nextNodeError) + prevNodeHeight := backtrackingNode.Header.Height - 1 + backtrackingNode, backtrackingNodeExists, backtrackingNodeError = srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight( + backtrackingNode.Header.PrevBlockHash, + &prevNodeHeight, true) + if backtrackingNodeError != nil { + glog.Errorf("Server._handleGetHeadersMessage: Error getting prev node by height: %v", backtrackingNodeError) break } - if !nextNodeExists || nextNode == nil { + if !backtrackingNodeExists || backtrackingNode == nil { break } } @@ -2148,7 +2171,7 @@ func (srv *Server) _relayTransactions() { for _, pp := range allPeers { if !pp.canReceiveInvMessages { - glog.V(1).Infof("Skipping invs for peer %v because not ready "+ + glog.V(2).Infof("Skipping invs for peer %v because not ready "+ "yet: %v", pp, pp.canReceiveInvMessages) continue } From 9ed19df924afd0fe4802306eff3fe32a2c933ded Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 10 Dec 2024 18:51:23 -0500 Subject: [PATCH 069/131] fix typo --- lib/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server.go b/lib/server.go index 81369349b..c3269c875 100644 --- a/lib/server.go +++ b/lib/server.go @@ -774,7 +774,7 @@ func (srv *Server) _handleGetHeaders(pp *Peer, msg *MsgDeSoGetHeaders) { maxHeadersPerMsg = MaxHeadersPerMsgPos } - headers, err := srv.GetHeadersForLocatorAndStopHash2(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) + headers, err := srv.GetHeadersForLocatorAndStopHash(msg.BlockLocator, msg.StopHash, maxHeadersPerMsg) if err != nil { glog.Errorf("Server._handleGetHeadersMessage: Error getting headers: %v", err) } From 969cb1c49f41c7a68dfa8c9263d582e00f348fa1 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 11 Dec 2024 19:18:11 -0500 Subject: [PATCH 070/131] include committed tip in header locator, fix walkback to committed tip in get lineage, fix get block noders to fetch to walk back on the best chain --- lib/blockchain.go | 110 +++++++++++++++++++----------------------- lib/pos_blockchain.go | 25 +++++++--- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 442032132..9ef607518 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -12,6 +12,7 @@ import ( "net/http" "reflect" "runtime/debug" + "slices" "sort" "strings" "sync" @@ -1215,15 +1216,20 @@ func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { bc.ChainLock.RLock() defer bc.ChainLock.RUnlock() headerTip := bc.headerTip() + committedTip, _ := bc.GetCommittedTip() - return []*BlockHash{headerTip.Hash} + return []*BlockHash{headerTip.Hash, committedTip.Hash} } func (bc *Blockchain) GetBlockNodesToFetch( numBlocks int, _maxHeight int, blocksToIgnore map[BlockHash]bool) []*BlockNode { // Get the tip of the main block chain. - bestBlockTip := bc.blockTip() + bestBlockTip, ok := bc.GetCommittedTip() + if !ok { + glog.Errorf("GetBlockToFetch: Problem getting best block tip") + return nil + } // If the maxHeight is set to < 0, then we don't want to use it as a constraint. maxHeight := uint32(math.MaxUint32) @@ -1231,78 +1237,62 @@ func (bc *Blockchain) GetBlockNodesToFetch( maxHeight = uint32(_maxHeight) } - // If the tip of the best block chain is in the main header chain, make that - // the start point for our fetch. - bestBlockTipHeight := uint64(bestBlockTip.Height) - headerNodeStart, blockTipExistsInBestHeaderChain, err := bc.GetBlockFromBestChainByHashAndOptionalHeight( - bestBlockTip.Hash, &bestBlockTipHeight, true) - if err != nil { - glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) - return nil + // Find the height of the node we want to back track from. + // If the max height is greater than the current header tip, + // we should use the header tip. + // If the header tip is still greater than bestBlockTip.Height + numBlocks, + // then we need to reduce the height limit. + blockNodesToFetch := []*BlockNode{} + heightLimit := uint64(maxHeight) + if heightLimit > bc.blockIndex.GetHeaderTip().Header.Height { + heightLimit = bc.blockIndex.GetHeaderTip().Header.Height } - if !blockTipExistsInBestHeaderChain { - // If the hash of the tip of the best blockchain is not in the best header chain, then - // this is a case where the header chain has forked off from the best block - // chain. In this situation, the best header chain is taken as the source of truth - // and so we iterate backward over the best header chain starting at the tip - // until we find the first block that has StatusBlockProcessed. Then we fetch - // blocks starting from there. Note that, at minimum, the genesis block has - // StatusBlockProcessed so this loop is guaranteed to terminate successfully. - headerNodeStart = bc.headerTip() - for headerNodeStart != nil && (headerNodeStart.Status&StatusBlockProcessed) == 0 { - headerNodeStart = headerNodeStart.GetParent(bc.blockIndex) - } - - if headerNodeStart == nil { - // If for some reason we ended up with the headerNode being nil, log - // an error and set it to the genesis block. - glog.Errorf("GetBlockToFetch: headerNode was nil after iterating " + - "backward through best header chain; using genesis block") - var err error - var genesisBlockExists bool - headerNodeStart, genesisBlockExists, err = bc.GetBlockFromBestChainByHeight(0, true) - if err != nil { - glog.Errorf("GetBlockToFetch: Problem getting genesis block: %v", err) - return nil - } - if !genesisBlockExists { - glog.Errorf("GetBlockToFetch: Genesis block not found") - return nil - } - } + if heightLimit > bestBlockTip.Header.Height+uint64(numBlocks) { + heightLimit = bestBlockTip.Header.Height + uint64(numBlocks) } - - // At this point, headerNodeStart should point to a node in the best header - // chain that has StatusBlockProcessed set. As such, the blocks we need to - // fetch are those right after this one. Fetch the desired number. - currentHeight := headerNodeStart.Height + 1 - blockNodesToFetch := []*BlockNode{} - heightLimit := maxHeight - if heightLimit >= bc.blockIndex.GetHeaderTip().Height { - heightLimit = bc.blockIndex.GetHeaderTip().Height - 1 + currentHeight := heightLimit + maxNode, maxNodeExists, maxNodeError := bc.GetBlockFromBestChainByHeight(currentHeight, true) + if maxNodeError != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", maxNodeError) + return nil } - for currentHeight <= heightLimit && - len(blockNodesToFetch) < numBlocks { - - // Get the current hash and increment the height. - currentNode, currentNodeExists, err := bc.GetBlockFromBestChainByHeight(uint64(currentHeight), true) - if err != nil { - glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", err) + if !maxNodeExists || maxNode == nil { + glog.Errorf("GetBlockToFetch: Block at height %d not found", heightLimit) + return nil + } + currentHash := maxNode.Hash + // Walk back from the maxNode to the bestBlockTip. + for len(blockNodesToFetch) < numBlocks && + currentHeight >= uint64(bestBlockTip.Height) { + backtrackingNode, backtrackingNodeExists, backtrackingNodeError := + bc.GetBlockFromBestChainByHashAndOptionalHeight(currentHash, ¤tHeight, true) + if backtrackingNodeError != nil { + glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", backtrackingNodeError) return nil } - if !currentNodeExists { + if !backtrackingNodeExists || backtrackingNode == nil { glog.Errorf("GetBlockToFetch: Block at height %d not found", currentHeight) return nil } - currentHeight++ - if _, exists := blocksToIgnore[*currentNode.Hash]; exists { + currentHeight-- + currentHash = backtrackingNode.Header.PrevBlockHash + + // Exclude any blocks we're supposed to ignore. + if _, exists := blocksToIgnore[*backtrackingNode.Hash]; exists { + continue + } + + // Skip any stored blocks as we already have them. + if backtrackingNode.IsStored() { continue } - blockNodesToFetch = append(blockNodesToFetch, currentNode) + blockNodesToFetch = append(blockNodesToFetch, backtrackingNode) } + slices.Reverse(blockNodesToFetch) + // Return the nodes for the blocks we should fetch. return blockNodesToFetch } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index fd963e259..95655d501 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -652,6 +652,14 @@ func (bc *Blockchain) validateAndIndexBlockPoS( // Base case - Check if the block is validated or validate failed. If so, we can return early. blockNode, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, block.Header.Height) if exists && (blockNode.IsValidateFailed() || blockNode.IsValidated()) { + // If the block isn't stored, we store it now. + if !blockNode.IsStored() { + var err error + blockNode, err = bc.storeBlockInBlockIndex(block, blockHash) + if err != nil { + return nil, errors.Wrap(err, "validateAndIndexBlockPoS: Problem storing block to block index") + } + } return blockNode, nil } @@ -1295,8 +1303,8 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( currentHash := header.PrevBlockHash.NewBlockHash() currentHeight := header.Height - 1 ancestors := []*BlockNode{} - prevHeight := header.Height - prevView := header.GetView() + childHeight := header.Height + childView := header.GetView() for { // TODO: is currentHeight correct here? currentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) @@ -1312,10 +1320,10 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( if currentBlock.IsValidateFailed() { return nil, nil, RuleErrorAncestorBlockValidationFailed } - if uint64(currentBlock.Header.Height)+1 != prevHeight { + if uint64(currentBlock.Header.Height)+1 != childHeight { return nil, nil, RuleErrorParentBlockHeightNotSequentialWithChildBlockHeight } - if currentBlock.Header.GetView() >= prevView { + if currentBlock.Header.GetView() >= childView { return nil, nil, RuleErrorParentBlockHasViewGreaterOrEqualToChildBlock } @@ -1329,8 +1337,9 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( ancestors = append(ancestors, currentBlock) currentHash = currentBlock.Header.PrevBlockHash - prevHeight = currentBlock.Header.Height - prevView = currentBlock.Header.GetView() + currentHeight = currentBlock.Header.Height - 1 + childHeight = currentBlock.Header.Height + childView = currentBlock.Header.GetView() } return collections.Reverse(ancestors), nil, nil } @@ -1610,6 +1619,10 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool if chainTip.Hash.IsEqual(blockNode.Header.PrevBlockHash) { return false } + // If the block is proposed in a view less than the current tip, there's no need to reorg. + if blockNode.Header.ProposedInView < chainTip.Header.ProposedInView { + return false + } // If the block is proposed in a view less than the current view, there's no need to reorg. return blockNode.Header.ProposedInView >= currentView } From a8384cbfb061808c3f5f395122676e53270db62b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 13:40:30 -0500 Subject: [PATCH 071/131] adjust log level for highest committed block in GetUtxoViewandUtxoOpsAtBlockHash --- lib/pos_blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 95655d501..c846f5712 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1929,7 +1929,7 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc } highestCommittedBlock, exists := bc.GetCommittedTip() - glog.Infof("Highest committed block: %v", highestCommittedBlock) + glog.V(4).Infof("Highest committed block: %v", highestCommittedBlock) if !exists || highestCommittedBlock == nil { return nil, errors.Errorf("GetUtxoViewAndUtxoOpsAtBlockHash: No committed blocks found") } From 8aa423e02d1cf5489fb56b2a73eb81ed9cfbd527 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 14:25:22 -0500 Subject: [PATCH 072/131] clean up comments --- lib/blockchain.go | 1 + lib/server.go | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 9ef607518..37c1fe95c 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -584,6 +584,7 @@ func NewBlockIndex(db *badger.DB, snapshot *Snapshot, tipNode *BlockNode) *Block snapshot: snapshot, blockIndexByHash: blockIndexByHash, tip: tipNode, + headerTip: tipNode, } } diff --git a/lib/server.go b/lib/server.go index c3269c875..fa067edee 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1126,9 +1126,6 @@ func (srv *Server) shouldVerifySignatures(header *MsgDeSoHeader, isHeaderChain b if checkpointBlockInfo == nil { return true, false } - // TODO: @diamondhands - why can't we move this up in this function? It seems like we can avoid - // checking if we have the checkpoint block node if the header we're processing is below the height. - // This will save us 17-18% of the time it takes to process headers. // If the current header has a height below the checkpoint block height, we should skip signature verification // even if we've seen the checkpoint block hash. if header.Height < checkpointBlockInfo.Height { From e6aeff6ac0c68e523beda9e81b3d9c58ab875038 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 16:27:53 -0500 Subject: [PATCH 073/131] remove load block index from height function. it's not necessary --- lib/blockchain.go | 5 ----- lib/db_utils.go | 57 ----------------------------------------------- 2 files changed, 62 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 37c1fe95c..b9f12fcc4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1068,11 +1068,6 @@ func (bc *Blockchain) _initChain() error { currBlockCounter++ } } - // Load all blocks greater than the tip node into the block index. This allows us to - // have quick access to all uncommitted blocks and orphans. - if err = bc.blockIndex.LoadBlockIndexFromHeight(tipNode.Height, bc.params); err != nil { - return errors.Wrapf(err, "_initChain: Problem loading block index from db") - } // We start by simply setting the chain tip and header tip to the tip node. bc.blockIndex.setTip(tipNode) diff --git a/lib/db_utils.go b/lib/db_utils.go index 680d1fedc..864a862c9 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5622,63 +5622,6 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( return blockIndex, nil } -// LoadBlockIndexFromHeight loads the block index from the database starting at -// a given height. This is only used by initChain, where we want to load all blocks -// that could be descendents of the current tip block in the database. -func (bi *BlockIndex) LoadBlockIndexFromHeight(height uint32, params *DeSoParams) error { - // Get the prefix for the provided height. - prefix := _heightHashToNodePrefixByHeight(height, false) - - return bi.db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - nodeIterator := txn.NewIterator(opts) - defer nodeIterator.Close() - for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { - var blockNode *BlockNode - - // Don't bother checking the key. We assume that the key lines up - // with what we've stored in the value in terms of (height, block hash). - item := nodeIterator.Item() - err := item.Value(func(blockNodeBytes []byte) error { - // Deserialize the block node. - var err error - blockNode, err = DeserializeBlockNode(blockNodeBytes) - if err != nil { - return err - } - return nil - }) - if err != nil { - return err - } - - // If we got here it means we read a blockNode successfully. Store it - // into our node index. - bi.addNewBlockNodeToBlockIndex(blockNode) - - // Find the parent of this block, which should already have been read - // in and connect it. Skip the genesis block, which has height 0. - if blockNode.Height == 0 || (*blockNode.Header.PrevBlockHash == BlockHash{}) { - continue - } - if parent, ok := bi.GetBlockNodeByHashAndHeight(blockNode.Header.PrevBlockHash, uint64(blockNode.Height)); ok { - // We found the parent node so connect it. - blockNode.Parent = parent - continue - } - // If we're syncing a DeSo node and we hit a PoS block, we expect there to - // be orphan blocks in the block index. In this case, we don't throw an error. - if params.IsPoSBlockHeight(uint64(blockNode.Height)) { - continue - } - // In this case we didn't find the parent so error. There shouldn't - // be any unconnectedTxns in our block index. - return fmt.Errorf("GetBlockIndex: Could not find parent for blockNode: %+v", blockNode) - } - return nil - }) -} - // RunBlockIndexMigration runs a migration to populate the hash to height index from the height hash to // block node index. We can't use the encoder migrations to handle this situation since it's a new index // and not a modification of the existing entry type stored. This migration simply iterates over the keys in the From 8ad81559b4f8ab170f0eb2ad5495f4d96c4e2d4b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 16:32:16 -0500 Subject: [PATCH 074/131] fix TestShouldReorg --- lib/pos_blockchain_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 81bf502aa..42a7afeef 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1234,11 +1234,17 @@ func TestShouldReorg(t *testing.T) { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, Height: 0, + Header: &MsgDeSoHeader{ + ProposedInView: 0, + }, }, { Hash: hash3, Status: StatusBlockStored | StatusBlockValidated, Height: 1, + Header: &MsgDeSoHeader{ + ProposedInView: 1, + }, }, } bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ From 86568731acfac37bb3a2fac4cfa8e346f86d3d53 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 17:06:15 -0500 Subject: [PATCH 075/131] revert change to should reorg --- lib/blockchain.go | 1 + lib/pos_blockchain.go | 4 ---- lib/pos_blockchain_test.go | 6 ------ 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index b9f12fcc4..71abbf143 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -2298,6 +2298,7 @@ func (bc *Blockchain) ProcessBlock(desoBlock *MsgDeSoBlock, verifySignatures boo // If the block's height is after the PoS cut-over fork height, then we use the PoS block processing logic. // Otherwise, fall back to the PoW logic. if bc.params.IsPoSBlockHeight(desoBlock.Header.Height) { + // TODO: can we pass some legit view number in here for non-validator nodes? return bc.processBlockPoS(desoBlock, 1, verifySignatures) } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index c846f5712..bfd6ef9fa 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1619,10 +1619,6 @@ func (bc *Blockchain) shouldReorg(blockNode *BlockNode, currentView uint64) bool if chainTip.Hash.IsEqual(blockNode.Header.PrevBlockHash) { return false } - // If the block is proposed in a view less than the current tip, there's no need to reorg. - if blockNode.Header.ProposedInView < chainTip.Header.ProposedInView { - return false - } // If the block is proposed in a view less than the current view, there's no need to reorg. return blockNode.Header.ProposedInView >= currentView } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 42a7afeef..81bf502aa 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -1234,17 +1234,11 @@ func TestShouldReorg(t *testing.T) { Hash: hash1, Status: StatusBlockStored | StatusBlockValidated | StatusBlockCommitted, Height: 0, - Header: &MsgDeSoHeader{ - ProposedInView: 0, - }, }, { Hash: hash3, Status: StatusBlockStored | StatusBlockValidated, Height: 1, - Header: &MsgDeSoHeader{ - ProposedInView: 1, - }, }, } bc.blockIndex.setBlockIndexFromMap(map[BlockHash]*BlockNode{ From 5f00b2617bd134575cdcd9e3db9bb5400fa25977 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 12 Dec 2024 17:51:43 -0500 Subject: [PATCH 076/131] only compute BMF if we're passed PoS block height --- lib/block_view.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 3a8eacb12..6d937082d 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -4416,9 +4416,6 @@ func (bav *UtxoView) ConnectBlock( if includeFeesInBlockReward { if txn.TxnMeta.GetTxnType() != TxnTypeAtomicTxnsWrapper { - // Compute the BMF given the current fees paid in the block. - _, utilityFee = computeBMF(currentFees) - // Add the fees from this txn to the total fees. If any overflow occurs // mark the block as invalid and return a rule error. Note that block reward // txns should count as having zero fees. @@ -4426,14 +4423,19 @@ func (bav *UtxoView) ConnectBlock( return nil, RuleErrorTxnOutputWithInvalidAmount } totalFees += currentFees - - // For PoS, the maximum block reward is based on the maximum utility fee. - // Add the utility fees to the max utility fees. If any overflow - // occurs mark the block as invalid and return a rule error. - maxUtilityFee, err = SafeUint64().Add(maxUtilityFee, utilityFee) - if err != nil { - return nil, errors.Wrapf(RuleErrorPoSBlockRewardWithInvalidAmount, - "ConnectBlock: error computing maxUtilityFee: %v", err) + // Only compute BMF if we're passed the PoS cutover. + if blockHeight >= uint64(bav.Params.ForkHeights.ProofOfStake2ConsensusCutoverBlockHeight) { + // Compute the BMF given the current fees paid in the block. + _, utilityFee = computeBMF(currentFees) + + // For PoS, the maximum block reward is based on the maximum utility fee. + // Add the utility fees to the max utility fees. If any overflow + // occurs mark the block as invalid and return a rule error. + maxUtilityFee, err = SafeUint64().Add(maxUtilityFee, utilityFee) + if err != nil { + return nil, errors.Wrapf(RuleErrorPoSBlockRewardWithInvalidAmount, + "ConnectBlock: error computing maxUtilityFee: %v", err) + } } } else { txnMeta, ok := txn.TxnMeta.(*AtomicTxnsWrapperMetadata) @@ -4450,11 +4452,14 @@ func (bav *UtxoView) ConnectBlock( return nil, errors.Wrap( err, "ConnectBlock: error adding non-block-reward recipient fees from atomic transaction") } - _, utilityFee = computeBMF(nonBlockRewardRecipientFees) - maxUtilityFee, err = SafeUint64().Add(maxUtilityFee, utilityFee) - if err != nil { - return nil, errors.Wrap(err, - "ConnectBlock: error computing maxUtilityFee: %v") + // Only compute BMF if we're passed the PoS cutover. + if blockHeight >= uint64(bav.Params.ForkHeights.ProofOfStake2ConsensusCutoverBlockHeight) { + _, utilityFee = computeBMF(nonBlockRewardRecipientFees) + maxUtilityFee, err = SafeUint64().Add(maxUtilityFee, utilityFee) + if err != nil { + return nil, errors.Wrap(err, + "ConnectBlock: error computing maxUtilityFee: %v") + } } } } From c58b2a1eaf9f008a4602e4ccfe79f288b6a33dd7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 14 Dec 2024 14:49:47 -0500 Subject: [PATCH 077/131] remove usage of ParsePubKey from btcec lib in connect block logic --- lib/block_producer.go | 39 +++++++++++++++++++++-------------- lib/block_view.go | 23 ++++++++++++--------- lib/block_view_atomic_txns.go | 14 +++++++------ lib/pos_block_producer.go | 9 +------- 4 files changed, 45 insertions(+), 40 deletions(-) diff --git a/lib/block_producer.go b/lib/block_producer.go index 1ad40c2ee..0b814cc57 100644 --- a/lib/block_producer.go +++ b/lib/block_producer.go @@ -285,9 +285,12 @@ func (desoBlockProducer *DeSoBlockProducer) _getBlockTemplate(publicKey []byte) desoBlockProducer.postgres, desoBlockProducer.chain.snapshot, nil) // Parse the public key that should be used for the block reward. - blockRewardOutputPublicKey, err := btcec.ParsePubKey(blockRewardOutput.PublicKey) - if err != nil { - return nil, nil, nil, errors.Wrapf(err, "DeSoBlockProducer._getBlockTemplate: problem parsing block reward output public key: ") + blockRewardOutputPublicKey := NewPublicKey(blockRewardOutput.PublicKey) + if blockRewardOutputPublicKey == nil { + return nil, nil, nil, fmt.Errorf( + "DeSoBlockProducer._getBlockTemplate: problem parsing block reward output public key: %v", + blockRewardOutput.PublicKey, + ) } // Skip the block reward, which is the first txn in the block. @@ -305,13 +308,14 @@ func (desoBlockProducer *DeSoBlockProducer) _getBlockTemplate(publicKey []byte) if blockRet.Header.Height >= uint64(desoBlockProducer.params.ForkHeights.BlockRewardPatchBlockHeight) { if txnInBlock.TxnMeta.GetTxnType() != TxnTypeAtomicTxnsWrapper { // Parse the transactor's public key to compare with the block reward output public key. - transactorPublicKey, err := btcec.ParsePubKey(txnInBlock.PublicKey) - if err != nil { + transactorPublicKey := NewPublicKey(txnInBlock.PublicKey) + if transactorPublicKey == nil { return nil, nil, nil, - errors.Wrapf(err, - "DeSoBlockProducer._getBlockTemplate: problem parsing transactor public key: ") + fmt.Errorf( + "DeSoBlockProducer._getBlockTemplate: problem parsing transactor public key: %v", + txnInBlock.PublicKey) } - includeFeesInBlockReward = !transactorPublicKey.IsEqual(blockRewardOutputPublicKey) + includeFeesInBlockReward = !transactorPublicKey.Equal(*blockRewardOutputPublicKey) } else { // In the case of atomic transaction wrappers, we must parse and process each inner transaction // independently. We let includeFeesInBlockRewards remain true but decrement feeNanos whenever @@ -465,10 +469,12 @@ func RecomputeBlockRewardWithBlockRewardOutputPublicKey( blockRewardOutputPublicKeyBytes []byte, params *DeSoParams, ) (*MsgDeSoBlock, error) { - blockRewardOutputPublicKey, err := btcec.ParsePubKey(blockRewardOutputPublicKeyBytes) - if err != nil { - return nil, errors.Wrap( - fmt.Errorf("RecomputeBlockRewardWithBlockRewardOutpubPublicKey: Problem parsing block reward output public key: %v", err), "") + blockRewardOutputPublicKey := NewPublicKey(blockRewardOutputPublicKeyBytes) + if blockRewardOutputPublicKey == nil { + return nil, + fmt.Errorf( + "RecomputeBlockRewardWithBlockRewardOutpubPublicKey: Problem parsing block reward output public key: %v", + blockRewardOutputPublicKeyBytes) } // Find all transactions in block that have transactor == block reward output public key @@ -476,14 +482,15 @@ func RecomputeBlockRewardWithBlockRewardOutputPublicKey( totalFees := uint64(0) for _, txn := range block.Txns[1:] { if txn.TxnMeta.GetTxnType() != TxnTypeAtomicTxnsWrapper { - transactorPublicKey, err := btcec.ParsePubKey(txn.PublicKey) - if err != nil { - glog.Errorf("DeSoMiner._startThread: Error parsing transactor public key: %v", err) + transactorPublicKey := NewPublicKey(txn.PublicKey) + if transactorPublicKey == nil { + glog.Errorf("DeSoMiner._startThread: Error parsing transactor public key: %v", txn.PublicKey) continue } - if transactorPublicKey.IsEqual(blockRewardOutputPublicKey) { + if transactorPublicKey.Equal(*blockRewardOutputPublicKey) { continue } + var err error totalFees, err = SafeUint64().Add(totalFees, txn.TxnFeeNanos) if err != nil { glog.Errorf("DeSoMiner._startThread: Error adding txn fee: %v", err) diff --git a/lib/block_view.go b/lib/block_view.go index 6d937082d..c6b3f4f10 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -4348,7 +4348,7 @@ func (bav *UtxoView) ConnectBlock( } blockHeader := desoBlock.Header - var blockRewardOutputPublicKey *btcec.PublicKey + var blockRewardOutputPublicKey *PublicKey // If the block height is greater than or equal to the block reward patch height, // we will verify that there is only one block reward output and we'll parse // that public key @@ -4365,11 +4365,12 @@ func (bav *UtxoView) ConnectBlock( if len(desoBlock.Txns[0].TxOutputs) != 1 { return nil, errors.Wrap(RuleErrorBlockRewardTxnMustHaveOneOutput, "ConnectBlock: Block reward transaction must have exactly one output") } - var err error - blockRewardOutputPublicKey, err = - btcec.ParsePubKey(desoBlock.Txns[0].TxOutputs[0].PublicKey) - if err != nil { - return nil, fmt.Errorf("ConnectBlock: Problem parsing block reward public key: %v", err) + blockRewardOutputPublicKey = + NewPublicKey(desoBlock.Txns[0].TxOutputs[0].PublicKey) + if blockRewardOutputPublicKey == nil { + return nil, fmt.Errorf( + "ConnectBlock: Problem parsing block reward public key: incorrect number of bytes in public key: %v", + desoBlock.Txns[0].TxOutputs[0].PublicKey) } } @@ -4407,11 +4408,13 @@ func (bav *UtxoView) ConnectBlock( if blockHeight >= uint64(bav.Params.ForkHeights.BlockRewardPatchBlockHeight) && txn.TxnMeta.GetTxnType() != TxnTypeBlockReward && txn.TxnMeta.GetTxnType() != TxnTypeAtomicTxnsWrapper { - transactorPubKey, err := btcec.ParsePubKey(txn.PublicKey) - if err != nil { - return nil, fmt.Errorf("ConnectBlock: Problem parsing transactor public key: %v", err) + transactorPubKey := NewPublicKey(txn.PublicKey) + if transactorPubKey == nil { + return nil, fmt.Errorf( + "ConnectBlock: Problem parsing transactor public key: incorrect number of bytes in txn.PublicKey: %v", + txn.PublicKey) } - includeFeesInBlockReward = !transactorPubKey.IsEqual(blockRewardOutputPublicKey) + includeFeesInBlockReward = !transactorPubKey.Equal(*blockRewardOutputPublicKey) } if includeFeesInBlockReward { diff --git a/lib/block_view_atomic_txns.go b/lib/block_view_atomic_txns.go index 8e0fa950d..9554791cd 100644 --- a/lib/block_view_atomic_txns.go +++ b/lib/block_view_atomic_txns.go @@ -5,7 +5,6 @@ import ( "fmt" "io" - "github.com/btcsuite/btcd/btcec/v2" "github.com/pkg/errors" ) @@ -445,18 +444,21 @@ func _verifyAtomicTxnsChain(txnMeta *AtomicTxnsWrapperMetadata) error { return nil } -func filterOutBlockRewardRecipientFees(txns []*MsgDeSoTxn, publicRewardPublicKey *btcec.PublicKey) (uint64, error) { +func filterOutBlockRewardRecipientFees(txns []*MsgDeSoTxn, publicRewardPublicKey *PublicKey) (uint64, error) { var nonBlockRewardRecipientFees uint64 for _, txn := range txns { // If the transaction is performed by any public key other than block reward recipient transaction, // add the fees to the total. - transactorPublicKey, err := btcec.ParsePubKey(txn.PublicKey) - if err != nil { - return 0, errors.Wrap(err, "filterBlockRewardRecipientFees: failed to parse public key") + transactorPublicKey := NewPublicKey(txn.PublicKey) + if transactorPublicKey == nil { + return 0, fmt.Errorf( + "filterBlockRewardRecipientFees: failed to parse public key: incorrect number of bytes: %v", + txn.PublicKey) } - if transactorPublicKey.IsEqual(publicRewardPublicKey) { + if transactorPublicKey.Equal(*publicRewardPublicKey) { continue } + var err error nonBlockRewardRecipientFees, err = SafeUint64().Add(nonBlockRewardRecipientFees, txn.TxnFeeNanos) if err != nil { return 0, errors.Wrap(err, "filterBlockRewardRecipientFees: failed to add fees") diff --git a/lib/pos_block_producer.go b/lib/pos_block_producer.go index 705899a5b..e549e91a6 100644 --- a/lib/pos_block_producer.go +++ b/lib/pos_block_producer.go @@ -4,8 +4,6 @@ import ( "math" "time" - "github.com/btcsuite/btcd/btcec/v2" - "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/bls" @@ -347,14 +345,9 @@ func (pbp *PosBlockProducer) getBlockTransactions( return nil, 0, errors.New("Error casting txn meta to AtomicSwapMetadata") } - blockProducerPublicKeyBtcec, err := btcec.ParsePubKey(blockProducerPublicKey.ToBytes()) - if err != nil { - return nil, 0, - errors.Wrapf(err, "Error parsing block producer public key: ") - } // Set fees to the sum of fees paid by public keys other than the block producer. fees, err = filterOutBlockRewardRecipientFees( - txnMeta.Txns, blockProducerPublicKeyBtcec) + txnMeta.Txns, blockProducerPublicKey) if err != nil { return nil, 0, errors.Wrapf(err, "error filtering out block reward recipient fees") From f727ec47c8f18f577756cc2a5bcea1c0eff79630 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 20 Dec 2024 18:23:49 -0500 Subject: [PATCH 078/131] adjust log level for get utxo view and utxo ops at block hash. add logging in run block index migration --- lib/db_utils.go | 5 +++++ lib/pos_blockchain.go | 2 +- lib/server.go | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/db_utils.go b/lib/db_utils.go index 864a862c9..85ee0f94d 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5648,6 +5648,7 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager var bestHashHeight uint32 // Iterate over all the keys in the height hash to node index, extract the height and hash, // and batch write every 10k entries to the hash to height index. + startTime := time.Now() for nodeIterator.Seek(prefix); nodeIterator.ValidForPrefix(prefix); nodeIterator.Next() { item := nodeIterator.Item().Key() @@ -5660,6 +5661,9 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager if bestHash != nil && bestHash.IsEqual(&hash) { bestHashHeight = height } + if height%100000 == 0 { + glog.V(0).Infof("Time to run block index migration to height %v: %v", height, time.Since(startTime)) + } // If we have fewer than 10K entries, continue. if len(hashToHeightMap) < 10000 { continue @@ -5679,6 +5683,7 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return errors.Wrap(innerErr, "RunBlockIndexMigration: Problem putting hash to height") } } + glog.V(0).Infof("Time to run block index migration: %v", time.Since(startTime)) // If we don't have a best hash, then we certainly haven't hit the first pos block height. if bestHash == nil { return nil diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index bfd6ef9fa..614966b9b 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1966,7 +1966,7 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { - glog.Infof("Connecting block %v", uncommittedAncestors[ii]) + glog.V(4).Infof("Connecting block %v", uncommittedAncestors[ii]) var err error // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) diff --git a/lib/server.go b/lib/server.go index fa067edee..c5f0bf4c0 100644 --- a/lib/server.go +++ b/lib/server.go @@ -371,7 +371,7 @@ func RunBlockIndexMigrationOnce(db *badger.DB, params *DeSoParams) error { glog.V(2).Info("Block index migration has already been run") return nil } - glog.V(2).Info("Running block index migration") + glog.V(0).Info("Running block index migration") if err = RunBlockIndexMigration(db, nil, nil, params); err != nil { return errors.Wrapf(err, "Problem running block index migration") } From 80f3bf737b5ad516c7265a70baecf16d6d398e7f Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 20 Dec 2024 18:33:30 -0500 Subject: [PATCH 079/131] add enhancement to get cached block view --- lib/pos_blockchain.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 614966b9b..d5b9eba11 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1960,13 +1960,19 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc // Connect the uncommitted blocks to the tip so that we can validate subsequent blocks utxoView := NewUtxoViewWithSnapshotCache(bc.db, bc.params, bc.postgres, bc.snapshot, bc.eventManager, bc.snapshotCache) - // TODO: there's another performance enhancement we can make here. If we have a view in the - // cache for one of the ancestors, we can skip fetching the block and connecting it by taking - // a copy of it and replacing the existing view. var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { glog.V(4).Infof("Connecting block %v", uncommittedAncestors[ii]) + // Check the cache to see if we already have a view for this block. + cachedView, cachedAncestorExists := bc.getCachedBlockViewAndUtxoOps(*uncommittedAncestors[ii].Hash) + if cachedAncestorExists { + cachedViewCopy := cachedView.Copy() + utxoView = cachedViewCopy.UtxoView + fullBlock = cachedViewCopy.Block + utxoOps = cachedViewCopy.UtxoOps + continue + } var err error // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) From 754b0a6d30538a408c8d8f32faebc47d5f4e79e1 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 20 Dec 2024 18:35:19 -0500 Subject: [PATCH 080/131] remove unnecessary log --- lib/pos_blockchain.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index d5b9eba11..57f116ce3 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1963,7 +1963,6 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { - glog.V(4).Infof("Connecting block %v", uncommittedAncestors[ii]) // Check the cache to see if we already have a view for this block. cachedView, cachedAncestorExists := bc.getCachedBlockViewAndUtxoOps(*uncommittedAncestors[ii].Hash) if cachedAncestorExists { From 65dc6a6079ab11078ad984932501902d7a9ef1c9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 21 Dec 2024 01:05:06 -0500 Subject: [PATCH 081/131] fix block index migration --- lib/db_utils.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/db_utils.go b/lib/db_utils.go index 85ee0f94d..0e0db9a46 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -5689,6 +5689,8 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager return nil } + glog.V(0).Infof("Running PoW block committed migration...") + // We want to mark all PoW blocks as committed, so we'll get the first pos block // and iterate backwards marking all blocks as committed. If the PoS cutover hasn't // happened yet, then we'll just the current best hash from the DB to determine the @@ -5715,30 +5717,47 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager // In this case we have not reached the cutover, we need to find pull the best hash // from the DB and iterate backwards. if len(valsFound) == 0 { + glog.V(0).Infof("Found multiple blocks at PoS cutover height: %v num blocks", len(valsFound)) blockNode = GetHeightHashToNodeInfoWithTxn(txn, snapshot, bestHashHeight, bestHash, false) if blockNode == nil { return fmt.Errorf("RunBlockIndexMigration: block with Best hash (%v) and height (%v) not found", bestHash, bestHashHeight) } } else { - // If we have 1 block at the cutover height, we'll use that block to iterate backwards. - blockNode, err = DeserializeBlockNode(valsFound[0]) - if err != nil { - return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node for pos cutover") + // If we have one or more blocks at the cutover height, we need to find the block that is committed. + for _, val := range valsFound { + blockNode, err = DeserializeBlockNode(val) + if err != nil { + return errors.Wrap(err, "RunBlockIndexMigration: Problem deserializing block node") + } + // If we found the committed block, break out. + if blockNode.IsCommitted() { + break + } + } + if !blockNode.IsCommitted() { + return fmt.Errorf("RunBlockIndexMigration: No committed block found at PoS cutover height") } } + startHeight := blockNode.Height + startTime = time.Now() var blockNodeBatch []*BlockNode for blockNode != nil { // If the block is not committed, mark it as committed. if !blockNode.IsCommitted() { blockNode.Status |= StatusBlockCommitted + // Add it to the batch. + blockNodeBatch = append(blockNodeBatch, blockNode) } - // Add it to the batch. - blockNodeBatch = append(blockNodeBatch, blockNode) // Find the parent of this block. - parentBlockNode := GetHeightHashToNodeInfoWithTxn(txn, snapshot, blockNode.Height, blockNode.Hash, false /*bitcoinNodes*/) + parentBlockNode := GetHeightHashToNodeInfoWithTxn( + txn, snapshot, blockNode.Height-1, blockNode.Header.PrevBlockHash, false /*bitcoinNodes*/) if blockNode.Height > 0 && parentBlockNode == nil { return errors.New("RunBlockIndexMigration: Parent block node not found") } + if blockNode.Height%10000 == 0 { + glog.V(0).Infof("Time to run PoW block committed migration from start height %v to height %v: %v", + startHeight, blockNode.Height, time.Since(startTime)) + } // Jump up to the parent block node. blockNode = parentBlockNode // If we have fewer than 10K entries, continue. From 760a5f0de142c654af3c05a926a1a9fc9e9dc52d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 19:24:32 -0500 Subject: [PATCH 082/131] don't use cache in get utxo view and utxo ops at block hash --- lib/pos_blockchain.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 57f116ce3..6da5b03c0 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1964,14 +1964,14 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { // Check the cache to see if we already have a view for this block. - cachedView, cachedAncestorExists := bc.getCachedBlockViewAndUtxoOps(*uncommittedAncestors[ii].Hash) - if cachedAncestorExists { - cachedViewCopy := cachedView.Copy() - utxoView = cachedViewCopy.UtxoView - fullBlock = cachedViewCopy.Block - utxoOps = cachedViewCopy.UtxoOps - continue - } + //cachedView, cachedAncestorExists := bc.getCachedBlockViewAndUtxoOps(*uncommittedAncestors[ii].Hash) + //if cachedAncestorExists { + // cachedViewCopy := cachedView.Copy() + // utxoView = cachedViewCopy.UtxoView + // fullBlock = cachedViewCopy.Block + // utxoOps = cachedViewCopy.UtxoOps + // continue + //} var err error // We need to get these blocks from badger fullBlock, err = GetBlock(uncommittedAncestors[ii].Hash, bc.db, bc.snapshot) From f6d8c1cb4a678aabae1aeb8bcd2c3704d495c39d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 19:44:57 -0500 Subject: [PATCH 083/131] use GetParent where applicable. fix start up --- lib/blockchain.go | 24 +++++++++++------------- lib/pos_blockchain.go | 13 ++++++------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 71abbf143..6f1db0f12 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1063,8 +1063,9 @@ func (bc *Blockchain) _initChain() error { // @diamondhands - we could reduce this to just the last hour if we want. // Walk back the last 24 hours of blocks. currBlockCounter := 1 - for currBlockCounter < 3600*24 && tipNode.Header.PrevBlockHash != nil { - bc.blockIndex.GetBlockNodeByHashAndHeight(tipNode.Header.PrevBlockHash, tipNode.Header.Height-1) + parentNode := tipNode.GetParent(bc.blockIndex) + for currBlockCounter < 3600*24 && parentNode != nil { + parentNode = parentNode.GetParent(bc.blockIndex) currBlockCounter++ } } @@ -1913,14 +1914,13 @@ func (bc *Blockchain) _FindCommonAncestor(node1 *BlockNode, node2 *BlockNode) *B // reach the end of the lists. We only need to check node1 for nil // since they're the same height and we are iterating both back // in tandem. - var exists bool for !node1.Hash.IsEqual(node2.Hash) { - node1, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node1.Header.PrevBlockHash, uint64(node1.Height-1)) - if !exists { + node1 = node1.GetParent(bc.blockIndex) + if node1 == nil { return nil } - node2, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(node2.Header.PrevBlockHash, uint64(node2.Height-1)) - if !exists { + node2 = node2.GetParent(bc.blockIndex) + if node2 == nil { return nil } } @@ -2038,9 +2038,8 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( *currentBlock = *tip for currentBlock != nil && *currentBlock.Hash != *commonAncestor.Hash { detachBlocks = append(detachBlocks, currentBlock) - var exists bool - currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) - if !exists { + currentBlock = currentBlock.GetParent(bc.blockIndex) + if currentBlock == nil { glog.Fatalf("GetReorgBlocks: Failed to find parent of block. Parent hash %v", currentBlock.Header.PrevBlockHash) } } @@ -2057,9 +2056,8 @@ func (bc *Blockchain) GetReorgBlocks(tip *BlockNode, newNode *BlockNode) ( *currentBlock = *newNode for *currentBlock.Hash != *commonAncestor.Hash { attachBlocks = append(attachBlocks, currentBlock) - var exists bool - currentBlock, exists = bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) - if !exists { + currentBlock = currentBlock.GetParent(bc.blockIndex) + if currentBlock == nil { // TODO: what should we do here? glog.Fatal("GetReorgBlocks: Failed to find parent of block") } diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 6da5b03c0..dce96cbb8 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -117,9 +117,8 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas if header.Height < 1 { return nil, false, errors.New("validateAndIndexHeaderPoS: Header height is less than 1 - no valid parent height") } - parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight( - header.PrevBlockHash, header.Height-1) - if !parentBlockNodeExists { + parentBlockNode := blockNode.GetParent(bc.blockIndex) + if parentBlockNode == nil { return nil, true, nil } @@ -1685,13 +1684,13 @@ func (bc *Blockchain) canCommitGrandparent(currentBlock *BlockNode) ( ) { // TODO: Is it sufficient that the current block's header points to the parent // or does it need to have something to do with the QC? - parent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentBlock.Header.PrevBlockHash, uint64(currentBlock.Height-1)) - if !exists { + parent := currentBlock.GetParent(bc.blockIndex) + if parent == nil { glog.Errorf("canCommitGrandparent: Parent block %v not found in best chain map", currentBlock.Header.PrevBlockHash.String()) return nil, false } - grandParent, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(parent.Header.PrevBlockHash, uint64(parent.Height-1)) - if !exists { + grandParent := parent.GetParent(bc.blockIndex) + if grandParent == nil { glog.Errorf("canCommitGrandparent: Grandparent block %v not found in best chain map", parent.Header.PrevBlockHash.String()) return nil, false } From 55cc6763b3de83eb2d3015c9fa4fd41637dc681a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 19:48:59 -0500 Subject: [PATCH 084/131] add comment about why 'optimization' actually doesn't work --- lib/pos_blockchain.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index dce96cbb8..e209a5c80 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1962,6 +1962,9 @@ func (bc *Blockchain) GetUtxoViewAndUtxoOpsAtBlockHash(blockHash BlockHash, bloc var utxoOps [][]*UtxoOperation var fullBlock *MsgDeSoBlock for ii := len(uncommittedAncestors) - 1; ii >= 0; ii-- { + // TODO: this optimization is disabled for now. Although this speeds up the fetching of + // a utxo view for a given block hash, it actually ends up building up a lot of data in the + // view of already committed data. // Check the cache to see if we already have a view for this block. //cachedView, cachedAncestorExists := bc.getCachedBlockViewAndUtxoOps(*uncommittedAncestors[ii].Hash) //if cachedAncestorExists { From bb09433595ccadcbeac56ff380ff225656f11f12 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 20:05:04 -0500 Subject: [PATCH 085/131] Revert one usage of GetParent --- lib/pos_blockchain.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index e209a5c80..d62e3e8ae 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -117,8 +117,9 @@ func (bc *Blockchain) validateAndIndexHeaderPoS(header *MsgDeSoHeader, headerHas if header.Height < 1 { return nil, false, errors.New("validateAndIndexHeaderPoS: Header height is less than 1 - no valid parent height") } - parentBlockNode := blockNode.GetParent(bc.blockIndex) - if parentBlockNode == nil { + parentBlockNode, parentBlockNodeExists := bc.blockIndex.GetBlockNodeByHashAndHeight( + header.PrevBlockHash, header.Height-1) + if !parentBlockNodeExists { return nil, true, nil } From e115f1c509a4f559406f0ab7ff767983cc418afd Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 22:30:53 -0500 Subject: [PATCH 086/131] log start of handleBlockProposalEvent, simplify get safe block nodes --- lib/pos_blockchain.go | 8 ++++++-- lib/pos_consensus.go | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index d62e3e8ae..d13e75399 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -2041,6 +2041,7 @@ func (bc *Blockchain) GetSafeBlocks() ([]*MsgDeSoHeader, error) { return headers, nil } +// TODO: this is pretty inefficient now. We may want to consider keeping all safe block nodes in memory. func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { // First get committed tip. committedTip, exists := bc.GetCommittedTip() @@ -2049,8 +2050,11 @@ func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { } // Now get all blocks from the committed tip to the best chain tip. safeBlocks := []*BlockNode{committedTip} - maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) - for ii := uint64(committedTip.Height + 1); ii < maxHeightWithSafeBlocks+1; ii++ { + // TODO: improve performance of getMaxSequentialBlockHeightAfter. It's too slow when + // we don't have the entire block index in memory. + //maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) + // Only get the blocks from the block index for the next 100 heights. + for ii := uint64(committedTip.Height + 1); ii < uint64(committedTip.Height+101); ii++ { blockNodes := bc.blockIndex.GetBlockNodesByHeight(ii) // If we don't have any blocks at this height, we know that any blocks at a later height are not safe blocks. if len(blockNodes) == 0 { diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index ab7a73190..0af8df1a3 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -238,6 +238,8 @@ func (fc *FastHotStuffConsensus) handleBlockProposalEvent( event *consensus.FastHotStuffEvent, expectedEventType consensus.FastHotStuffEventType, ) error { + // TODO: DELETE ME!! + glog.Infof("FastHotStuffConsensus.handleBlockProposalEvent: %s", event.ToString()) // Validate that the event's type is the expected proposal event type if !isValidBlockProposalEvent(event, expectedEventType) { return errors.Errorf("Unexpected event type: %v vs %v", event.EventType, expectedEventType) From 49758546c392c499f65adba8ef03b4e5efb00d5a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 22:55:40 -0500 Subject: [PATCH 087/131] put getMaxSequentialBlockHeightAfter back in --- lib/pos_blockchain.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index d13e75399..ff622583b 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -2052,9 +2052,8 @@ func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { safeBlocks := []*BlockNode{committedTip} // TODO: improve performance of getMaxSequentialBlockHeightAfter. It's too slow when // we don't have the entire block index in memory. - //maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) - // Only get the blocks from the block index for the next 100 heights. - for ii := uint64(committedTip.Height + 1); ii < uint64(committedTip.Height+101); ii++ { + maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) + for ii := uint64(committedTip.Height + 1); ii < maxHeightWithSafeBlocks+1; ii++ { blockNodes := bc.blockIndex.GetBlockNodesByHeight(ii) // If we don't have any blocks at this height, we know that any blocks at a later height are not safe blocks. if len(blockNodes) == 0 { From 2f3724dd1d4b320976ce6622eebb11e0eaccc60b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 22 Dec 2024 23:05:55 -0500 Subject: [PATCH 088/131] don't need to call getMaxSequentialBlockHeightAfter --- lib/pos_blockchain.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index ff622583b..a630f2130 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -2052,8 +2052,11 @@ func (bc *Blockchain) getSafeBlockNodes() ([]*BlockNode, error) { safeBlocks := []*BlockNode{committedTip} // TODO: improve performance of getMaxSequentialBlockHeightAfter. It's too slow when // we don't have the entire block index in memory. - maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) - for ii := uint64(committedTip.Height + 1); ii < maxHeightWithSafeBlocks+1; ii++ { + //maxHeightWithSafeBlocks := bc.getMaxSequentialBlockHeightAfter(uint64(committedTip.Height)) + //for ii := uint64(committedTip.Height + 1); ii < maxHeightWithSafeBlocks+1; ii++ { + // By excluding the end condition, we can get all safe blocks without doing the same + // iteration twice. + for ii := uint64(committedTip.Height + 1); ; ii++ { blockNodes := bc.blockIndex.GetBlockNodesByHeight(ii) // If we don't have any blocks at this height, we know that any blocks at a later height are not safe blocks. if len(blockNodes) == 0 { From 2ba0c662c86a88c42aff7c83489872295a290e40 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 23 Dec 2024 13:26:56 -0500 Subject: [PATCH 089/131] add param to enumerate keys for prefix to disable prefetch --- lib/block_view_like.go | 2 +- lib/block_view_post.go | 6 +-- lib/block_view_stake.go | 1 + lib/blockchain.go | 2 +- lib/db_utils.go | 76 +++++++++++++++++++++---------------- lib/pos_snapshot_entries.go | 2 +- 6 files changed, 50 insertions(+), 39 deletions(-) diff --git a/lib/block_view_like.go b/lib/block_view_like.go index 3af7983aa..a6dd2350a 100644 --- a/lib/block_view_like.go +++ b/lib/block_view_like.go @@ -77,7 +77,7 @@ func (bav *UtxoView) GetLikesForPostHash(postHash *BlockHash) (_likerPubKeys [][ handle := bav.Handle dbPrefix := append([]byte{}, Prefixes.PrefixLikedPostHashToLikerPubKey...) dbPrefix = append(dbPrefix, postHash[:]...) - keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true) + keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true, false) // Iterate over all the db keys & values and load them into the view. expectedKeyLength := 1 + HashSizeBytes + btcec.PubKeyBytesLenCompressed diff --git a/lib/block_view_post.go b/lib/block_view_post.go index a6b9e6d30..e76745fbb 100644 --- a/lib/block_view_post.go +++ b/lib/block_view_post.go @@ -592,7 +592,7 @@ func (bav *UtxoView) GetDiamondSendersForPostHash(postHash *BlockHash) (_pkidToD // FIXME: Db operation like this shouldn't happen in utxoview. dbPrefix := append([]byte{}, Prefixes.PrefixDiamondedPostHashDiamonderPKIDDiamondLevel...) dbPrefix = append(dbPrefix, postHash[:]...) - keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true) + keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true, false) diamondPostEntry := bav.GetPostEntryForPostHash(postHash) receiverPKIDEntry := bav.GetPKIDForPublicKey(diamondPostEntry.PosterPublicKey) @@ -633,7 +633,7 @@ func (bav *UtxoView) GetRepostsForPostHash(postHash *BlockHash) (_reposterPubKey // FIXME: Db operation like this shouldn't happen in utxoview. dbPrefix := append([]byte{}, Prefixes.PrefixRepostedPostHashReposterPubKey...) dbPrefix = append(dbPrefix, postHash[:]...) - keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true) + keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true, false) // Iterate over all the db keys & values and load them into the view. expectedKeyLength := 1 + HashSizeBytes + btcec.PubKeyBytesLenCompressed @@ -670,7 +670,7 @@ func (bav *UtxoView) GetQuoteRepostsForPostHash(postHash *BlockHash, // FIXME: Db operation like this shouldn't happen in utxoview. dbPrefix := append([]byte{}, Prefixes.PrefixRepostedPostHashReposterPubKeyRepostPostHash...) dbPrefix = append(dbPrefix, postHash[:]...) - keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true) + keysFound, _ := EnumerateKeysForPrefix(handle, dbPrefix, true, false) // Iterate over all the db keys & values and load them into the view. expectedKeyLength := 1 + HashSizeBytes + btcec.PubKeyBytesLenCompressed + HashSizeBytes diff --git a/lib/block_view_stake.go b/lib/block_view_stake.go index b83d5d7ac..90b27b4a7 100644 --- a/lib/block_view_stake.go +++ b/lib/block_view_stake.go @@ -2999,6 +2999,7 @@ func (bav *UtxoView) _checkStakeTxnSpendingLimitAndUpdateDerivedKey( for _, validatorPKID := range []*PKID{validatorEntry.ValidatorPKID, &ZeroPKID} { // Retrieve DerivedKeyEntry.TransactionSpendingLimit. stakeLimitKey := MakeStakeLimitKey(validatorPKID) + // TODO: check that StakeLimitMap is not nil. spendingLimit, exists := derivedKeyEntry.TransactionSpendingLimitTracker.StakeLimitMap[stakeLimitKey] if !exists { continue diff --git a/lib/blockchain.go b/lib/blockchain.go index 6f1db0f12..ae451bdcb 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -690,7 +690,7 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { } prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) // Enumerate all block nodes for the prefix. - _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false) + _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false, true) // skip prefetch. blockNodes := []*BlockNode{} for _, val := range valsFound { // Deserialize all block nodes. diff --git a/lib/db_utils.go b/lib/db_utils.go index 0e0db9a46..003e5f473 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -1527,18 +1527,28 @@ func DBDeletePKIDMappingsWithTxn(txn *badger.Txn, snap *Snapshot, publicKey []by return nil } -func EnumerateKeysForPrefix(db *badger.DB, dbPrefix []byte, keysOnly bool) (_keysFound [][]byte, _valsFound [][]byte) { - return _enumerateKeysForPrefix(db, dbPrefix, keysOnly) +func EnumerateKeysForPrefix( + db *badger.DB, + dbPrefix []byte, + keysOnly bool, + skipPrefetch bool, +) (_keysFound [][]byte, _valsFound [][]byte) { + return _enumerateKeysForPrefix(db, dbPrefix, keysOnly, skipPrefetch) } // A helper function to enumerate all of the values for a particular prefix. -func _enumerateKeysForPrefix(db *badger.DB, dbPrefix []byte, keysOnly bool) (_keysFound [][]byte, _valsFound [][]byte) { +func _enumerateKeysForPrefix( + db *badger.DB, + dbPrefix []byte, + keysOnly bool, + skipPrefetch bool, +) (_keysFound [][]byte, _valsFound [][]byte) { keysFound := [][]byte{} valsFound := [][]byte{} dbErr := db.View(func(txn *badger.Txn) error { var err error - keysFound, valsFound, err = _enumerateKeysForPrefixWithTxn(txn, dbPrefix, keysOnly) + keysFound, valsFound, err = _enumerateKeysForPrefixWithTxn(txn, dbPrefix, keysOnly, skipPrefetch) if err != nil { return err } @@ -1552,12 +1562,12 @@ func _enumerateKeysForPrefix(db *badger.DB, dbPrefix []byte, keysOnly bool) (_ke return keysFound, valsFound } -func _enumerateKeysForPrefixWithTxn(txn *badger.Txn, dbPrefix []byte, keysOnly bool) (_keysFound [][]byte, _valsFound [][]byte, _err error) { +func _enumerateKeysForPrefixWithTxn(txn *badger.Txn, dbPrefix []byte, keysOnly bool, skipPrefetch bool) (_keysFound [][]byte, _valsFound [][]byte, _err error) { keysFound := [][]byte{} valsFound := [][]byte{} opts := badger.DefaultIteratorOptions - if keysOnly { + if keysOnly || skipPrefetch { opts.PrefetchValues = false } opts.Prefix = dbPrefix @@ -1918,7 +1928,7 @@ func DBGetMessageEntriesForPublicKey(handle *badger.DB, publicKey []byte) ( // Goes backwards to get messages in time sorted order. // Limit the number of keys to speed up load times. - _, valuesFound := _enumerateKeysForPrefix(handle, prefix, false) + _, valuesFound := _enumerateKeysForPrefix(handle, prefix, false, false) privateMessages := []*MessageEntry{} for _, valBytes := range valuesFound { @@ -2139,7 +2149,7 @@ func DBGetMessagingGroupEntriesForOwnerWithTxn(txn *badger.Txn, ownerPublicKey * // Setting the prefix to owner's public key will allow us to fetch all messaging keys // for the user. We enumerate this prefix. prefix := _dbSeekPrefixForMessagingGroupEntry(ownerPublicKey) - _, valuesFound, err := _enumerateKeysForPrefixWithTxn(txn, prefix, false) + _, valuesFound, err := _enumerateKeysForPrefixWithTxn(txn, prefix, false, false) if err != nil { return nil, errors.Wrapf(err, "DBGetMessagingGroupEntriesForOwnerWithTxn: "+ "problem enumerating messaging key entries for prefix (%v)", prefix) @@ -3362,7 +3372,7 @@ func DBGetAllMessagingGroupEntriesForMemberWithTxn(txn *badger.Txn, ownerPublicK // This function is used to fetch all messaging var messagingGroupEntries []*MessagingGroupEntry prefix := _dbSeekPrefixForMessagingGroupMember(ownerPublicKey) - _, valuesFound, err := _enumerateKeysForPrefixWithTxn(txn, prefix, false) + _, valuesFound, err := _enumerateKeysForPrefixWithTxn(txn, prefix, false, false) if err != nil { return nil, errors.Wrapf(err, "DBGetAllMessagingGroupEntriesForMemberWithTxn: "+ "problem enumerating messaging key entries for prefix (%v)", prefix) @@ -3628,7 +3638,7 @@ func DbGetPostHashesYouLike(handle *badger.DB, yourPublicKey []byte) ( _postHashes []*BlockHash, _err error) { prefix := _dbSeekPrefixForPostHashesYouLike(yourPublicKey) - keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true, false) postHashesYouLike := []*BlockHash{} for _, keyBytes := range keysFound { @@ -3645,7 +3655,7 @@ func DbGetLikerPubKeysLikingAPostHash(handle *badger.DB, likedPostHash BlockHash _pubKeys [][]byte, _err error) { prefix := _dbSeekPrefixForLikerPubKeysLikingAPostHash(likedPostHash) - keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true, false) userPubKeys := [][]byte{} for _, keyBytes := range keysFound { @@ -3758,7 +3768,7 @@ func DbGetReposterPubKeyRepostedPostHashToRepostEntryWithTxn(txn *badger.Txn, snap *Snapshot, userPubKey []byte, repostedPostHash BlockHash) *RepostEntry { key := _dbSeekKeyForReposterPubKeyRepostedPostHashToRepostPostHash(userPubKey, repostedPostHash) - keysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, key, true) + keysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, key, true, false) if err != nil { return nil } @@ -3806,7 +3816,7 @@ func DbDeleteRepostMappingsWithTxn(txn *badger.Txn, snap *Snapshot, repostEntry func DbDeleteAllRepostMappingsWithTxn(txn *badger.Txn, snap *Snapshot, userPubKey []byte, repostedPostHash BlockHash, eventManager *EventManager, entryIsDeleted bool) error { key := _dbSeekKeyForReposterPubKeyRepostedPostHashToRepostPostHash(userPubKey, repostedPostHash) - keysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, key, true) + keysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, key, true, false) if err != nil { return nil } @@ -3823,7 +3833,7 @@ func DbGetPostHashesYouRepost(handle *badger.DB, yourPublicKey []byte) ( _postHashes []*BlockHash, _err error) { prefix := _dbSeekPrefixForPostHashesYouRepost(yourPublicKey) - keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true, false) postHashesYouRepost := []*BlockHash{} for _, keyBytes := range keysFound { @@ -3986,7 +3996,7 @@ func DbGetPKIDsYouFollow(handle *badger.DB, yourPKID *PKID) ( _pkids []*PKID, _err error) { prefix := _dbSeekPrefixForPKIDsYouFollow(yourPKID) - keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true, false) pkidsYouFollow := []*PKID{} for _, keyBytes := range keysFound { @@ -4004,7 +4014,7 @@ func DbGetPKIDsFollowingYou(handle *badger.DB, yourPKID *PKID) ( _pkids []*PKID, _err error) { prefix := _dbSeekPrefixForPKIDsFollowingYou(yourPKID) - keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, prefix, true, false) pkidsFollowingYou := []*PKID{} for _, keyBytes := range keysFound { @@ -4263,7 +4273,7 @@ func DbGetPKIDsThatDiamondedYouMap(handle *badger.DB, yourPKID *PKID, fetchYouDi diamondReceiverStartIdx = 1 + btcec.PubKeyBytesLenCompressed diamondReceiverEndIdx = 1 + 2*btcec.PubKeyBytesLenCompressed } - keysFound, valsFound := _enumerateKeysForPrefix(handle, prefix, false) + keysFound, valsFound := _enumerateKeysForPrefix(handle, prefix, false, false) pkidsToDiamondEntryMap := make(map[PKID][]*DiamondEntry) for ii, keyBytes := range keysFound { @@ -4337,7 +4347,7 @@ func DbGetDiamondEntriesForSenderToReceiver(handle *badger.DB, receiverPKID *PKI _diamondEntries []*DiamondEntry, _err error) { prefix := _dbSeekPrefixForReceiverPKIDAndSenderPKID(receiverPKID, senderPKID) - keysFound, valsFound := _enumerateKeysForPrefix(handle, prefix, false) + keysFound, valsFound := _enumerateKeysForPrefix(handle, prefix, false, false) var diamondEntries []*DiamondEntry for ii, keyBytes := range keysFound { // The DiamondEntry found must not be nil. @@ -4427,7 +4437,7 @@ func DbDeleteBitcoinBurnTxIDWithTxn(txn *badger.Txn, snap *Snapshot, bitcoinBurn } func DbGetAllBitcoinBurnTxIDs(handle *badger.DB) (_bitcoinBurnTxIDs []*BlockHash) { - keysFound, _ := _enumerateKeysForPrefix(handle, Prefixes.PrefixBitcoinBurnTxIDs, true) + keysFound, _ := _enumerateKeysForPrefix(handle, Prefixes.PrefixBitcoinBurnTxIDs, true, false) bitcoinBurnTxIDs := []*BlockHash{} for _, key := range keysFound { bbtxid := &BlockHash{} @@ -5705,7 +5715,7 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager firstPoSBlockHeight := params.GetFirstPoSBlockHeight() // Look up blocks at cutover height. prefixKey := _heightHashToNodePrefixByHeight(uint32(firstPoSBlockHeight), false) - _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, prefixKey, false) + _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, prefixKey, false, true) if err != nil { return errors.Wrap(err, "RunBlockIndexMigration: Problem enumerating keys for prefix") } @@ -5904,7 +5914,7 @@ func DbTxindexPublicKeyIndexToTxnKey(publicKey []byte, index uint32) []byte { func DbGetTxindexTxnsForPublicKeyWithTxn(txn *badger.Txn, publicKey []byte) []*BlockHash { txIDs := []*BlockHash{} - _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, DbTxindexPublicKeyPrefix(publicKey), false) + _, valsFound, err := _enumerateKeysForPrefixWithTxn(txn, DbTxindexPublicKeyPrefix(publicKey), false, false) if err != nil { return txIDs } @@ -8606,7 +8616,7 @@ func DBGetNFTEntriesForPostHash(handle *badger.DB, nftPostHash *BlockHash) (_nft nftEntries := []*NFTEntry{} prefix := append([]byte{}, Prefixes.PrefixPostHashSerialNumberToNFTEntry...) keyPrefix := append(prefix, nftPostHash[:]...) - _, entryByteStringsFound := _enumerateKeysForPrefix(handle, keyPrefix, false) + _, entryByteStringsFound := _enumerateKeysForPrefix(handle, keyPrefix, false, false) for _, byteString := range entryByteStringsFound { currentEntry := &NFTEntry{} rr := bytes.NewReader(byteString) @@ -8653,7 +8663,7 @@ func DBGetNFTEntriesForPKID(handle *badger.DB, ownerPKID *PKID) (_nftEntries []* var nftEntries []*NFTEntry prefix := append([]byte{}, Prefixes.PrefixPKIDIsForSaleBidAmountNanosPostHashSerialNumberToNFTEntry...) keyPrefix := append(prefix, ownerPKID[:]...) - _, entryByteStringsFound := _enumerateKeysForPrefix(handle, keyPrefix, false) + _, entryByteStringsFound := _enumerateKeysForPrefix(handle, keyPrefix, false, false) for _, byteString := range entryByteStringsFound { currentEntry := &NFTEntry{} rr := bytes.NewReader(byteString) @@ -8895,7 +8905,7 @@ func DBGetNFTBidEntriesForPKID(handle *badger.DB, bidderPKID *PKID) (_nftBidEntr { prefix := append([]byte{}, Prefixes.PrefixBidderPKIDPostHashSerialNumberToBidNanos...) keyPrefix := append(prefix, bidderPKID[:]...) - keysFound, valuesFound := _enumerateKeysForPrefix(handle, keyPrefix, false) + keysFound, valuesFound := _enumerateKeysForPrefix(handle, keyPrefix, false, false) bidderPKIDLength := len(bidderPKID[:]) for ii, keyFound := range keysFound { @@ -8932,7 +8942,7 @@ func DBGetNFTBidEntries(handle *badger.DB, nftPostHash *BlockHash, serialNumber prefix := append([]byte{}, Prefixes.PrefixPostHashSerialNumberBidNanosBidderPKID...) keyPrefix := append(prefix, nftPostHash[:]...) keyPrefix = append(keyPrefix, EncodeUint64(serialNumber)...) - keysFound, _ := _enumerateKeysForPrefix(handle, keyPrefix, true) + keysFound, _ := _enumerateKeysForPrefix(handle, keyPrefix, true, false) for _, keyFound := range keysFound { bidAmountStartIdx := 1 + HashSizeBytes + 8 // The length of prefix + the post hash + the serial #. bidAmountEndIdx := bidAmountStartIdx + 8 // Add the length of the bid amount (uint64). @@ -9100,7 +9110,7 @@ func DBGetAllOwnerToDerivedKeyMappings(handle *badger.DB, ownerPublicKey PublicK _entries []*DerivedKeyEntry, _err error) { prefix := _dbSeekPrefixForDerivedKeyMappings(ownerPublicKey) - _, valsFound := _enumerateKeysForPrefix(handle, prefix, false) + _, valsFound := _enumerateKeysForPrefix(handle, prefix, false, false) var derivedEntries []*DerivedKeyEntry for _, keyBytes := range valsFound { @@ -9568,7 +9578,7 @@ func DbGetBalanceEntriesYouHold(db *badger.DB, snap *Snapshot, pkid *PKID, filte { prefix := _dbGetPrefixForHODLerPKIDCreatorPKIDToBalanceEntry(isDAOCoin) keyPrefix := append(prefix, pkid[:]...) - _, entryByteStringsFound := _enumerateKeysForPrefix(db, keyPrefix, false) + _, entryByteStringsFound := _enumerateKeysForPrefix(db, keyPrefix, false, false) for _, byteString := range entryByteStringsFound { currentEntry := &BalanceEntry{} rr := bytes.NewReader(byteString) @@ -9590,7 +9600,7 @@ func DbGetBalanceEntriesHodlingYou(db *badger.DB, snap *Snapshot, pkid *PKID, fi { prefix := _dbGetPrefixForCreatorPKIDHODLerPKIDToBalanceEntry(isDAOCoin) keyPrefix := append(prefix, pkid[:]...) - _, entryByteStringsFound := _enumerateKeysForPrefix(db, keyPrefix, false) + _, entryByteStringsFound := _enumerateKeysForPrefix(db, keyPrefix, false, false) for _, byteString := range entryByteStringsFound { currentEntry := &BalanceEntry{} rr := bytes.NewReader(byteString) @@ -10111,7 +10121,7 @@ func DBGetAllDAOCoinLimitOrdersForThisTransactor( func _DBGetAllDAOCoinLimitOrdersByPrefix(handle *badger.DB, prefixKey []byte) ([]*DAOCoinLimitOrderEntry, error) { // Get all DAO coin limit orders containing this prefix. - _, valsFound := _enumerateKeysForPrefix(handle, prefixKey, false) + _, valsFound := _enumerateKeysForPrefix(handle, prefixKey, false, false) orders := []*DAOCoinLimitOrderEntry{} // Cast resulting values from bytes to order entries. @@ -10243,7 +10253,7 @@ func DbGetMempoolTxn(db *badger.DB, snap *Snapshot, mempoolTx *MempoolTx) *MsgDe } func DbGetAllMempoolTxnsSortedByTimeAdded(handle *badger.DB) (_mempoolTxns []*MsgDeSoTxn, _error error) { - _, valuesFound := _enumerateKeysForPrefix(handle, Prefixes.PrefixMempoolTxnHashToMsgDeSoTxn, false) + _, valuesFound := _enumerateKeysForPrefix(handle, Prefixes.PrefixMempoolTxnHashToMsgDeSoTxn, false, false) mempoolTxns := []*MsgDeSoTxn{} for _, mempoolTxnBytes := range valuesFound { @@ -10262,7 +10272,7 @@ func DbGetAllMempoolTxnsSortedByTimeAdded(handle *badger.DB) (_mempoolTxns []*Ms } func DbDeleteAllMempoolTxnsWithTxn(txn *badger.Txn, snap *Snapshot, eventManager *EventManager, entryIsDeleted bool) error { - txnKeysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, Prefixes.PrefixMempoolTxnHashToMsgDeSoTxn, true) + txnKeysFound, _, err := _enumerateKeysForPrefixWithTxn(txn, Prefixes.PrefixMempoolTxnHashToMsgDeSoTxn, true, false) if err != nil { return errors.Wrapf(err, "DbDeleteAllMempoolTxnsWithTxn: ") } @@ -10345,7 +10355,7 @@ func DbDeleteMempoolTxnKeyWithTxn(txn *badger.Txn, snap *Snapshot, txnKey []byte func LogDBSummarySnapshot(db *badger.DB) { keyCountMap := make(map[byte]int) for prefixByte := byte(0); prefixByte < byte(40); prefixByte++ { - keysForPrefix, _ := EnumerateKeysForPrefix(db, []byte{prefixByte}, true) + keysForPrefix, _ := EnumerateKeysForPrefix(db, []byte{prefixByte}, true, false) keyCountMap[prefixByte] = len(keysForPrefix) } glog.Info(spew.Printf("LogDBSummarySnapshot: Current DB summary snapshot: %v", keyCountMap)) @@ -12099,7 +12109,7 @@ func DbGetTransactorNonceEntriesToExpireAtBlockHeightWithTxn(txn *badger.Txn, bl } func DbGetAllTransactorNonceEntries(handle *badger.DB) []*TransactorNonceEntry { - keys, _ := EnumerateKeysForPrefix(handle, Prefixes.PrefixNoncePKIDIndex, true) + keys, _ := EnumerateKeysForPrefix(handle, Prefixes.PrefixNoncePKIDIndex, true, false) nonceEntries := []*TransactorNonceEntry{} for _, key := range keys { // Convert key to nonce entry. diff --git a/lib/pos_snapshot_entries.go b/lib/pos_snapshot_entries.go index b727c384a..02cb16ceb 100644 --- a/lib/pos_snapshot_entries.go +++ b/lib/pos_snapshot_entries.go @@ -1335,7 +1335,7 @@ func DBSeekSnapshotLeaderSchedule( snapshotAtEpochNumber uint64, ) (map[uint16]*PKID, error) { seekKey := DBSSeekKeyForSnapshotLeaderSchedule(snapshotAtEpochNumber) - keysFound, valsFound := EnumerateKeysForPrefix(handle, seekKey, false) + keysFound, valsFound := EnumerateKeysForPrefix(handle, seekKey, false, false) leaderIdxToPKID := make(map[uint16]*PKID) for idx, keyFound := range keysFound { // TODO: Make sure this decode is correct From 11aedb64edbe316d3dff614c4c7ff25d4a1427c0 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 23 Dec 2024 15:52:54 -0500 Subject: [PATCH 090/131] try using stream for iteration --- lib/blockchain.go | 7 ++++- lib/db_utils.go | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ae451bdcb..72ef877a2 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -690,7 +690,12 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { } prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) // Enumerate all block nodes for the prefix. - _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false, true) // skip prefetch. + //_, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false, true) // skip prefetch. + _, valsFound, err := _enumerateKeysForPrefixWithStream(bi.db, prefixKey, false) + if err != nil { + glog.Errorf("GetBlockNodesByHeight: Problem enumerating keys for prefix: %v", err) + return nil + } blockNodes := []*BlockNode{} for _, val := range valsFound { // Deserialize all block nodes. diff --git a/lib/db_utils.go b/lib/db_utils.go index 003e5f473..4704e3df5 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -2,18 +2,21 @@ package lib import ( "bytes" + "context" "crypto/rand" "encoding/binary" "encoding/hex" "encoding/json" "fmt" "github.com/deso-protocol/core/collections" + "github.com/dgraph-io/ristretto/z" "io" "log" "math" "math/big" "path/filepath" "reflect" + "runtime" "sort" "strings" "time" @@ -1527,6 +1530,76 @@ func DBDeletePKIDMappingsWithTxn(txn *badger.Txn, snap *Snapshot, publicKey []by return nil } +// _enumerateKeysForPrefixWithStream demonstrates scanning keys (and optional values) +// that share a prefix via the Badger Stream API. +func _enumerateKeysForPrefixWithStream(db *badger.DB, dbPrefix []byte, keysOnly bool) ([][]byte, [][]byte, error) { + keysFound := [][]byte{} + valsFound := [][]byte{} + + // Create a new stream on the DB. + stream := db.NewStream() + stream.NumGo = runtime.NumCPU() // use all cores + + // Restrict the stream to process only keys that match this prefix. + // The Stream API will fetch items in key order, parallelizing internally. + stream.Prefix = dbPrefix + + type StreamEntry struct { + key []byte + value []byte + } + + streamEntries := []StreamEntry{} + + // The Send callback receives batches of KVs. + stream.Send = func(buf *z.Buffer) error { + list, err := badger.BufferToKVList(buf) + if err != nil { + return err + } + for _, kv := range list.Kv { + // Double-check the prefix if you want a safeguard: + if !bytes.HasPrefix(kv.Key, dbPrefix) { + continue + } + + // Copy the key so it doesn't get overwritten by subsequent batches. + keyCopy := make([]byte, len(kv.Key)) + copy(keyCopy, kv.Key) + streamEntry := StreamEntry{ + key: keyCopy, + } + + // If we aren't in keysOnly mode, retrieve the value. + if !keysOnly { + // If KeyOnly = true above, kv.Value is empty. + // If KeyOnly = false, we can copy the value here. + valCopy := make([]byte, len(kv.Value)) + copy(valCopy, kv.Value) + //valsFound = append(valsFound, valCopy) + streamEntry.value = valCopy + } + streamEntries = append(streamEntries, streamEntry) + } + return nil + } + + // Execute the stream scan. + err := stream.Orchestrate(context.Background()) + if err != nil { + return nil, nil, err + } + + for _, streamEntry := range streamEntries { + keysFound = append(keysFound, streamEntry.key) + if !keysOnly { + valsFound = append(valsFound, streamEntry.value) + } + } + + return keysFound, valsFound, nil +} + func EnumerateKeysForPrefix( db *badger.DB, dbPrefix []byte, From d2475dab9fdb96ddf1e8a764940407b6be983981 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 23 Dec 2024 18:44:11 -0500 Subject: [PATCH 091/131] revert usage of stream api in GetBlockNodesByHeight --- lib/blockchain.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 72ef877a2..ae451bdcb 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -690,12 +690,7 @@ func (bi *BlockIndex) GetBlockNodesByHeight(height uint64) []*BlockNode { } prefixKey := _heightHashToNodePrefixByHeight(uint32(height), false) // Enumerate all block nodes for the prefix. - //_, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false, true) // skip prefetch. - _, valsFound, err := _enumerateKeysForPrefixWithStream(bi.db, prefixKey, false) - if err != nil { - glog.Errorf("GetBlockNodesByHeight: Problem enumerating keys for prefix: %v", err) - return nil - } + _, valsFound := EnumerateKeysForPrefix(bi.db, prefixKey, false, true) // skip prefetch. blockNodes := []*BlockNode{} for _, val := range valsFound { // Deserialize all block nodes. From 57a43a7121fc89c7da46b33b6217aaf817abb61a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 30 Dec 2024 13:44:45 -0500 Subject: [PATCH 092/131] reduce logging in pos_consensus.go --- lib/pos_consensus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pos_consensus.go b/lib/pos_consensus.go index 0af8df1a3..0f412bba2 100644 --- a/lib/pos_consensus.go +++ b/lib/pos_consensus.go @@ -238,8 +238,7 @@ func (fc *FastHotStuffConsensus) handleBlockProposalEvent( event *consensus.FastHotStuffEvent, expectedEventType consensus.FastHotStuffEventType, ) error { - // TODO: DELETE ME!! - glog.Infof("FastHotStuffConsensus.handleBlockProposalEvent: %s", event.ToString()) + glog.V(3).Infof("FastHotStuffConsensus.handleBlockProposalEvent: %s", event.ToString()) // Validate that the event's type is the expected proposal event type if !isValidBlockProposalEvent(event, expectedEventType) { return errors.Errorf("Unexpected event type: %v vs %v", event.EventType, expectedEventType) From 652e4e2fb3f5ef3b1ab9d289244bdbe59abe8f45 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 2 Jan 2025 15:22:51 -0500 Subject: [PATCH 093/131] add more logging --- lib/legacy_mempool.go | 2 +- lib/pos_mempool.go | 9 +++++- lib/pos_mempool_persister.go | 7 +++++ lib/server.go | 53 ++++++++++++++++++++---------------- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index d87039e32..cdef00150 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -2768,7 +2768,7 @@ func (mp *DeSoMempool) LoadTxnsFromDB() { } } endTime := time.Now() - glog.V(1).Infof("LoadTxnsFromDB: Loaded %v txns in %v seconds", len(dbMempoolTxnsOrderedByTime), endTime.Sub(startTime).Seconds()) + glog.V(0).Infof("LoadTxnsFromDB: Loaded %v txns in %v seconds", len(dbMempoolTxnsOrderedByTime), endTime.Sub(startTime).Seconds()) } func (mp *DeSoMempool) Stop() { diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 7a17864c8..b2217c121 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -2,6 +2,7 @@ package lib import ( "bytes" + "encoding/hex" "fmt" "github.com/deso-protocol/core/collections" "path/filepath" @@ -683,6 +684,7 @@ func (mp *PosMempool) loadPersistedTransactions() error { if err != nil { return errors.Wrapf(err, "PosMempool.Start: Problem retrieving transactions from persister") } + glog.V(0).Infof("PosMempool.loadPersistedTransactions: Retrieved %d transactions from persister", len(txns)) // We set the persistToDb flag to false so that persister doesn't try to save the transactions. for _, txn := range txns { if err := mp.addTransactionNoLock(txn, false); err != nil { @@ -847,7 +849,12 @@ func (mp *PosMempool) validateTransactions() error { // try to resubmit it. txn.SetValidated(false) mp.recentRejectedTxnCache.Put(*txn.Hash, err) - + txnBytes, toBytesErr := txn.Tx.ToBytes(false) + if toBytesErr != nil { + glog.Errorf("PosMempool.validateTransactions: Problem converting txn to bytes: %v", toBytesErr) + } + glog.V(0).Infof("PosMempool.validateTransactions: Removing txn %v from mempool: %v\nTxn Hex: %v", + txn.Hash, err, hex.EncodeToString(txnBytes)) // Try to remove the transaction with a lock. mp.removeTransaction(txn, true) diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index 05a1ef617..593aede42 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -168,6 +168,8 @@ func (mp *MempoolPersister) persistBatchNoLock() error { return nil } + glog.V(0).Infof("MempoolPersister: Persisting batch of %d mempool events", len(mp.updateBatch)) + // If there are no transactions to persist, return. if len(mp.updateBatch) == 0 { return nil @@ -176,6 +178,7 @@ func (mp *MempoolPersister) persistBatchNoLock() error { wb := mp.db.NewWriteBatch() defer wb.Cancel() + addEvents, removeEvents := 0, 0 for _, event := range mp.updateBatch { if event.Txn == nil || event.Txn.Hash == nil { continue @@ -194,10 +197,12 @@ func (mp *MempoolPersister) persistBatchNoLock() error { if err := wb.Set(key, value); err != nil { glog.Errorf("MempoolPersister: Error setting key: %v", err) } + addEvents++ case MempoolEventRemove: if err := wb.Delete(key); err != nil { glog.Errorf("MempoolPersister: Error deleting key: %v", err) } + removeEvents++ } } err := wb.Flush() @@ -207,6 +212,8 @@ func (mp *MempoolPersister) persistBatchNoLock() error { mp.updateBatch = nil + glog.V(0).Infof("MempoolPersister: Persisted %d add events and %d remove events", addEvents, removeEvents) + return nil } diff --git a/lib/server.go b/lib/server.go index c5f0bf4c0..03b56152c 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1041,6 +1041,8 @@ func (srv *Server) RequestBlocksUpToHeight(pp *Peer, maxHeight int) { numBlocksToFetch, maxHeight, pp.requestedBlocks, ) if len(blockNodesToFetch) == 0 { + glog.V(0).Infof("GetBlocks: No blocks to fetch from peer %v: maxBlocksInFlight: %d, peer requested blocks: %d", + pp, srv.getMaxBlocksInFlight(pp), len(pp.requestedBlocks)) // This can happen if, for example, we're already requesting the maximum // number of blocks we can. Just return in this case. return @@ -1055,7 +1057,8 @@ func (srv *Server) RequestBlocksUpToHeight(pp *Peer, maxHeight int) { pp.AddDeSoMessage(&MsgDeSoGetBlocks{HashList: hashList}, false) - glog.V(1).Infof("GetBlocks: Downloading %d blocks from header %v to header %v from peer %v", + // TODO: revert to V(1) after testing. + glog.V(0).Infof("GetBlocks: Downloading %d blocks from header %v to header %v from peer %v", len(blockNodesToFetch), blockNodesToFetch[0].Header, blockNodesToFetch[len(blockNodesToFetch)-1].Header, @@ -1203,24 +1206,24 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { headerHash, _ := headerReceived.Hash() hasHeader := srv.blockchain.HasHeaderByHashAndHeight(headerHash, headerReceived.Height) if hasHeader { - if srv.blockchain.isSyncing() { - - glog.Warningf("Server._handleHeaderBundle: Duplicate header %v received from peer %v "+ - "in state %s. Local header tip height %d "+ - "hash %s with duplicate %v", - headerHash, - pp, srv.blockchain.chainState(), srv.blockchain.headerTip().Height, - hex.EncodeToString(srv.blockchain.headerTip().Hash[:]), headerHash) - - // TODO: This logic should really be commented back in, but there was a bug that - // arises when a program is killed forcefully whereby a partial write leads to this - // logic causing the sync to stall. As such, it's more trouble than it's worth - // at the moment but we should consider being more strict about it in the future. - /* - pp.Disconnect() - return - */ - } + //if srv.blockchain.isSyncing() { + // Always log a warning if we get a duplicate header. This is useful for debugging. + glog.Warningf("Server._handleHeaderBundle: Duplicate header %v received from peer %v "+ + "in state %s. Local header tip height %d "+ + "hash %s with duplicate %v", + headerHash, + pp, srv.blockchain.chainState(), srv.blockchain.headerTip().Height, + hex.EncodeToString(srv.blockchain.headerTip().Hash[:]), headerHash) + + // TODO: This logic should really be commented back in, but there was a bug that + // arises when a program is killed forcefully whereby a partial write leads to this + // logic causing the sync to stall. As such, it's more trouble than it's worth + // at the moment but we should consider being more strict about it in the future. + /* + pp.Disconnect() + return + */ + //} // Don't process duplicate headers. continue @@ -1282,7 +1285,11 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { glog.Errorf("Server._handleHeaderBundle: Problem writing block nodes to db, error: (%v)", err) return } - glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) + if len(blockNodeBatch) > 0 { + glog.V(0).Info("Server._handleHeaderBundle: PutHeightHashToNodeInfoBatch took: ", time.Since(currTime)) + } else { + glog.V(0).Info("Server._handleHeaderBundle: No block nodes to write to db") + } // After processing all the headers this will check to see if we are fully current // and send a request to our Peer to start a Mempool sync if so. @@ -1457,7 +1464,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // are 100% positive the peer has them. hasHeader := srv.blockchain.HasHeaderByHashAndHeight(msg.TipHash, uint64(msg.TipHeight)) if !hasHeader { - glog.V(1).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ + glog.V(0).Infof("Server._handleHeaderBundle: Peer's tip is not in our "+ "blockchain so not requesting anything else from them. Our block "+ "tip %v, their tip %v:%d, peer: %v", srv.blockchain.blockTip().Header, msg.TipHash, msg.TipHeight, pp) @@ -1469,7 +1476,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // them should be available as long as they don't exceed the peer's // tip height. blockTip := srv.blockchain.blockTip() - glog.V(1).Infof("Server._handleHeaderBundle: *Downloading* blocks starting at "+ + glog.V(0).Infof("Server._handleHeaderBundle: *Downloading* blocks starting at "+ "block tip %v out of %d from peer %v", blockTip.Header, msg.TipHeight, pp) srv.RequestBlocksUpToHeight(pp, int(msg.TipHeight)) @@ -2682,7 +2689,7 @@ func (srv *Server) _handleBlockBundle(pp *Peer, bundle *MsgDeSoBlockBundle) { // TODO: We should fetch the next batch of blocks while we process this batch. // This requires us to modify GetBlocks to take a start hash and a count // of the number of blocks we want. Or we could make the existing GetBlocks - // take a start hash and the other node can just return as many blcoks as it + // take a start hash and the other node can just return as many blocks as it // can. // Process each block in the bundle. Record our blocks per second. From 2e830719f4db755a704a062dc101b791712c8bf1 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 2 Jan 2025 17:05:27 -0500 Subject: [PATCH 094/131] add more logging as suggested by DH --- lib/blockchain.go | 4 ++-- lib/network.go | 5 +++++ lib/peer.go | 9 ++++++++- lib/server.go | 44 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 55 insertions(+), 7 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index ae451bdcb..0ece1354e 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1206,7 +1206,7 @@ func NewBlockchain( // LatestHeaderLocator calls returns a block locator for the current tip of the // header chain. -func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { +func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { // We can acquire the ChainLock here because all calls to this function happen in peer.go // and server.go, which don't hold the lock. // If we do not acquire the lock, we may hit a concurrent map read write error which causes panic. @@ -1215,7 +1215,7 @@ func (bc *Blockchain) LatestHeaderLocator() []*BlockHash { headerTip := bc.headerTip() committedTip, _ := bc.GetCommittedTip() - return []*BlockHash{headerTip.Hash, committedTip.Hash} + return []*BlockHash{headerTip.Hash, committedTip.Hash}, []uint32{headerTip.Height, committedTip.Height} } func (bc *Blockchain) GetBlockNodesToFetch( diff --git a/lib/network.go b/lib/network.go index e6c0aca83..611c89e2b 100644 --- a/lib/network.go +++ b/lib/network.go @@ -2757,6 +2757,11 @@ func (msg *MsgDeSoHeader) String() string { return fmt.Sprintf("< %d, %s, %v >", msg.Height, hash, msg.Version) } +func (msg *MsgDeSoHeader) ShortString() string { + hash, _ := msg.Hash() + return fmt.Sprintf("< %d, %s >", msg.Height, hash) +} + // ================================================================== // BLOCK Message // ================================================================== diff --git a/lib/peer.go b/lib/peer.go index 1e32d8bf2..517bed0ea 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -377,7 +377,14 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { // - When the blocks come in, we process them by adding them to the chain // one-by-one. if len(blockHashList) > 0 { - locator := pp.srv.blockchain.LatestHeaderLocator() + locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() + headerTip := pp.srv.blockchain.headerTip() + blockTip := pp.srv.blockchain.blockTip() + glog.V(0).Infof("Server._handleInv: Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", + pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + blockTip.Hash, blockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, BlockLocator: locator, diff --git a/lib/server.go b/lib/server.go index 03b56152c..fa421d825 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1184,6 +1184,14 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { len(msg.Headers), srv.blockchain.chainState(), pp, srv.blockchain.headerTip().Header.Height, printHeight, srv.getCheckpointSyncingStatus(true)))) + headerStrings := collections.Transform(msg.Headers, func(header *MsgDeSoHeader) string { return header.ShortString() }) + if len(msg.Headers) < 50 { + glog.Infof("Received headers :\n %v", strings.Join(headerStrings, "\n")) + } else { + glog.Infof("Received headers :\n %v", strings.Join( + append(headerStrings[:10], headerStrings[len(headerStrings)-10:]...), "\n")) + } + // If we get here, it means that the node is not currently running a Fast-HotStuff // validator or that the node is syncing. In either case, we sync headers according // to the blocksync rules. @@ -1501,11 +1509,18 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { // even if the peer has a long fork with more work than our current header // chain. lastHash, _ := msg.Headers[len(msg.Headers)-1].Hash() + lastHeight := msg.Headers[len(msg.Headers)-1].Height + headerTip := srv.blockchain.headerTip() + currentBlockTip := srv.blockchain.blockTip() + glog.V(0).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) \n"+ + "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", + pp, lastHash, lastHeight, headerTip.Hash, headerTip.Height, + currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, BlockLocator: []*BlockHash{lastHash}, }, false) - headerTip := srv.blockchain.headerTip() glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", headerTip.Header, msg.TipHeight, pp) @@ -2019,7 +2034,14 @@ func (srv *Server) _startSync() { // Send a GetHeaders message to the Peer to start the headers sync. // Note that we include an empty BlockHash as the stopHash to indicate we want as // many headers as the Peer can give us. - locator := srv.blockchain.LatestHeaderLocator() + locator, locatorHeights := bestPeer.srv.blockchain.LatestHeaderLocator() + headerTip := bestPeer.srv.blockchain.headerTip() + currentBlockTip := bestPeer.srv.blockchain.blockTip() + glog.V(0).Infof("Server._startSync: Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", + bestPeer, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + currentBlockTip.Hash, currentBlockTip.Height) bestPeer.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, BlockLocator: locator, @@ -2606,7 +2628,14 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { glog.Warningf("Server._handleBlock: Received block while syncing headers: %v", blk) glog.Infof("Requesting headers: %v", pp) - locator := srv.blockchain.LatestHeaderLocator() + locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() + headerTip := pp.srv.blockchain.headerTip() + currentBlockTip := pp.srv.blockchain.blockTip() + glog.V(0).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", + pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, BlockLocator: locator, @@ -2650,7 +2679,14 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { // and worst case the peer will return an empty header bundle that will // result in us not sending anything back because there won’t be any new // blocks to request. - locator := srv.blockchain.LatestHeaderLocator() + locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() + headerTip := pp.srv.blockchain.headerTip() + currentBlockTip := pp.srv.blockchain.blockTip() + glog.V(0).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", + pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, BlockLocator: locator, From cd0c78e4144b968b24ff9aff147bc637d291ffaa Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 4 Jan 2025 22:38:41 -0500 Subject: [PATCH 095/131] add logging in getStoredLineageFromCommittedTip --- lib/pos_blockchain.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index a630f2130..8d279c587 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1305,25 +1305,38 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( ancestors := []*BlockNode{} childHeight := header.Height childView := header.GetView() + calledWithHeader := fmt.Sprintf("\nCalled with header for block at height %v", header.Height) for { // TODO: is currentHeight correct here? - currentBlock, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) - if !exists { + currentBlock, currentBlockExists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) + if !currentBlockExists { + glog.Errorf("getStoredLineageFromCommittedTip: Missing block %v - does not exist.%v", + currentHash, calledWithHeader) return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } if currentBlock.Hash.IsEqual(highestCommittedBlock.Hash) { break } if currentBlock.IsCommitted() { + glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) is committed. Committed tip is %v (%v). %v", + currentHash, currentHeight, highestCommittedBlock.Hash, highestCommittedBlock.Height, calledWithHeader) return nil, nil, RuleErrorDoesNotExtendCommittedTip } if currentBlock.IsValidateFailed() { + glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) has failed validation. %v", + currentHash, currentHeight, calledWithHeader) return nil, nil, RuleErrorAncestorBlockValidationFailed } if uint64(currentBlock.Header.Height)+1 != childHeight { + glog.Errorf("getStoredLineageFromCommittedTip: "+ + "Parent block height %v is not sequential with child block height %v. %v", + currentBlock.Header.Height, childHeight, calledWithHeader) return nil, nil, RuleErrorParentBlockHeightNotSequentialWithChildBlockHeight } if currentBlock.Header.GetView() >= childView { + glog.Errorf("getStoredLineageFromCommittedTip: "+ + "Parent block view %v is greater than or equal to child block view %v. %v", + currentBlock.Header.GetView(), childView, calledWithHeader) return nil, nil, RuleErrorParentBlockHasViewGreaterOrEqualToChildBlock } @@ -1332,6 +1345,8 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( // we previously saw its header. We need to request the block again from a peer and // consider it to be missing. if !currentBlock.IsStored() { + glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) is not stored. %v", + currentHash, currentHeight, calledWithHeader) return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } From 3ac6ba2f715372be3f58f953d4c1efb3544a3750 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 4 Jan 2025 22:40:15 -0500 Subject: [PATCH 096/131] add one more log --- lib/pos_blockchain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 8d279c587..b8db62866 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -258,6 +258,7 @@ func (bc *Blockchain) processBlockPoS(block *MsgDeSoBlock, currentView uint64, v return false, false, nil, errors.Wrap(err, "processBlockPoS: ") } if errors.Is(err, RuleErrorMissingAncestorBlock) { + glog.V(0).Infof("processBlockPoS: Missing ancestor block for block %v", block.Header.String()) // In this case, the block is an orphan that does not extend from any blocks // on our best chain. Try to process the orphan by running basic validations. // If it passes basic integrity checks, we'll store it with the hope that we From 87c35e493b08f72c2cd3fac56adb033c99156d45 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 7 Jan 2025 18:30:29 -0500 Subject: [PATCH 097/131] add more logging in getStoredLineage --- lib/blockchain.go | 4 ++-- lib/pos_blockchain.go | 51 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 0ece1354e..46f9a71ca 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -405,8 +405,8 @@ func (nn *BlockNode) String() string { if nn.Header != nil { tstamp = uint32(nn.Header.GetTstampSecs()) } - return fmt.Sprintf("< TstampSecs: %d, Height: %d, Hash: %s, ParentHash %s, Status: %s, CumWork: %v>", - tstamp, nn.Header.Height, nn.Hash, parentHash, nn.Status, nn.CumWork) + return fmt.Sprintf("< TstampSecs: %d, Height: %d, Hash: %s, ParentHash %s, Status: %s, CumWork: %v, IsCommitted: %v >", + tstamp, nn.Header.Height, nn.Hash, parentHash, nn.Status, nn.CumWork, nn.IsCommitted()) } // NewBlockNode is a helper function to create a BlockNode diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index b8db62866..536a498a0 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1307,12 +1307,58 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( childHeight := header.Height childView := header.GetView() calledWithHeader := fmt.Sprintf("\nCalled with header for block at height %v", header.Height) + + bigLogger := func(otherBlock *BlockNode) { + // Log HighestCommittedBlock + glog.V(0).Infof("HighestCommittedBlock: %v", highestCommittedBlock.String()) + // Get the highest committed block from the block index cache. + highestCommittedBlockFromCache, highestCommittedBlockExistsInCache := + bc.blockIndex.blockIndexByHash.Get(*highestCommittedBlock.Hash) + if highestCommittedBlockExistsInCache { + glog.V(0).Infof("HighestCommittedBlockFromCache: %v", highestCommittedBlockFromCache.String()) + } else { + glog.V(0).Infof("HighestCommittedBlockFromCache: nil") + } + + highestCommittedBlockFromDB := GetHeightHashToNodeInfo( + bc.db, bc.snapshot, highestCommittedBlock.Height, highestCommittedBlock.Hash, false) + if highestCommittedBlockFromDB != nil { + glog.V(0).Infof("HighestCommittedBlockFromDB: %v", highestCommittedBlockFromDB.String()) + } else { + glog.V(0).Infof("HighestCommittedBlockFromDB: nil") + } + + if otherBlock == nil { + glog.V(0).Infof("OtherBlock: nil") + return + } + // Log OtherBlock + glog.V(0).Infof("OtherBlock: %v", otherBlock.String()) + + // Get the other block from the block index cache. + otherBlockFromCache, otherBlockExistsInCache := + bc.blockIndex.blockIndexByHash.Get(*otherBlock.Hash) + if otherBlockExistsInCache { + glog.V(0).Infof("OtherBlockFromCache: %v", otherBlockFromCache.String()) + } else { + glog.V(0).Infof("OtherBlockFromCache: nil") + } + + otherBlockFromDB := GetHeightHashToNodeInfo( + bc.db, bc.snapshot, otherBlock.Height, otherBlock.Hash, false) + if otherBlockFromDB != nil { + glog.V(0).Infof("OtherBlockFromDB: %v", otherBlockFromDB.String()) + } else { + glog.V(0).Infof("OtherBlockFromDB: nil") + } + } for { // TODO: is currentHeight correct here? currentBlock, currentBlockExists := bc.blockIndex.GetBlockNodeByHashAndHeight(currentHash, currentHeight) if !currentBlockExists { glog.Errorf("getStoredLineageFromCommittedTip: Missing block %v - does not exist.%v", currentHash, calledWithHeader) + bigLogger(currentBlock) return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } if currentBlock.Hash.IsEqual(highestCommittedBlock.Hash) { @@ -1321,23 +1367,27 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( if currentBlock.IsCommitted() { glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) is committed. Committed tip is %v (%v). %v", currentHash, currentHeight, highestCommittedBlock.Hash, highestCommittedBlock.Height, calledWithHeader) + bigLogger(currentBlock) return nil, nil, RuleErrorDoesNotExtendCommittedTip } if currentBlock.IsValidateFailed() { glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) has failed validation. %v", currentHash, currentHeight, calledWithHeader) + bigLogger(currentBlock) return nil, nil, RuleErrorAncestorBlockValidationFailed } if uint64(currentBlock.Header.Height)+1 != childHeight { glog.Errorf("getStoredLineageFromCommittedTip: "+ "Parent block height %v is not sequential with child block height %v. %v", currentBlock.Header.Height, childHeight, calledWithHeader) + bigLogger(currentBlock) return nil, nil, RuleErrorParentBlockHeightNotSequentialWithChildBlockHeight } if currentBlock.Header.GetView() >= childView { glog.Errorf("getStoredLineageFromCommittedTip: "+ "Parent block view %v is greater than or equal to child block view %v. %v", currentBlock.Header.GetView(), childView, calledWithHeader) + bigLogger(currentBlock) return nil, nil, RuleErrorParentBlockHasViewGreaterOrEqualToChildBlock } @@ -1348,6 +1398,7 @@ func (bc *Blockchain) getStoredLineageFromCommittedTip(header *MsgDeSoHeader) ( if !currentBlock.IsStored() { glog.Errorf("getStoredLineageFromCommittedTip: Block %v (%v) is not stored. %v", currentHash, currentHeight, calledWithHeader) + bigLogger(currentBlock) return nil, []*BlockHash{currentHash}, RuleErrorMissingAncestorBlock } From 0c8606c7bdc5be2e6bc2eaa1d7d11907bcfc26dd Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:02:28 -0500 Subject: [PATCH 098/131] Ln/replace best chain plus no blocknode parent (#1439) * remove blockNode Parent attribute * patch GetBestChain * try to resolve nil pointer * clean up code --- lib/block_view.go | 4 ++-- lib/block_view_bitcoin_test.go | 1 - lib/block_view_lockups_test.go | 4 ++-- lib/blockchain.go | 32 +++++--------------------------- lib/blockchain_test.go | 8 -------- lib/constants.go | 2 -- lib/db_utils.go | 11 +++-------- lib/db_utils_test.go | 12 ++++-------- lib/pos_blockchain.go | 8 +------- lib/pos_blockchain_test.go | 16 +++++++--------- lib/postgres.go | 19 ++----------------- lib/server.go | 6 +++--- 12 files changed, 29 insertions(+), 94 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index c6b3f4f10..93dd9d919 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -5123,8 +5123,8 @@ func (bav *UtxoView) GetSpendableDeSoBalanceNanosForPublicKey(pkBytes []byte, // we only process the block reward for the previous block instead of all immature block rewards // as defined by the params. // NOTE: we are not using .GetParent here as it changes the meaning of this code. - if blockNode.Parent != nil { - nextBlockHash = blockNode.Parent.Hash + if blockNode.Header != nil { + nextBlockHash = blockNode.Header.PrevBlockHash } else { nextBlockHash = GenesisBlockHash } diff --git a/lib/block_view_bitcoin_test.go b/lib/block_view_bitcoin_test.go index c12b8af0a..b68a18efd 100644 --- a/lib/block_view_bitcoin_test.go +++ b/lib/block_view_bitcoin_test.go @@ -50,7 +50,6 @@ func GetTestParamsCopy( paramsCopy := *paramss headerHash := (BlockHash)(startHeader.BlockHash()) paramsCopy.BitcoinStartBlockNode = NewBlockNode( - nil, /*ParentNode*/ &headerHash, /*Hash*/ startHeight, _difficultyBitsToHash(startHeader.Bits), diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 48f435363..3ab91972a 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -2462,7 +2462,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk2.Header.Height)) // Update the tip - testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.Parent + testMeta.chain.blockIndex.tip = testMeta.chain.blockIndex.tip.GetParent(testMeta.chain.blockIndex) // Validate the state update utxoView = NewUtxoView( @@ -2517,7 +2517,7 @@ func TestLockupBlockConnectsAndDisconnects(t *testing.T) { require.NoError(t, utxoView.FlushToDb(blk1.Header.Height)) // Update the tip - testMeta.chain.blockIndex.setTip(testMeta.chain.blockIndex.tip.Parent) + testMeta.chain.blockIndex.setTip(testMeta.chain.blockIndex.tip.GetParent(testMeta.chain.blockIndex)) // Verify we return back to the initial state utxoView = NewUtxoView( diff --git a/lib/blockchain.go b/lib/blockchain.go index 46f9a71ca..e249627a5 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -192,9 +192,6 @@ func (blockStatus BlockStatus) String() string { // Add some fields in addition to the header to aid in the selection // of the best chain. type BlockNode struct { - // Pointer to a node representing the block's parent. - Parent *BlockNode `fake:"skip"` - // The hash computed on this block. Hash *BlockHash @@ -248,9 +245,6 @@ func (nn *BlockNode) GetEncoderType() EncoderType { } func (nn *BlockNode) GetParent(blockIndex *BlockIndex) *BlockNode { - if nn.Parent != nil { - return nn.Parent - } // If we don't have a parent, try to get it from the block index. We // no longer have a guarantee that we have set the parent node since // we no longer keep the entire block index in memory. @@ -259,7 +253,6 @@ func (nn *BlockNode) GetParent(blockIndex *BlockIndex) *BlockNode { return nil } - nn.Parent = parentNode return parentNode } @@ -395,12 +388,7 @@ func ExtractBitcoinExchangeTransactionsFromBitcoinBlock( } func (nn *BlockNode) String() string { - var parentHash *BlockHash - if nn.Parent != nil { - parentHash = nn.Parent.Hash - } else { - parentHash = nn.Header.PrevBlockHash - } + parentHash := nn.Header.PrevBlockHash tstamp := uint32(0) if nn.Header != nil { tstamp = uint32(nn.Header.GetTstampSecs()) @@ -414,7 +402,6 @@ func (nn *BlockNode) String() string { // have a committed status of COMMITTED. // TODO: Height not needed in this since it's in the header. func NewBlockNode( - parent *BlockNode, hash *BlockHash, height uint32, difficultyTarget *BlockHash, @@ -423,7 +410,6 @@ func NewBlockNode( status BlockStatus) *BlockNode { return &BlockNode{ - Parent: parent, Hash: hash, Height: height, DifficultyTarget: difficultyTarget, @@ -439,18 +425,11 @@ func (nn *BlockNode) Ancestor(height uint32, blockIndex *BlockIndex) *BlockNode } node := nn - // NOTE: using .Parent here is okay b/c it explicitly set it - // if we don't already have it when we fetch the parent from - // the block index. - for ; node != nil && node.Height != height; node = node.Parent { + for ; node != nil && node.Height != height; node = node.GetParent(blockIndex) { // Keep iterating node until the condition no longer holds. - if node.Parent == nil { - var exists bool - node.Parent, exists = blockIndex.GetBlockNodeByHashAndHeight( - node.Header.PrevBlockHash, uint64(node.Height-1)) - if !exists { - return nil - } + parent := node.GetParent(blockIndex) + if parent == nil { + return nil } } @@ -2211,7 +2190,6 @@ func (bc *Blockchain) processHeaderPoW( newWork := BytesToBigint(ExpectedWorkForBlockHash(diffTarget)[:]) cumWork := newWork.Add(newWork, parentNode.CumWork) newNode := NewBlockNode( - parentNode, headerHash, uint32(blockHeader.Height), diffTarget, diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index ef2a7d6e3..85817fb24 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -1243,7 +1243,6 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1281,7 +1280,6 @@ func TestCalcNextDifficultyTargetHalvingDoublingHitLimit(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1345,7 +1343,6 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1383,7 +1380,6 @@ func TestCalcNextDifficultyTargetHittingLimitsSlow(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1447,7 +1443,6 @@ func TestCalcNextDifficultyTargetHittingLimitsFast(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1506,7 +1501,6 @@ func TestCalcNextDifficultyTargetJustRight(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1566,7 +1560,6 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, @@ -1604,7 +1597,6 @@ func TestCalcNextDifficultyTargetSlightlyOff(t *testing.T) { nextDiff, err := bc.CalcNextDifficultyTarget(lastNode, HeaderVersion0) require.NoErrorf(err, "Block index: %d", ii) nodes = append(nodes, NewBlockNode( - lastNode, NewBlockHash(RandomBytes(32)), uint32(ii), nextDiff, diff --git a/lib/constants.go b/lib/constants.go index 9a76bf346..0d882b952 100644 --- a/lib/constants.go +++ b/lib/constants.go @@ -1160,7 +1160,6 @@ var DeSoMainnetParams = DeSoParams{ // choose and the tip. This is done by running once, letting it fail, and then rerunning // with the value it outputs. BitcoinStartBlockNode: NewBlockNode( - nil, mustDecodeHexBlockHashBitcoin("000000000000000000092d577cc673bede24b6d7199ee69c67eeb46c18fc978c"), // Note the height is always one greater than the parent node. 653184, @@ -1523,7 +1522,6 @@ var DeSoTestnetParams = DeSoParams{ // See comment in mainnet config. BitcoinStartBlockNode: NewBlockNode( - nil, mustDecodeHexBlockHashBitcoin("000000000000003aae8fb976056413aa1d863eb5bee381ff16c9642283b1da1a"), 1897056, _difficultyBitsToHash(424073553), diff --git a/lib/db_utils.go b/lib/db_utils.go index 4704e3df5..f477441e5 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -4976,7 +4976,6 @@ func SerializeBlockNode(blockNode *BlockNode) ([]byte, error) { func DeserializeBlockNode(data []byte) (*BlockNode, error) { blockNode := NewBlockNode( - nil, // Parent &BlockHash{}, // Hash 0, // Height &BlockHash{}, // DifficultyTarget @@ -5463,7 +5462,6 @@ func InitDbWithDeSoGenesisBlock(params *DeSoParams, handle *badger.DB, diffTarget := MustDecodeHexBlockHash(params.MinDifficultyTargetHex) blockHash := MustDecodeHexBlockHash(params.GenesisBlockHashHex) genesisNode := NewBlockNode( - nil, // Parent blockHash, 0, // Height diffTarget, @@ -5682,10 +5680,7 @@ func GetBlockIndex(handle *badger.DB, bitcoinNodes bool, params *DeSoParams) ( if blockNode.Height == 0 || (*blockNode.Header.PrevBlockHash == BlockHash{}) { continue } - if parent, ok := blockIndex.Get(*blockNode.Header.PrevBlockHash); ok { - // We found the parent node so connect it. - blockNode.Parent = parent - } else { + if _, ok := blockIndex.Get(*blockNode.Header.PrevBlockHash); !ok { // If we're syncing a DeSo node and we hit a PoS block, we expect there to // be orphan blocks in the block index. In this case, we don't throw an error. if bitcoinNodes == false && params.IsPoSBlockHeight(uint64(blockNode.Height)) { @@ -5867,7 +5862,7 @@ func RunBlockIndexMigration(handle *badger.DB, snapshot *Snapshot, eventManager // TODO: refactor to actually get the whole best chain if that's // what someone wants. It'll take a while and a lot of memory. -func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { +func GetBestChain(tipNode *BlockNode, blockIndex *BlockIndex) ([]*BlockNode, error) { reversedBestChain := []*BlockNode{} maxBestChainInitLength := 3600 * 100 // Cache up to 100 hours of blocks. for tipNode != nil && len(reversedBestChain) < maxBestChainInitLength { @@ -5878,7 +5873,7 @@ func GetBestChain(tipNode *BlockNode) ([]*BlockNode, error) { } reversedBestChain = append(reversedBestChain, tipNode) - tipNode = tipNode.Parent + tipNode = tipNode.GetParent(blockIndex) } bestChain := make([]*BlockNode, len(reversedBestChain)) diff --git a/lib/db_utils_test.go b/lib/db_utils_test.go index 6a6a0f208..095a1db44 100644 --- a/lib/db_utils_test.go +++ b/lib/db_utils_test.go @@ -193,15 +193,10 @@ func TestBlockNodePutGet(t *testing.T) { require.Equal(b3.Hash[:], b3Ret.Hash[:]) require.Equal(b4.Hash[:], b4Ret.Hash[:]) - // Make sure the nodes are connected properly. - require.Nil(b1Ret.Parent) - require.Equal(b2Ret.Parent, b1Ret) - require.Equal(b3Ret.Parent, b2Ret) - require.Equal(b4Ret.Parent, b1Ret) - // Check that getting the best chain works. { - bestChain, err := GetBestChain(b3Ret) + bi := NewBlockIndex(db, nil, b4) + bestChain, err := GetBestChain(b3Ret, bi) require.NoError(err) require.Len(bestChain, 3) require.Equal(b1Ret, bestChain[0]) @@ -234,7 +229,8 @@ func TestInitDbWithGenesisBlock(t *testing.T) { require.Equal(&genesisHash, genesis.Hash) // Check the bestChain. - bestChain, err := GetBestChain(genesis) + bi := NewBlockIndex(db, nil, genesis) + bestChain, err := GetBestChain(genesis, bi) require.NoError(err) require.Len(bestChain, 1) require.Equal(genesis, bestChain[0]) diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 536a498a0..81e49be49 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -1425,15 +1425,9 @@ func (bc *Blockchain) getOrCreateBlockNodeFromBlockIndex(block *MsgDeSoBlock, ha } blockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(hash, block.Header.Height) if blockNode != nil { - // If the block node already exists, we should set its parent if it doesn't have one already. - if blockNode.Parent == nil { - prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) - blockNode.Parent = prevBlockNode - } return blockNode, nil } - prevBlockNode, _ := bc.blockIndex.GetBlockNodeByHashAndHeight(block.Header.PrevBlockHash, block.Header.Height-1) - newBlockNode := NewBlockNode(prevBlockNode, hash, uint32(block.Header.Height), nil, nil, block.Header, StatusNone) + newBlockNode := NewBlockNode(hash, uint32(block.Header.Height), nil, nil, block.Header, StatusNone) bc.addNewBlockNodeToBlockIndex(newBlockNode) return newBlockNode, nil } diff --git a/lib/pos_blockchain_test.go b/lib/pos_blockchain_test.go index 81bf502aa..254ff0f91 100644 --- a/lib/pos_blockchain_test.go +++ b/lib/pos_blockchain_test.go @@ -244,7 +244,7 @@ func TestHasValidBlockHeight(t *testing.T) { bc, _, _ := NewTestBlockchain(t) hash := NewBlockHash(RandomBytes(32)) nowTimestamp := time.Now().UnixNano() - genesisBlock := NewBlockNode(nil, hash, 1, nil, nil, &MsgDeSoHeader{ + genesisBlock := NewBlockNode(hash, 1, nil, nil, &MsgDeSoHeader{ Version: 2, TstampNanoSecs: nowTimestamp - time.Minute.Nanoseconds(), Height: 1, @@ -319,12 +319,12 @@ func TestUpsertBlockAndBlockNodeToDB(t *testing.T) { t.Cleanup(resetGlobalDeSoParams) hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) - genesisNode := NewBlockNode(nil, hash1, 0, nil, nil, &MsgDeSoHeader{ + genesisNode := NewBlockNode(hash1, 0, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 0, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated) - block2 := NewBlockNode(genesisNode, hash2, 1, nil, nil, &MsgDeSoHeader{ + block2 := NewBlockNode(hash2, 1, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 1, ProposedInView: 2, @@ -458,12 +458,12 @@ func TestHasValidBlockViewPoS(t *testing.T) { bc, _, _ := NewTestBlockchain(t) hash1 := NewBlockHash(RandomBytes(32)) hash2 := NewBlockHash(RandomBytes(32)) - genesisNode := NewBlockNode(nil, hash1, 1, nil, nil, &MsgDeSoHeader{ + genesisNode := NewBlockNode(hash1, 1, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 1, ProposedInView: 1, }, StatusBlockStored|StatusBlockValidated) - block2 := NewBlockNode(genesisNode, hash2, 2, nil, nil, &MsgDeSoHeader{ + block2 := NewBlockNode(hash2, 2, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 2, ProposedInView: 2, @@ -797,7 +797,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { } t.Cleanup(resetGlobalDeSoParams) hash1 := NewBlockHash(RandomBytes(32)) - genesisNode := NewBlockNode(nil, hash1, 1, nil, nil, &MsgDeSoHeader{ + genesisNode := NewBlockNode(hash1, 1, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 1, ProposedInView: 1, @@ -832,7 +832,7 @@ func TestGetLineageFromCommittedTip(t *testing.T) { block.Header.PrevBlockHash = hash1 // add another block to the best chain. hash2 := NewBlockHash(RandomBytes(32)) - block2 := NewBlockNode(genesisNode, hash2, 2, nil, nil, &MsgDeSoHeader{ + block2 := NewBlockNode(hash2, 2, nil, nil, &MsgDeSoHeader{ Version: 2, Height: 2, ProposedInView: 2, @@ -1412,8 +1412,6 @@ func TestTryApplyNewTip(t *testing.T) { newBlockNode.Header.ProposedInView = 7 newBlockNode.Header.Height = 7 newBlockNode.Height = 7 - // Clear parent out for safety. - newBlockNode.Parent = nil require.NoError(t, err) ancestors, _, err = bc.getStoredLineageFromCommittedTip(newBlock.Header) require.NoError(t, err) diff --git a/lib/postgres.go b/lib/postgres.go index 088752833..f4da6642c 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -1296,8 +1296,8 @@ func (postgres *Postgres) UpsertBlockTx(tx *pg.Tx, blockNode *BlockNode) error { } // The genesis block has a nil parent - if blockNode.Parent != nil { - block.ParentHash = blockNode.Parent.Hash + if blockNode.Header.PrevBlockHash != nil { + block.ParentHash = blockNode.Header.PrevBlockHash } else if !blockNode.Header.PrevBlockHash.IsEqual(GenesisBlockHash) { // TODO: LN - why did I need to add this? block.ParentHash = blockNode.Header.PrevBlockHash @@ -1335,20 +1335,6 @@ func (postgres *Postgres) GetBlockIndex() (*collections.LruCache[BlockHash, *Blo }) } - // Setup parent pointers - for _, key := range blockMap.Keys() { - blockNode, _ := blockMap.Get(key) - // Genesis block has nil parent - parentHash := blockNode.Header.PrevBlockHash - if parentHash != nil { - parent, exists := blockMap.Get(*parentHash) - if !exists && blockNode.Height > 0 { - glog.Fatal("Parent block not found in block map") - } - blockNode.Parent = parent - } - } - return blockMap, nil } @@ -4097,7 +4083,6 @@ func (postgres *Postgres) InitGenesisBlock(params *DeSoParams, db *badger.DB) er diffTarget := MustDecodeHexBlockHash(params.MinDifficultyTargetHex) blockHash := MustDecodeHexBlockHash(params.GenesisBlockHashHex) genesisNode := NewBlockNode( - nil, blockHash, 0, diffTarget, diff --git a/lib/server.go b/lib/server.go index fa421d825..7c5591e84 100644 --- a/lib/server.go +++ b/lib/server.go @@ -2679,9 +2679,9 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { // and worst case the peer will return an empty header bundle that will // result in us not sending anything back because there won’t be any new // blocks to request. - locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() - headerTip := pp.srv.blockchain.headerTip() - currentBlockTip := pp.srv.blockchain.blockTip() + locator, locatorHeights := srv.blockchain.LatestHeaderLocator() + headerTip := srv.blockchain.headerTip() + currentBlockTip := srv.blockchain.blockTip() glog.V(0).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", From 844a28ed48e6527085187383fab915fed7572601 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 8 Jan 2025 14:26:51 -0500 Subject: [PATCH 099/131] Adjust log levels and only handle inv requests when chain state is fully current --- lib/peer.go | 3 ++- lib/pos_mempool_persister.go | 4 ++-- lib/server.go | 17 +++++++++-------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/peer.go b/lib/peer.go index 517bed0ea..aee7bbc67 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -395,7 +395,8 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { func (pp *Peer) HandleInv(msg *MsgDeSoInv) { // Ignore invs while we're still syncing and before we've requested // all mempool transactions from one of our peers to bootstrap. - if pp.srv.blockchain.isSyncing() { + //if pp.srv.blockchain.isSyncing() { + if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { glog.V(1).Infof("Server._handleInv: Ignoring INV while syncing from Peer %v", pp) return } diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index 593aede42..43ee7c449 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -168,7 +168,7 @@ func (mp *MempoolPersister) persistBatchNoLock() error { return nil } - glog.V(0).Infof("MempoolPersister: Persisting batch of %d mempool events", len(mp.updateBatch)) + glog.V(1).Infof("MempoolPersister: Persisting batch of %d mempool events", len(mp.updateBatch)) // If there are no transactions to persist, return. if len(mp.updateBatch) == 0 { @@ -212,7 +212,7 @@ func (mp *MempoolPersister) persistBatchNoLock() error { mp.updateBatch = nil - glog.V(0).Infof("MempoolPersister: Persisted %d add events and %d remove events", addEvents, removeEvents) + glog.V(1).Infof("MempoolPersister: Persisted %d add events and %d remove events", addEvents, removeEvents) return nil } diff --git a/lib/server.go b/lib/server.go index 7c5591e84..45235a5fd 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1184,14 +1184,15 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { len(msg.Headers), srv.blockchain.chainState(), pp, srv.blockchain.headerTip().Header.Height, printHeight, srv.getCheckpointSyncingStatus(true)))) - headerStrings := collections.Transform(msg.Headers, func(header *MsgDeSoHeader) string { return header.ShortString() }) - if len(msg.Headers) < 50 { - glog.Infof("Received headers :\n %v", strings.Join(headerStrings, "\n")) - } else { - glog.Infof("Received headers :\n %v", strings.Join( - append(headerStrings[:10], headerStrings[len(headerStrings)-10:]...), "\n")) + if glog.V(2) { + headerStrings := collections.Transform(msg.Headers, func(header *MsgDeSoHeader) string { return header.ShortString() }) + if len(msg.Headers) < 50 { + glog.V(2).Infof("Received headers :\n %v", strings.Join(headerStrings, "\n")) + } else { + glog.V(2).Infof("Received headers :\n %v", strings.Join( + append(headerStrings[:10], headerStrings[len(headerStrings)-10:]...), "\n")) + } } - // If we get here, it means that the node is not currently running a Fast-HotStuff // validator or that the node is syncing. In either case, we sync headers according // to the blocksync rules. @@ -2682,7 +2683,7 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { locator, locatorHeights := srv.blockchain.LatestHeaderLocator() headerTip := srv.blockchain.headerTip() currentBlockTip := srv.blockchain.blockTip() - glog.V(0).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ + glog.V(0).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, From d130837bbcce28053b25ef5a5eb4ff676a0a9542 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 8 Jan 2025 16:39:47 -0500 Subject: [PATCH 100/131] reduce log level, fix PG tests --- lib/blockchain.go | 7 +++---- lib/peer.go | 2 +- lib/server.go | 13 ++++++------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index e249627a5..7f0453a73 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1048,11 +1048,10 @@ func (bc *Blockchain) _initChain() error { currBlockCounter++ } } - - // We start by simply setting the chain tip and header tip to the tip node. - bc.blockIndex.setTip(tipNode) - bc.blockIndex.setHeaderTip(tipNode) } + // We start by simply setting the chain tip and header tip to the tip node. + bc.blockIndex.setTip(tipNode) + bc.blockIndex.setHeaderTip(tipNode) bc.isInitialized = true return nil diff --git a/lib/peer.go b/lib/peer.go index aee7bbc67..a47a47e6a 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -380,7 +380,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() headerTip := pp.srv.blockchain.headerTip() blockTip := pp.srv.blockchain.blockTip() - glog.V(0).Infof("Server._handleInv: Sending GET_HEADERS message to peer %v\n"+ + glog.V(2).Infof("Server._handleInv: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, diff --git a/lib/server.go b/lib/server.go index 45235a5fd..268ba983a 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1041,7 +1041,7 @@ func (srv *Server) RequestBlocksUpToHeight(pp *Peer, maxHeight int) { numBlocksToFetch, maxHeight, pp.requestedBlocks, ) if len(blockNodesToFetch) == 0 { - glog.V(0).Infof("GetBlocks: No blocks to fetch from peer %v: maxBlocksInFlight: %d, peer requested blocks: %d", + glog.V(1).Infof("RequestBlocksUpToHeight: No blocks to fetch from peer %v: maxBlocksInFlight: %d, peer requested blocks: %d", pp, srv.getMaxBlocksInFlight(pp), len(pp.requestedBlocks)) // This can happen if, for example, we're already requesting the maximum // number of blocks we can. Just return in this case. @@ -1057,8 +1057,7 @@ func (srv *Server) RequestBlocksUpToHeight(pp *Peer, maxHeight int) { pp.AddDeSoMessage(&MsgDeSoGetBlocks{HashList: hashList}, false) - // TODO: revert to V(1) after testing. - glog.V(0).Infof("GetBlocks: Downloading %d blocks from header %v to header %v from peer %v", + glog.V(1).Infof("RequestBlocksUpToHeight: Downloading %d blocks from header %v to header %v from peer %v", len(blockNodesToFetch), blockNodesToFetch[0].Header, blockNodesToFetch[len(blockNodesToFetch)-1].Header, @@ -1513,7 +1512,7 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { lastHeight := msg.Headers[len(msg.Headers)-1].Height headerTip := srv.blockchain.headerTip() currentBlockTip := srv.blockchain.blockTip() - glog.V(0).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ + glog.V(2).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) \n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", pp, lastHash, lastHeight, headerTip.Hash, headerTip.Height, @@ -2038,7 +2037,7 @@ func (srv *Server) _startSync() { locator, locatorHeights := bestPeer.srv.blockchain.LatestHeaderLocator() headerTip := bestPeer.srv.blockchain.headerTip() currentBlockTip := bestPeer.srv.blockchain.blockTip() - glog.V(0).Infof("Server._startSync: Sending GET_HEADERS message to peer %v\n"+ + glog.V(2).Infof("Server._startSync: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", bestPeer, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, @@ -2632,7 +2631,7 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() headerTip := pp.srv.blockchain.headerTip() currentBlockTip := pp.srv.blockchain.blockTip() - glog.V(0).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ + glog.V(2).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, @@ -2683,7 +2682,7 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { locator, locatorHeights := srv.blockchain.LatestHeaderLocator() headerTip := srv.blockchain.headerTip() currentBlockTip := srv.blockchain.blockTip() - glog.V(0).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ + glog.V(2).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, From 36b605ef13142c54b3d1850a09ae921947805b09 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 8 Jan 2025 16:46:03 -0500 Subject: [PATCH 101/131] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 801cfd02d..b33f9ec7a 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/dgraph-io/ristretto v0.2.0 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.14.11 github.com/fatih/color v1.17.0 @@ -74,7 +75,6 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/dgraph-io/ristretto v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.0 // indirect From 0a9499e0003c8d88a1078c1e044c2852b43d453a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 8 Jan 2025 17:13:49 -0500 Subject: [PATCH 102/131] reduce size of walkback in initChain to 6 hours --- lib/blockchain.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 7f0453a73..896cb45d4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1039,11 +1039,10 @@ func (bc *Blockchain) _initChain() error { if !tipNodeExists { return fmt.Errorf("_initChain: Best hash (%#v) not found in block index", bestBlockHash) } - // @diamondhands - we could reduce this to just the last hour if we want. - // Walk back the last 24 hours of blocks. + // Walk back the last 6 hours of blocks. currBlockCounter := 1 parentNode := tipNode.GetParent(bc.blockIndex) - for currBlockCounter < 3600*24 && parentNode != nil { + for currBlockCounter < 3600*6 && parentNode != nil { parentNode = parentNode.GetParent(bc.blockIndex) currBlockCounter++ } From 11773dd009c8331fb5b2487ca399befc677c581d Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 9 Jan 2025 13:01:42 -0500 Subject: [PATCH 103/131] update GetSpendableDeSoBalanceNanosForPublicKey --- lib/block_view.go | 89 +++-------------------------------------------- 1 file changed, 4 insertions(+), 85 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 93dd9d919..89411a998 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -5054,95 +5054,14 @@ func (bav *UtxoView) GetUnspentUtxoEntrysForPublicKey(pkBytes []byte) ([]*UtxoEn // but should be fixed soon. func (bav *UtxoView) GetSpendableDeSoBalanceNanosForPublicKey(pkBytes []byte, tipHeight uint32) (_spendableBalance uint64, _err error) { - // After the cut-over to Proof Of Stake, we no longer check for immature block rewards. - // All block rewards are immediately mature. - if tipHeight >= bav.Params.ForkHeights.ProofOfStake2ConsensusCutoverBlockHeight { - balanceNanos, err := bav.GetDeSoBalanceNanosForPublicKey(pkBytes) - if err != nil { - return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: ") - } - return balanceNanos, nil - } - // In order to get the spendable balance, we need to account for any immature block rewards. - // We get these by starting at the chain tip and iterating backwards until we have collected - // all the immature block rewards for this public key. - nextBlockHash := bav.TipHash - numImmatureBlocks := uint32(bav.Params.BlockRewardMaturity / bav.Params.TimeBetweenBlocks) - immatureBlockRewards := uint64(0) - - if bav.Postgres != nil { - // Note: badger is only getting the block reward for the previous block, so we make postgres - // do the same thing. This is not ideal, but it is the simplest way to get the same behavior - // and we will address the issue soon. - // Filter out immature block rewards in postgres. UtxoType needs to be set correctly when importing blocks - var startHeight uint32 - if tipHeight > 0 { - startHeight = tipHeight - 1 - } - // This is a special case to support tests where the number of immature blocks is 0. - if numImmatureBlocks == 0 { - startHeight = tipHeight - } - outputs := bav.Postgres.GetBlockRewardsForPublicKey(NewPublicKey(pkBytes), startHeight, tipHeight) - - var err error - for _, output := range outputs { - immatureBlockRewards, err = SafeUint64().Add(immatureBlockRewards, output.AmountNanos) - if err != nil { - return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem "+ - "adding immature block rewards") - } - } - } else { - for ii := uint64(1); ii < uint64(numImmatureBlocks); ii++ { - // Don't look up the genesis block since it isn't in the DB. - if GenesisBlockHashHex == nextBlockHash.String() { - break - } - - blockNode := GetHeightHashToNodeInfo(bav.Handle, bav.Snapshot, tipHeight, nextBlockHash, false) - if blockNode == nil { - return 0, fmt.Errorf( - "GetSpendableDeSoBalanceNanosForPublicKey: Problem getting block for blockhash %s", - nextBlockHash.String()) - } - blockRewardForPK, err := DbGetBlockRewardForPublicKeyBlockHash(bav.Handle, bav.Snapshot, pkBytes, nextBlockHash) - if err != nil { - return 0, errors.Wrapf( - err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem getting block reward for "+ - "public key %s blockhash %s", PkToString(pkBytes, bav.Params), nextBlockHash.String()) - } - immatureBlockRewards, err = SafeUint64().Add(immatureBlockRewards, blockRewardForPK) - if err != nil { - return 0, errors.Wrapf(err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem adding "+ - "block reward (%d) to immature block rewards (%d)", blockRewardForPK, immatureBlockRewards) - } - // TODO: This is the specific line that causes the bug. We should be using blockNode.Header.PrevBlockHash - // instead. We are not serializing the Parent attribute when the block node is put into the DB, - // but we do have the header. As a result, this condition always evaluates to false and thus - // we only process the block reward for the previous block instead of all immature block rewards - // as defined by the params. - // NOTE: we are not using .GetParent here as it changes the meaning of this code. - if blockNode.Header != nil { - nextBlockHash = blockNode.Header.PrevBlockHash - } else { - nextBlockHash = GenesisBlockHash - } - } - } - + // This function used to look up immature block rewards and reduce the total balance + // by the immature block rewards to compute the spendable balance. However, this + // is no longer necessary since the move to PoS. balanceNanos, err := bav.GetDeSoBalanceNanosForPublicKey(pkBytes) - if err != nil { return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: ") } - spendableBalanceNanos, err := SafeUint64().Sub(balanceNanos, immatureBlockRewards) - if err != nil { - return 0, errors.Wrapf(err, - "GetSpendableDeSoBalanceNanosForPublicKey: error subtract immature block rewards (%d) from "+ - "balance nanos (%d)", immatureBlockRewards, balanceNanos) - } - return spendableBalanceNanos, nil + return balanceNanos, nil } func copyExtraData(extraData map[string][]byte) map[string][]byte { From 587ac72d11e9fd389a2b3725a75d022a98e9f32a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 9 Jan 2025 13:26:50 -0500 Subject: [PATCH 104/131] fix GetSpendableUtxosForPublicKey --- lib/blockchain.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 896cb45d4..a2d4820e7 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -3282,10 +3282,11 @@ func (bc *Blockchain) GetSpendableUtxosForPublicKey(spendPublicKeyBytes []byte, // // Note we add one to the current block height since it is presumed this // transaction will at best be mined into the next block. - blockHeight := bc.blockTip().Height + 1 - if _isEntryImmatureBlockReward(utxoEntry, blockHeight, bc.params) { - continue - } + // We no longer need to worry about immature block rewards. + //blockHeight := bc.blockTip().Height + 1 + //if _isEntryImmatureBlockReward(utxoEntry, blockHeight, bc.params) { + // continue + //} // Don't consider utxos that are already consumed by the mempool. if !isInterfaceValueNil(mempool) && mempool.CheckSpend(*utxoEntry.UtxoKey) != nil { From 05a2ae371f6bed9c0a011235897bfa60977c27b6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 9 Jan 2025 13:35:59 -0500 Subject: [PATCH 105/131] revert changes for get spendable balance and utxos. update get spendable balance to not use Header.PrevBlockHash --- lib/block_view.go | 91 ++++++++++++++++++++++++++++++++++++++++++++--- lib/blockchain.go | 9 +++-- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 89411a998..40eac24e9 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -5054,14 +5054,97 @@ func (bav *UtxoView) GetUnspentUtxoEntrysForPublicKey(pkBytes []byte) ([]*UtxoEn // but should be fixed soon. func (bav *UtxoView) GetSpendableDeSoBalanceNanosForPublicKey(pkBytes []byte, tipHeight uint32) (_spendableBalance uint64, _err error) { - // This function used to look up immature block rewards and reduce the total balance - // by the immature block rewards to compute the spendable balance. However, this - // is no longer necessary since the move to PoS. + // After the cut-over to Proof Of Stake, we no longer check for immature block rewards. + // All block rewards are immediately mature. + if tipHeight >= bav.Params.ForkHeights.ProofOfStake2ConsensusCutoverBlockHeight { + balanceNanos, err := bav.GetDeSoBalanceNanosForPublicKey(pkBytes) + if err != nil { + return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: ") + } + return balanceNanos, nil + } + // In order to get the spendable balance, we need to account for any immature block rewards. + // We get these by starting at the chain tip and iterating backwards until we have collected + // all the immature block rewards for this public key. + nextBlockHash := bav.TipHash + numImmatureBlocks := uint32(bav.Params.BlockRewardMaturity / bav.Params.TimeBetweenBlocks) + immatureBlockRewards := uint64(0) + + if bav.Postgres != nil { + // Note: badger is only getting the block reward for the previous block, so we make postgres + // do the same thing. This is not ideal, but it is the simplest way to get the same behavior + // and we will address the issue soon. + // Filter out immature block rewards in postgres. UtxoType needs to be set correctly when importing blocks + var startHeight uint32 + if tipHeight > 0 { + startHeight = tipHeight - 1 + } + // This is a special case to support tests where the number of immature blocks is 0. + if numImmatureBlocks == 0 { + startHeight = tipHeight + } + outputs := bav.Postgres.GetBlockRewardsForPublicKey(NewPublicKey(pkBytes), startHeight, tipHeight) + + var err error + for _, output := range outputs { + immatureBlockRewards, err = SafeUint64().Add(immatureBlockRewards, output.AmountNanos) + if err != nil { + return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem "+ + "adding immature block rewards") + } + } + } else { + for ii := uint64(1); ii < uint64(numImmatureBlocks); ii++ { + // Don't look up the genesis block since it isn't in the DB. + if GenesisBlockHashHex == nextBlockHash.String() { + break + } + + blockNode := GetHeightHashToNodeInfo(bav.Handle, bav.Snapshot, tipHeight, nextBlockHash, false) + if blockNode == nil { + return 0, fmt.Errorf( + "GetSpendableDeSoBalanceNanosForPublicKey: Problem getting block for blockhash %s", + nextBlockHash.String()) + } + blockRewardForPK, err := DbGetBlockRewardForPublicKeyBlockHash(bav.Handle, bav.Snapshot, pkBytes, nextBlockHash) + if err != nil { + return 0, errors.Wrapf( + err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem getting block reward for "+ + "public key %s blockhash %s", PkToString(pkBytes, bav.Params), nextBlockHash.String()) + } + immatureBlockRewards, err = SafeUint64().Add(immatureBlockRewards, blockRewardForPK) + if err != nil { + return 0, errors.Wrapf(err, "GetSpendableDeSoBalanceNanosForPublicKey: Problem adding "+ + "block reward (%d) to immature block rewards (%d)", blockRewardForPK, immatureBlockRewards) + } + // TODO: This is the specific line that causes the bug. We should be using blockNode.Header.PrevBlockHash + // instead. We are not serializing the Parent attribute when the block node is put into the DB, + // but we do have the header. As a result, this condition always evaluates to false and thus + // we only process the block reward for the previous block instead of all immature block rewards + // as defined by the params. + // NOTE: we are not using .GetParent here as it changes the meaning of this code. + // In order to minimize code changes, we just jump back to the genesis block as the code did previously + // since the Parent attribute was removed from the BlockNode struct. + //if blockNode.Header != nil { + // nextBlockHash = blockNode.Header.PrevBlockHash + //} else { + nextBlockHash = GenesisBlockHash + //} + } + } + balanceNanos, err := bav.GetDeSoBalanceNanosForPublicKey(pkBytes) + if err != nil { return 0, errors.Wrap(err, "GetSpendableDeSoBalanceNanosForPublicKey: ") } - return balanceNanos, nil + spendableBalanceNanos, err := SafeUint64().Sub(balanceNanos, immatureBlockRewards) + if err != nil { + return 0, errors.Wrapf(err, + "GetSpendableDeSoBalanceNanosForPublicKey: error subtract immature block rewards (%d) from "+ + "balance nanos (%d)", immatureBlockRewards, balanceNanos) + } + return spendableBalanceNanos, nil } func copyExtraData(extraData map[string][]byte) map[string][]byte { diff --git a/lib/blockchain.go b/lib/blockchain.go index a2d4820e7..896cb45d4 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -3282,11 +3282,10 @@ func (bc *Blockchain) GetSpendableUtxosForPublicKey(spendPublicKeyBytes []byte, // // Note we add one to the current block height since it is presumed this // transaction will at best be mined into the next block. - // We no longer need to worry about immature block rewards. - //blockHeight := bc.blockTip().Height + 1 - //if _isEntryImmatureBlockReward(utxoEntry, blockHeight, bc.params) { - // continue - //} + blockHeight := bc.blockTip().Height + 1 + if _isEntryImmatureBlockReward(utxoEntry, blockHeight, bc.params) { + continue + } // Don't consider utxos that are already consumed by the mempool. if !isInterfaceValueNil(mempool) && mempool.CheckSpend(*utxoEntry.UtxoKey) != nil { From f9635706558fcdebae67ba8ae0f3ce538c56b92c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 20 Jan 2025 16:00:04 -0500 Subject: [PATCH 106/131] update some error messages --- lib/blockchain.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 896cb45d4..4d51a183e 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1227,7 +1227,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( currentHeight := heightLimit maxNode, maxNodeExists, maxNodeError := bc.GetBlockFromBestChainByHeight(currentHeight, true) if maxNodeError != nil { - glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", maxNodeError) + glog.Errorf("GetBlockToFetch: Problem getting maxNode block by height: %v", maxNodeError) return nil } if !maxNodeExists || maxNode == nil { @@ -1241,7 +1241,8 @@ func (bc *Blockchain) GetBlockNodesToFetch( backtrackingNode, backtrackingNodeExists, backtrackingNodeError := bc.GetBlockFromBestChainByHashAndOptionalHeight(currentHash, ¤tHeight, true) if backtrackingNodeError != nil { - glog.Errorf("GetBlockToFetch: Problem getting block by height: %v", backtrackingNodeError) + glog.Errorf("GetBlockToFetch: Problem getting block by hash and height (%v, %v): %v", + currentHash, currentHeight, backtrackingNodeError) return nil } if !backtrackingNodeExists || backtrackingNode == nil { From 8d8bb6dd3e002d5d3cb958d6912e3b99d73a9277 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 20 Jan 2025 16:11:04 -0500 Subject: [PATCH 107/131] switch HandleInv to respond based on isSyncing function rather than chain state = fully current --- lib/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/peer.go b/lib/peer.go index a47a47e6a..5a08cf98e 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -395,8 +395,8 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { func (pp *Peer) HandleInv(msg *MsgDeSoInv) { // Ignore invs while we're still syncing and before we've requested // all mempool transactions from one of our peers to bootstrap. - //if pp.srv.blockchain.isSyncing() { - if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { + if pp.srv.blockchain.isSyncing() { + //if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { glog.V(1).Infof("Server._handleInv: Ignoring INV while syncing from Peer %v", pp) return } From db8469b439c781ec97d37e45cd6da6b40529ec41 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 20 Jan 2025 16:32:05 -0500 Subject: [PATCH 108/131] skipping invs for peer - log level 3 --- lib/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server.go b/lib/server.go index 268ba983a..a59a52190 100644 --- a/lib/server.go +++ b/lib/server.go @@ -2197,7 +2197,7 @@ func (srv *Server) _relayTransactions() { for _, pp := range allPeers { if !pp.canReceiveInvMessages { - glog.V(2).Infof("Skipping invs for peer %v because not ready "+ + glog.V(3).Infof("Skipping invs for peer %v because not ready "+ "yet: %v", pp, pp.canReceiveInvMessages) continue } From f2bf0d77b71d18ccb01a25466c66714ab7a2a9c3 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 24 Jan 2025 13:59:12 -0500 Subject: [PATCH 109/131] loop over locators in GetHeadersForLocatorAndStopHash --- lib/server.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/server.go b/lib/server.go index a59a52190..485e7ba3a 100644 --- a/lib/server.go +++ b/lib/server.go @@ -808,10 +808,22 @@ func (srv *Server) GetHeadersForLocatorAndStopHash( } return []*MsgDeSoHeader{stopNode.Header}, nil } - startNode, startNodeExists, startNodeError := srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(locator[0], nil, true) - if startNodeError != nil || !startNodeExists || startNode == nil { - return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: Start hash provided but no start node found") + var startNode *BlockNode + var startNodeExists bool + var startNodeError error + for _, blockNodeHash := range locator { + startNode, startNodeExists, startNodeError = srv.blockchain.GetBlockFromBestChainByHashAndOptionalHeight(blockNodeHash, nil, true) + if startNodeError != nil || !startNodeExists || startNode == nil { + glog.Errorf("GetHeadersForLocatorAndStopHash: locator provided but no block node found at %v", blockNodeHash) + } + if startNodeExists && startNode != nil { + break + } } + if startNode == nil { + return nil, fmt.Errorf("GetHeadersForLocatorAndStopHash: No start node found after looping through locators") + } + var backtrackingNode *BlockNode var backtrackingNodeExists bool var backtrackingNodeError error From ca638f967937b9c59132d872299a4eed59b96998 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 28 Jan 2025 23:54:22 -0500 Subject: [PATCH 110/131] switch handleInv to only when fully current --- lib/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/peer.go b/lib/peer.go index 5a08cf98e..a47a47e6a 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -395,8 +395,8 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { func (pp *Peer) HandleInv(msg *MsgDeSoInv) { // Ignore invs while we're still syncing and before we've requested // all mempool transactions from one of our peers to bootstrap. - if pp.srv.blockchain.isSyncing() { - //if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { + //if pp.srv.blockchain.isSyncing() { + if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { glog.V(1).Infof("Server._handleInv: Ignoring INV while syncing from Peer %v", pp) return } From ea85d4e8bd934bd30667e2f7476ac71af75b91b5 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 30 Jan 2025 02:45:01 -0500 Subject: [PATCH 111/131] skip update after connect/disconnect block hooks in legacy mempool if block is pos block --- lib/legacy_mempool.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index cdef00150..d489f1f7d 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -388,6 +388,9 @@ func (mp *DeSoMempool) resetPool(newPool *DeSoMempool) { // TODO: This is fairly inefficient but the story is the same as for // UpdateAfterDisconnectBlock. func (mp *DeSoMempool) UpdateAfterConnectBlock(blk *MsgDeSoBlock) (_txnsAddedToMempool []*MempoolTx) { + if mp.bc.params.IsPoSBlockHeight(blk.Header.Height) { + return + } // Protect concurrent access. mp.mtx.Lock() defer mp.mtx.Unlock() @@ -501,6 +504,9 @@ func (mp *DeSoMempool) UpdateAfterConnectBlock(blk *MsgDeSoBlock) (_txnsAddedToM // But until then doing it this way significantly reduces complexity and should hold up // for a while. func (mp *DeSoMempool) UpdateAfterDisconnectBlock(blk *MsgDeSoBlock) { + if mp.bc.params.IsPoSBlockHeight(blk.Header.Height) { + return + } // Protect concurrent access. mp.mtx.Lock() defer mp.mtx.Unlock() From e73d998f4b45b7fe384c77f99bfbed4ab2816bfe Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 15 Feb 2025 16:33:55 -0500 Subject: [PATCH 112/131] update GetBlockNodesToFetch to use GetParent --- lib/blockchain.go | 51 +++++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 4d51a183e..def4969f0 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1201,7 +1201,7 @@ func (bc *Blockchain) GetBlockNodesToFetch( // Get the tip of the main block chain. bestBlockTip, ok := bc.GetCommittedTip() if !ok { - glog.Errorf("GetBlockToFetch: Problem getting best block tip") + glog.Errorf("GetBlockNodesToFetch: Problem getting best block tip") return nil } @@ -1225,45 +1225,34 @@ func (bc *Blockchain) GetBlockNodesToFetch( heightLimit = bestBlockTip.Header.Height + uint64(numBlocks) } currentHeight := heightLimit - maxNode, maxNodeExists, maxNodeError := bc.GetBlockFromBestChainByHeight(currentHeight, true) + backtrackingNode, backtrackingNodeExists, maxNodeError := bc.GetBlockFromBestChainByHeight(currentHeight, true) if maxNodeError != nil { - glog.Errorf("GetBlockToFetch: Problem getting maxNode block by height: %v", maxNodeError) + glog.Errorf("GetBlockNodesToFetch: Problem getting maxNode block by height: %v", maxNodeError) return nil } - if !maxNodeExists || maxNode == nil { - glog.Errorf("GetBlockToFetch: Block at height %d not found", heightLimit) + if !backtrackingNodeExists || backtrackingNode == nil { + glog.Errorf("GetBlockNodesToFetch: Block at height %d not found. Error finding max node.", heightLimit) return nil } - currentHash := maxNode.Hash // Walk back from the maxNode to the bestBlockTip. for len(blockNodesToFetch) < numBlocks && - currentHeight >= uint64(bestBlockTip.Height) { - backtrackingNode, backtrackingNodeExists, backtrackingNodeError := - bc.GetBlockFromBestChainByHashAndOptionalHeight(currentHash, ¤tHeight, true) - if backtrackingNodeError != nil { - glog.Errorf("GetBlockToFetch: Problem getting block by hash and height (%v, %v): %v", - currentHash, currentHeight, backtrackingNodeError) + backtrackingNode.Height > bestBlockTip.Height { + // only include blocks we're not supposed to ignore and that are not already stored. + if _, exists := blocksToIgnore[*backtrackingNode.Hash]; !exists && !backtrackingNode.IsStored() { + blockNodesToFetch = append(blockNodesToFetch, backtrackingNode) + } + // Get these values before we try to update the back tracking node. + prevHeight := backtrackingNode.Height + prevHash := backtrackingNode.Hash + prevParentHash := backtrackingNode.Header.PrevBlockHash + backtrackingNode = backtrackingNode.GetParent(bc.blockIndex) + if backtrackingNode == nil { + glog.Errorf("GetBlockNodesToFetch: Parent of block (%v, %d) not found with back tracking. Parent hash %v", + prevHash, + prevHeight, + prevParentHash) return nil } - if !backtrackingNodeExists || backtrackingNode == nil { - glog.Errorf("GetBlockToFetch: Block at height %d not found", currentHeight) - return nil - } - - currentHeight-- - currentHash = backtrackingNode.Header.PrevBlockHash - - // Exclude any blocks we're supposed to ignore. - if _, exists := blocksToIgnore[*backtrackingNode.Hash]; exists { - continue - } - - // Skip any stored blocks as we already have them. - if backtrackingNode.IsStored() { - continue - } - - blockNodesToFetch = append(blockNodesToFetch, backtrackingNode) } slices.Reverse(blockNodesToFetch) From ca1d91b3071b88d85ca7b2b2643214f49c9be799 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 20 Feb 2025 17:53:15 -0500 Subject: [PATCH 113/131] Clean up --- integration_testing/connection_bridge.go | 2 +- lib/remote_node.go | 2 +- scripts/mempool/mempool_dumper.go | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/integration_testing/connection_bridge.go b/integration_testing/connection_bridge.go index cc79f19b3..b5a194d09 100644 --- a/integration_testing/connection_bridge.go +++ b/integration_testing/connection_bridge.go @@ -103,7 +103,7 @@ func (bridge *ConnectionBridge) createInboundConnection(node *cmd.Node) *lib.Pee panic(err) } - ip, _, err := net.ParseCIDR(netAddress.IP.String()) + ip, _, err := net.ParseCIDR(netAddress.ToLegacy().IP.String()) if err != nil { panic(err) } diff --git a/lib/remote_node.go b/lib/remote_node.go index 1fc832991..b81ce41f8 100644 --- a/lib/remote_node.go +++ b/lib/remote_node.go @@ -409,7 +409,7 @@ func (rn *RemoteNode) Disconnect(disconnectReason string) { func (rn *RemoteNode) SendMessage(desoMsg DeSoMessage) error { rn.mtx.RLock() - rn.mtx.RUnlock() + defer rn.mtx.RUnlock() if rn.connectionStatus != RemoteNodeStatus_HandshakeCompleted { return fmt.Errorf("SendMessage: Remote node is not connected") diff --git a/scripts/mempool/mempool_dumper.go b/scripts/mempool/mempool_dumper.go index ce4e809d8..457dcef87 100644 --- a/scripts/mempool/mempool_dumper.go +++ b/scripts/mempool/mempool_dumper.go @@ -56,7 +56,7 @@ func main() { } netAddr2 := net.TCPAddr{ - IP: netAddrss.IP, + IP: netAddrss.ToLegacy().IP, Port: int(netAddrss.Port), } conn, err := net.DialTimeout(netAddr2.Network(), netAddr2.String(), lib.DeSoMainnetParams.DialTimeout) @@ -69,9 +69,7 @@ func main() { 10000, 0, &lib.DeSoMainnetParams, messagesFromPeer, nil, nil, lib.NodeSyncTypeAny, nil) time.Sleep(1 * time.Second) - if err := peer.NegotiateVersion(lib.DeSoMainnetParams.VersionNegotiationTimeout); err != nil { - panic(err) - } + peer.Start() // As a test, send a GetHeaders request and see if we get it back if *flagCommand == "get_headers" { From 301891fe3286902a73f4f4c387c35f0d8b98b76a Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 24 Feb 2025 15:03:57 -0500 Subject: [PATCH 114/131] revert HandleInv to use isSyncing instead of checking for SyncStateFullyCurrent --- lib/peer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/peer.go b/lib/peer.go index a47a47e6a..f88735a4c 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -395,8 +395,7 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { func (pp *Peer) HandleInv(msg *MsgDeSoInv) { // Ignore invs while we're still syncing and before we've requested // all mempool transactions from one of our peers to bootstrap. - //if pp.srv.blockchain.isSyncing() { - if pp.srv.blockchain.chainState() != SyncStateFullyCurrent { + if pp.srv.blockchain.isSyncing() { glog.V(1).Infof("Server._handleInv: Ignoring INV while syncing from Peer %v", pp) return } From 04375bea18e8f606aa9942ad8db15514f2fc48bd Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 4 Mar 2025 14:52:48 -0500 Subject: [PATCH 115/131] fix IsFullyStored --- lib/blockchain.go | 16 ++++++++++++++-- lib/txindex.go | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index def4969f0..54effc73c 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -906,7 +906,7 @@ func (bc *Blockchain) IsFullyStored() bool { // TODO: figure out how to iterate over best chain w/o having entire thing in memory. chainState := bc.ChainState() if chainState != SyncStateFullyCurrent && !(chainState == SyncStateNeedBlocksss && - bc.headerTip().Height-bc.blockTip().Height >= 10) { + bc.headerTip().Height-bc.blockTip().Height <= 25) { return false } // Get a sampling of blocks from the best chain and check if they are fully stored. @@ -1191,8 +1191,20 @@ func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { defer bc.ChainLock.RUnlock() headerTip := bc.headerTip() committedTip, _ := bc.GetCommittedTip() + currNode := headerTip + hashes := []*BlockHash{headerTip.Hash} + heights := []uint32{headerTip.Height} + for !currNode.Hash.IsEqual(committedTip.Hash) { + currNode = currNode.GetParent(bc.blockIndex) + if currNode == nil { + glog.Errorf("LatestHeaderLocator: Block node not found for hash %v", currNode.Hash) + break + } + hashes = append(hashes, currNode.Hash) + heights = append(heights, currNode.Height) + } - return []*BlockHash{headerTip.Hash, committedTip.Hash}, []uint32{headerTip.Height, committedTip.Height} + return hashes, heights } func (bc *Blockchain) GetBlockNodesToFetch( diff --git a/lib/txindex.go b/lib/txindex.go index cd3916f95..3fadfae8a 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -172,7 +172,8 @@ func (txi *TXIndex) Start() { return default: chainState := txi.CoreChain.ChainState() - if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && txi.CoreChain.headerTip().Height-txi.CoreChain.blockTip().Height < 10) { + if chainState == SyncStateFullyCurrent || (chainState == SyncStateNeedBlocksss && + txi.CoreChain.headerTip().Height-txi.CoreChain.blockTip().Height < 25) { if !txi.CoreChain.IsFullyStored() { glog.V(1).Infof("TXIndex: Waiting, blockchain is not fully stored") break From d0428c2bdb2d85d54a4c7875a8185b090f544863 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 4 Mar 2025 15:06:05 -0500 Subject: [PATCH 116/131] Revert LatestHeaderLocator changes --- lib/blockchain.go | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 54effc73c..34a6b2adf 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1191,20 +1191,21 @@ func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { defer bc.ChainLock.RUnlock() headerTip := bc.headerTip() committedTip, _ := bc.GetCommittedTip() - currNode := headerTip - hashes := []*BlockHash{headerTip.Hash} - heights := []uint32{headerTip.Height} - for !currNode.Hash.IsEqual(committedTip.Hash) { - currNode = currNode.GetParent(bc.blockIndex) - if currNode == nil { - glog.Errorf("LatestHeaderLocator: Block node not found for hash %v", currNode.Hash) - break - } - hashes = append(hashes, currNode.Hash) - heights = append(heights, currNode.Height) - } - - return hashes, heights + return []*BlockHash{headerTip.Hash, committedTip.Hash}, []uint32{headerTip.Height, committedTip.Height} + //currNode := headerTip + //hashes := []*BlockHash{headerTip.Hash} + //heights := []uint32{headerTip.Height} + //for !currNode.Hash.IsEqual(committedTip.Hash) { + // currNode = currNode.GetParent(bc.blockIndex) + // if currNode == nil { + // glog.Errorf("LatestHeaderLocator: Block node not found for hash %v", currNode.Hash) + // break + // } + // hashes = append(hashes, currNode.Hash) + // heights = append(heights, currNode.Height) + //} + // + //return hashes, heights } func (bc *Blockchain) GetBlockNodesToFetch( From d748ca6f9c63109c3b74fc630fa0d057989513e4 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 10 Mar 2025 15:45:54 -0400 Subject: [PATCH 117/131] more closely replicate old LatestLocator function, also use this function in new HeaderLocatorWithNodeHashAndHeight which replaces HeaderLocatorWithNodeHash --- lib/blockchain.go | 133 ++++++++++++++++++++++++++++++++++++++++++---- lib/peer.go | 4 +- lib/server.go | 29 ++++++---- 3 files changed, 144 insertions(+), 22 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 34a6b2adf..83822ad4d 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1181,17 +1181,103 @@ func NewBlockchain( return bc, nil } -// LatestHeaderLocator calls returns a block locator for the current tip of the -// header chain. -func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { - // We can acquire the ChainLock here because all calls to this function happen in peer.go - // and server.go, which don't hold the lock. - // If we do not acquire the lock, we may hit a concurrent map read write error which causes panic. - bc.ChainLock.RLock() - defer bc.ChainLock.RUnlock() - headerTip := bc.headerTip() +// LatestLocator returns a block locator for the passed block node. Holding a chain +// lock isn't strictly necessary anymore. +// +// Leaving comment below as it provides context on the implementation of this +// function prior to the changes that core codebase in which the representation +// of the best chain and block index is not in memory. +// +// LatestLocator returns a block locator for the passed block node. The passed +// node can be nil in which case the block locator for the current tip +// associated with the view will be returned. +// +// BlockLocator is used to help locate a specific block. The algorithm for +// building the block locator is to add the hashes in reverse order until +// the genesis block is reached. In order to keep the list of locator hashes +// to a reasonable number of entries, first the most recent previous 12 block +// hashes are added, then the step is doubled each loop iteration to +// exponentially decrease the number of hashes as a function of the distance +// from the block being located. +// +// For example, assume a block chain with a side chain as depicted below: +// +// genesis -> 1 -> 2 -> ... -> 15 -> 16 -> 17 -> 18 +// \-> 16a -> 17a +// +// The block locator for block 17a would be the hashes of blocks: +// [17a 16a 15 14 13 12 11 10 9 8 7 6 4 genesis] +// +// Caller is responsible for acquiring the ChainLock before calling this function. +func (bc *Blockchain) LatestLocator(tip *BlockNode) ([]*BlockHash, []uint32) { + blockTip := bc.blockTip() committedTip, _ := bc.GetCommittedTip() - return []*BlockHash{headerTip.Hash, committedTip.Hash}, []uint32{headerTip.Height, committedTip.Height} + locator := make([]*BlockHash, 0) + locatorHeights := make([]uint32, 0) + step := uint32(1) + for tip != nil { + locator = append(locator, tip.Hash) + locatorHeights = append(locatorHeights, tip.Height) + if tip.Height == 0 { + break + } + + // Calculate height of previous node to include ensuring we + // include the block tip and the committed tip. + if step == 1 { + // We use .GetParent as it is likely faster. + prevHash := tip.Header.PrevBlockHash + tip = tip.GetParent(bc.blockIndex) + if tip == nil { + glog.Errorf("LatestLocator: Block node not found for hash %v", prevHash) + continue + } + } else { + + var height uint32 + if tip.Height < step { + height = 0 + } else { + height = tip.Height - step + } + // Special cases to ensure we include the block tip, the committed tip, + // and the genesis block. + + // If the current node's height is greater than the block tip's height and + // value the height variable is less than the block tip's height, we set the + // variable height to the block tip's height. + // If the current node's height is greater than the committed tip's height and + // value the height variable is less than the committed tip's height, we set the + // variable height to the committed tip's height. + if tip.Height > blockTip.Height && height < blockTip.Height { + height = blockTip.Height + } else if tip.Height > committedTip.Height && height < committedTip.Height { + height = committedTip.Height + } + + var exists bool + var err error + tip, exists, err = bc.GetBlockFromBestChainByHeight(uint64(height), false) + if err != nil { + glog.Errorf("LatestLocator: Problem getting block from best chain by height: %v - continuing", err) + continue + } + if !exists { + glog.Errorf("LatestLocator: Block at height %d not found - continuing", height) + continue + } + } + + // Once 11 entries have been included, start doubling the distance + // between included hashes. + if len(locator) > 11 { + step *= 2 + } + } + return locator, locatorHeights + // OLD RETURN VALUE: + // return []*BlockHash{headerTip.Hash, committedTip.Hash}, []uint32{headerTip.Height, committedTip.Height} + //currNode := headerTip //hashes := []*BlockHash{headerTip.Hash} //heights := []uint32{headerTip.Height} @@ -1208,6 +1294,33 @@ func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { //return hashes, heights } +func (bc *Blockchain) HeaderLocatorWithNodeHashAndHeight( + blockHash *BlockHash, + height uint64, +) ( + []*BlockHash, + []uint32, + error, +) { + node, exists := bc.blockIndex.GetBlockNodeByHashAndHeight(blockHash, height) + if !exists { + return nil, nil, fmt.Errorf( + "Blockchain.HeaderLocatorWithNodeHashAndHeight: Node for hash %v and height %v is not in our block index", + blockHash, + height, + ) + } + locator, locatorHeights := bc.LatestLocator(node) + return locator, locatorHeights, nil +} + +// LatestHeaderLocator calls returns a block locator for the current tip of the +// header chain. +func (bc *Blockchain) LatestHeaderLocator() ([]*BlockHash, []uint32) { + headerTip := bc.headerTip() + return bc.LatestLocator(headerTip) +} + func (bc *Blockchain) GetBlockNodesToFetch( numBlocks int, _maxHeight int, blocksToIgnore map[BlockHash]bool) []*BlockNode { diff --git a/lib/peer.go b/lib/peer.go index f88735a4c..81915653a 100644 --- a/lib/peer.go +++ b/lib/peer.go @@ -381,9 +381,9 @@ func (pp *Peer) HelpHandleInv(msg *MsgDeSoInv) { headerTip := pp.srv.blockchain.headerTip() blockTip := pp.srv.blockchain.blockTip() glog.V(2).Infof("Server._handleInv: Sending GET_HEADERS message to peer %v\n"+ - "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Block Locator Hashes & Heights: (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", - pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + pp, locator, locatorHeights, headerTip.Hash, headerTip.Height, blockTip.Hash, blockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, diff --git a/lib/server.go b/lib/server.go index 485e7ba3a..0a9710ba9 100644 --- a/lib/server.go +++ b/lib/server.go @@ -1524,14 +1524,23 @@ func (srv *Server) _handleHeaderBundle(pp *Peer, msg *MsgDeSoHeaderBundle) { lastHeight := msg.Headers[len(msg.Headers)-1].Height headerTip := srv.blockchain.headerTip() currentBlockTip := srv.blockchain.blockTip() + locator, locatorHeights, err := srv.blockchain.HeaderLocatorWithNodeHashAndHeight(lastHash, lastHeight) + if err != nil { + glog.Warningf("Server._handleHeaderBundle: Disconnecting peer %v because "+ + "she indicated that she has more headers but the last hash %v in "+ + "the header bundle does not correspond to a block in our index.", + pp, lastHash) + pp.Disconnect("Last hash in header bundle not in our index") + return + } glog.V(2).Infof("Server._handleHeaderBundle: Sending GET_HEADERS message to peer %v\n"+ "Block Locator Hashes & Heights: (%v, %v) \n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", - pp, lastHash, lastHeight, headerTip.Hash, headerTip.Height, + pp, locator, locatorHeights, headerTip.Hash, headerTip.Height, currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, - BlockLocator: []*BlockHash{lastHash}, + BlockLocator: locator, }, false) glog.V(1).Infof("Server._handleHeaderBundle: *Syncing* headers for blocks starting at "+ "header tip %v out of %d from peer %v", @@ -2050,9 +2059,9 @@ func (srv *Server) _startSync() { headerTip := bestPeer.srv.blockchain.headerTip() currentBlockTip := bestPeer.srv.blockchain.blockTip() glog.V(2).Infof("Server._startSync: Sending GET_HEADERS message to peer %v\n"+ - "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + "Block Locator Hashes & Heights: (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", - bestPeer, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + bestPeer, locator, locatorHeights, headerTip.Hash, headerTip.Height, currentBlockTip.Hash, currentBlockTip.Height) bestPeer.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, @@ -2643,10 +2652,10 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { locator, locatorHeights := pp.srv.blockchain.LatestHeaderLocator() headerTip := pp.srv.blockchain.headerTip() currentBlockTip := pp.srv.blockchain.blockTip() - glog.V(2).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ - "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + glog.V(2).Infof("Server._handleBlock (chainState = SYNCING_HEADERS): Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v) \n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", - pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + pp, locator, locatorHeights, headerTip.Hash, headerTip.Height, currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, @@ -2694,10 +2703,10 @@ func (srv *Server) _handleBlock(pp *Peer, blk *MsgDeSoBlock, isLastBlock bool) { locator, locatorHeights := srv.blockchain.LatestHeaderLocator() headerTip := srv.blockchain.headerTip() currentBlockTip := srv.blockchain.blockTip() - glog.V(2).Infof("Server._handleBlock: Sending GET_HEADERS message to peer %v\n"+ - "Block Locator Hashes & Heights: (%v, %v) and (%v, %v)\n"+ + glog.V(2).Infof("Server._handleBlock (chain state = NEEDS_BLOCKS): Sending GET_HEADERS message to peer %v\n"+ + "Block Locator Hashes & Heights: (%v, %v)\n"+ "Header Tip: (%v, %v)\nBlock Tip: (%v, %v)", - pp, locator[0], locatorHeights[0], locator[1], locatorHeights[1], headerTip.Hash, headerTip.Height, + pp, locator, locatorHeights, headerTip.Hash, headerTip.Height, currentBlockTip.Hash, currentBlockTip.Height) pp.AddDeSoMessage(&MsgDeSoGetHeaders{ StopHash: &BlockHash{}, From bc87a146461c6cef0c8f85f6fa844d4fe09e8ad8 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Thu, 13 Mar 2025 17:04:42 -0400 Subject: [PATCH 118/131] use DerivedKeyEntry.Copy in utxo view copy --- lib/block_view.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/block_view.go b/lib/block_view.go index 40eac24e9..c026437d5 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -547,8 +547,7 @@ func (bav *UtxoView) CopyUtxoView() *UtxoView { // Copy the Derived Key data newView.DerivedKeyToDerivedEntry = make(map[DerivedKeyMapKey]*DerivedKeyEntry, len(bav.DerivedKeyToDerivedEntry)) for entryKey, entry := range bav.DerivedKeyToDerivedEntry { - newEntry := *entry - newView.DerivedKeyToDerivedEntry[entryKey] = &newEntry + newView.DerivedKeyToDerivedEntry[entryKey] = entry.Copy() } // Copy the DAO Coin Limit Order Entries From dfcdb1491fdb3798d2407afe97e9fc80f0a66baa Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 17 Mar 2025 21:28:44 -0400 Subject: [PATCH 119/131] add optimization in get block nodes to fetch --- lib/blockchain.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 83822ad4d..592bf2d78 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1351,14 +1351,21 @@ func (bc *Blockchain) GetBlockNodesToFetch( heightLimit = bestBlockTip.Header.Height + uint64(numBlocks) } currentHeight := heightLimit - backtrackingNode, backtrackingNodeExists, maxNodeError := bc.GetBlockFromBestChainByHeight(currentHeight, true) - if maxNodeError != nil { - glog.Errorf("GetBlockNodesToFetch: Problem getting maxNode block by height: %v", maxNodeError) - return nil - } - if !backtrackingNodeExists || backtrackingNode == nil { - glog.Errorf("GetBlockNodesToFetch: Block at height %d not found. Error finding max node.", heightLimit) - return nil + var backtrackingNode *BlockNode + if heightLimit == bc.blockIndex.GetHeaderTip().Header.Height { + backtrackingNode = bc.blockIndex.GetHeaderTip() + } else { + var backtrackingNodeExists bool + var backtrackingNodeErr error + backtrackingNode, backtrackingNodeExists, backtrackingNodeErr = bc.GetBlockFromBestChainByHeight(currentHeight, true) + if backtrackingNodeErr != nil { + glog.Errorf("GetBlockNodesToFetch: Problem getting maxNode block by height: %v", backtrackingNodeErr) + return nil + } + if !backtrackingNodeExists || backtrackingNode == nil { + glog.Errorf("GetBlockNodesToFetch: Block at height %d not found. Error finding max node.", heightLimit) + return nil + } } // Walk back from the maxNode to the bestBlockTip. for len(blockNodesToFetch) < numBlocks && From e07c9423bbd2577bc1b9f8b65dd4efa5e598452e Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Mon, 14 Apr 2025 23:20:34 -0400 Subject: [PATCH 120/131] add validatorPK in error about duplicate validator public key --- lib/network_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/network_manager.go b/lib/network_manager.go index 8361b1a5b..495d83bab 100644 --- a/lib/network_manager.go +++ b/lib/network_manager.go @@ -1288,7 +1288,7 @@ func (nm *NetworkManager) handleHandshakeCompletePoSMessage(remoteNode *RemoteNo if remoteNode.IsInbound() { _, ok := nm.GetValidatorInboundIndex().Get(validatorPk.Serialize()) if ok { - return fmt.Errorf("NetworkManager.handleHandshakeCompletePoSMessage: Inbound RemoteNode with duplicate validator public key") + return fmt.Errorf("NetworkManager.handleHandshakeCompletePoSMessage: Inbound RemoteNode with duplicate validator public key: %v", validatorPk.ToString()) } return nil } From ba0df5fa9db9eb6bec54244e57ddded873ab00de Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 25 Apr 2025 15:44:39 -0400 Subject: [PATCH 121/131] replace best chain branch w/ dep upgrades --- cmd/node.go | 2 +- go.mod | 157 ++++--- go.sum | 450 ++++++++++---------- integration_testing/tools.go | 2 +- lib/block_view.go | 2 +- lib/block_view_atomic_txns_test.go | 2 +- lib/block_view_bitcoin_test.go | 2 +- lib/block_view_creator_coin_test.go | 2 +- lib/block_view_dao_coin_limit_order_test.go | 2 +- lib/block_view_dao_coin_test.go | 2 +- lib/block_view_derived_key_test.go | 2 +- lib/block_view_flush.go | 2 +- lib/block_view_follow_test.go | 2 +- lib/block_view_like_test.go | 2 +- lib/block_view_lockups.go | 2 +- lib/block_view_lockups_test.go | 2 +- lib/block_view_message_test.go | 2 +- lib/block_view_nft_test.go | 2 +- lib/block_view_post.go | 2 +- lib/block_view_post_test.go | 2 +- lib/block_view_profile_test.go | 2 +- lib/block_view_stake.go | 2 +- lib/block_view_test.go | 2 +- lib/block_view_validator.go | 2 +- lib/blockchain.go | 2 +- lib/blockchain_test.go | 2 +- lib/db_adapter.go | 2 +- lib/db_utils.go | 2 +- lib/db_utils_test.go | 2 +- lib/event_manager.go | 2 +- lib/legacy_mempool.go | 2 +- lib/notifier.go | 2 +- lib/pos_blockchain.go | 2 +- lib/pos_epoch.go | 2 +- lib/pos_mempool.go | 2 +- lib/pos_mempool_persister.go | 2 +- lib/pos_mempool_persister_test.go | 2 +- lib/pos_mempool_test.go | 2 +- lib/pos_random_seed.go | 2 +- lib/pos_snapshot_cache.go | 2 +- lib/pos_snapshot_entries.go | 2 +- lib/postgres.go | 2 +- lib/server.go | 2 +- lib/snapshot.go | 2 +- lib/txindex.go | 2 +- 45 files changed, 372 insertions(+), 321 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index b3f32f011..bebf6b233 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -19,7 +19,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/deso-protocol/core/lib" "github.com/deso-protocol/core/migrate" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/go-pg/pg/v10" "github.com/golang/glog" migrations "github.com/robinjoseph08/go-pg-migrations/v3" diff --git a/go.mod b/go.mod index b33f9ec7a..ea56f7bf1 100644 --- a/go.mod +++ b/go.mod @@ -1,49 +1,49 @@ module github.com/deso-protocol/core -go 1.23.0 +go 1.23.7 -toolchain go1.23.2 +toolchain go1.24.1 require ( - github.com/DataDog/datadog-go/v5 v5.5.0 + github.com/DataDog/datadog-go/v5 v5.6.0 github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/btcsuite/btcd v0.24.2 github.com/btcsuite/btcd/btcec/v2 v2.3.4 github.com/btcsuite/btcd/btcutil v1.1.6 github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 github.com/bxcodec/faker v2.0.1+incompatible - github.com/cloudflare/circl v1.5.0 + github.com/cloudflare/circl v1.6.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 github.com/deso-protocol/go-deadlock v1.0.1 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 - github.com/dgraph-io/badger/v3 v3.2103.5 + github.com/dgraph-io/badger/v4 v4.7.0 github.com/dgraph-io/ristretto v0.2.0 github.com/emirpasic/gods v1.18.1 - github.com/ethereum/go-ethereum v1.14.11 - github.com/fatih/color v1.17.0 + github.com/ethereum/go-ethereum v1.15.10 + github.com/fatih/color v1.18.0 github.com/gernest/mention v2.0.0+incompatible - github.com/go-pg/pg/v10 v10.13.0 - github.com/golang/glog v1.2.2 + github.com/go-pg/pg/v10 v10.14.0 + github.com/golang/glog v1.2.4 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mitchellh/go-homedir v1.1.0 github.com/oleiade/lane v1.0.1 - github.com/onflow/crypto v0.25.2 + github.com/onflow/crypto v0.25.3 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0 github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 - github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2 + github.com/spf13/cobra v1.9.1 + github.com/spf13/pflag v1.0.6 + github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 github.com/tyler-smith/go-bip39 v1.1.0 - github.com/unrolled/secure v1.16.0 - golang.org/x/crypto v0.29.0 - golang.org/x/sync v0.9.0 - gopkg.in/DataDog/dd-trace-go.v1 v1.69.0 + github.com/unrolled/secure v1.17.0 + golang.org/x/crypto v0.37.0 + golang.org/x/sync v0.13.0 + gopkg.in/DataDog/dd-trace-go.v1 v1.72.2 ) require ( @@ -55,104 +55,137 @@ require ( require ( dario.cat/mergo v1.0.1 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/DataDog/appsec-internal-go v1.8.0 // indirect - github.com/DataDog/datadog-agent/pkg/obfuscate v0.57.1 // indirect - github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.57.1 // indirect - github.com/DataDog/go-libddwaf/v3 v3.4.0 // indirect - github.com/DataDog/go-sqllexer v0.0.16 // indirect + github.com/DataDog/appsec-internal-go v1.11.2 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/obfuscate v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/proto v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/trace v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/log v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/util/scrubber v0.64.3 // indirect + github.com/DataDog/datadog-agent/pkg/version v0.64.3 // indirect + github.com/DataDog/go-libddwaf/v3 v3.5.4 // indirect + github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 // indirect + github.com/DataDog/go-sqllexer v0.1.6 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/gostackparse v0.7.0 // indirect - github.com/DataDog/sketches-go v1.4.6 // indirect + github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 // indirect + github.com/DataDog/sketches-go v1.4.7 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andygrunwald/go-jira v1.16.0 // indirect github.com/btcsuite/btclog v0.0.0-20241017175713-3428138b75c7 // indirect github.com/bwesterb/go-ristretto v1.2.3 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect - github.com/ebitengine/purego v0.8.0 // indirect + github.com/ebitengine/purego v0.8.2 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/git-chglog/git-chglog v0.15.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/pprof v0.0.0-20241009165004-a3522334989c // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect + github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect - github.com/hashicorp/hcl v1.0.1-vault-5 // indirect - github.com/holiman/uint256 v1.3.1 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/holiman/uint256 v1.3.2 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/kyokomi/emoji/v2 v2.2.13 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect + github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/goveralls v0.0.12 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/ginkgo v1.15.0 // indirect github.com/onsi/gomega v1.10.5 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect - github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/petermattis/goid v0.0.0-20250319124200-ccd6737f222a // indirect github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect + github.com/shirou/gopsutil/v4 v4.25.3 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tinylib/msgp v1.2.2 // indirect + github.com/tinylib/msgp v1.2.5 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/trivago/tgo v1.0.7 // indirect github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df // indirect - github.com/urfave/cli/v2 v2.27.5 // indirect + github.com/urfave/cli/v2 v2.27.6 // indirect github.com/vmihailenco/bufpool v0.1.11 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - go.opencensus.io v0.24.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/component v1.30.0 // indirect + go.opentelemetry.io/collector/featuregate v1.30.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect + go.opentelemetry.io/collector/pdata v1.30.0 // indirect + go.opentelemetry.io/collector/pdata/pprofile v0.124.0 // indirect + go.opentelemetry.io/collector/semconv v0.124.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/term v0.26.0 // indirect - golang.org/x/text v0.20.0 // indirect - golang.org/x/time v0.7.0 // indirect - golang.org/x/tools v0.26.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/term v0.31.0 // indirect + golang.org/x/text v0.24.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.32.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - gonum.org/v1/gonum v0.15.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect + gonum.org/v1/gonum v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect + google.golang.org/grpc v1.72.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index dd7d70b0c..610e678b8 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,45 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/appsec-internal-go v1.8.0 h1:1Tfn3LEogntRqZtf88twSApOCAAO3V+NILYhuQIo4J4= -github.com/DataDog/appsec-internal-go v1.8.0/go.mod h1:wW0cRfWBo4C044jHGwYiyh5moQV2x0AhnwqMuiX7O/g= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.57.1 h1:jtHJ/+WW/Ccj+3kFY48d7+NwoW5KK7tHKIfuRolE6CA= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.57.1/go.mod h1:Po5HwoDd4FmT/EqgrE9x7Zz4LjxtGBSIuNY1C1lppBQ= -github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.57.1 h1:iP8pFB9IhRA1ZpRpIxyXcsNDbg9hYIzdljCBnbMqjcY= -github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.57.1/go.mod h1:4Vo3SJ24uzfKHUHLoFa8t8o+LH+7TCQ7sPcZDtOpSP4= -github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= -github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-libddwaf/v3 v3.4.0 h1:NJ2W2vhYaOm1OWr1LJCbdgp7ezG/XLJcQKBmjFwhSuM= -github.com/DataDog/go-libddwaf/v3 v3.4.0/go.mod h1:n98d9nZ1gzenRSk53wz8l6d34ikxS+hs62A31Fqmyi4= -github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= -github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/appsec-internal-go v1.11.2 h1:Q00pPMQzqMIw7jT2ObaORIxBzSly+deS0Ely9OZ/Bj0= +github.com/DataDog/appsec-internal-go v1.11.2/go.mod h1:9YppRCpElfGX+emXOKruShFYsdPq7WEPq/Fen4tYYpk= +github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.64.3 h1:FNklG6UnKVA6uSWoMcNPsJJMfhlbp6iJxMFj7zZZYuM= +github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.64.3/go.mod h1:lzCtnMSGZm/3RMk5RBRW/6IuK1TNbDXx1ttHTxN5Ykc= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.64.3 h1:hgf4Yp2MRnNnlstRh+He2pixCOwLKGQmrWLRHwM7LGI= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.64.3/go.mod h1:izbemZjqzBn9upkZj8SyT9igSGPMALaQYgswJ0408vY= +github.com/DataDog/datadog-agent/pkg/proto v0.64.3 h1:ox1oqM50cXfVEK2CpVvMyodyF5PTTemt1PkJGxkG2d8= +github.com/DataDog/datadog-agent/pkg/proto v0.64.3/go.mod h1:q324yHcBN5hIeCU8eoinM7lP9c7MOA2FTj7oeWAl3Pc= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.64.3 h1:0Y7YoUzDbXjA0RnrH11QwOn3LFD+oW1/L5et0qEvyHY= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.64.3/go.mod h1:1AAhFoEuoXs8jfpj7EiGW6lsqvCYgQc0B0pRpYAPEW4= +github.com/DataDog/datadog-agent/pkg/trace v0.64.3 h1:zroFsJasdLhltdHKkr4MP2gzgpBVpffevlcYfDDIZnI= +github.com/DataDog/datadog-agent/pkg/trace v0.64.3/go.mod h1:RIHqRISquXGlmIGROSWp3pzwPETBIvhQkmj2BlCj4qY= +github.com/DataDog/datadog-agent/pkg/util/log v0.64.3 h1:JpT79bpG8nKNWSPsTWLxAZ2I8an7Dav3ArfH1p2yi+Q= +github.com/DataDog/datadog-agent/pkg/util/log v0.64.3/go.mod h1:T6qw2pFyCopAFWrk6Vp0QmCTSusCbU42NlVPSUIZkv8= +github.com/DataDog/datadog-agent/pkg/util/scrubber v0.64.3 h1:v9iVXXvHamynoEUoFeeXS/CvqnItQe7ZgpwjIFxLeV4= +github.com/DataDog/datadog-agent/pkg/util/scrubber v0.64.3/go.mod h1:W0q1265hKJowCAMm4mMd3XTXtzL0V7aLBAqY64sZeec= +github.com/DataDog/datadog-agent/pkg/version v0.64.3 h1:mgzZpyGLyaiiid2hvha1/Vlj0LfSJn4vXC8+YaYGc9E= +github.com/DataDog/datadog-agent/pkg/version v0.64.3/go.mod h1:DgOVsfSRaNV4GZNl/qgoZjG3hJjoYUNWPPhbfTfTqtY= +github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= +github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +github.com/DataDog/go-libddwaf/v3 v3.5.4 h1:cLV5lmGhrUBnHG50EUXdqPQAlJdVCp9n3aQ5bDWJEAg= +github.com/DataDog/go-libddwaf/v3 v3.5.4/go.mod h1:HoLUHdj0NybsPBth/UppTcg8/DKA4g+AXuk8cZ6nuoo= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6 h1:bpitH5JbjBhfcTG+H2RkkiUXpYa8xSuIPnyNtTaSPog= +github.com/DataDog/go-runtime-metrics-internal v0.0.4-0.20241206090539-a14610dc22b6/go.mod h1:quaQJ+wPN41xEC458FCpTwyROZm3MzmTZ8q8XOXQiPs= +github.com/DataDog/go-sqllexer v0.1.6 h1:skEXpWEVCpeZFIiydoIa2f2rf+ymNpjiIMqpW4w3YAk= +github.com/DataDog/go-sqllexer v0.1.6/go.mod h1:GGpo1h9/BVSN+6NJKaEcJ9Jn44Hqc63Rakeb+24Mjgo= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= -github.com/DataDog/sketches-go v1.4.6 h1:acd5fb+QdUzGrosfNLwrIhqyrbMORpvBy7mE+vHlT3I= -github.com/DataDog/sketches-go v1.4.6/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0 h1:5US5SqqhfkZkg/E64uvn7YmeTwnudJHtlPEH/LOT99w= +github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.27.0/go.mod h1:VRo4D6rj92AExpVBlq3Gcuol9Nm1bber12KyxRjKGWw= +github.com/DataDog/sketches-go v1.4.7 h1:eHs5/0i2Sdf20Zkj0udVFWuCrXGRFig2Dcfm5rtcTxc= +github.com/DataDog/sketches-go v1.4.7/go.mod h1:eAmQ/EBmtSO+nQp7IZMZVRPT4BQTmIc5RZQ+deGlTPM= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= -github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= +github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -33,12 +47,9 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -74,25 +85,18 @@ github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= -github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= +github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= -github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -104,8 +108,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deso-protocol/go-deadlock v1.0.1 h1:lRziC+SaEU0X0fjoprwDlPWxVHCxRIy5LgT8EcD1av8= github.com/deso-protocol/go-deadlock v1.0.1/go.mod h1:7TistpkoQ9QHKLRMQSuX2wQk9PoB/BEj2jVrhov1HK0= @@ -113,31 +117,28 @@ github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+Ls github.com/deso-protocol/go-merkle-tree v1.0.0/go.mod h1:V/vbg/maaNv6G7zf9VVs645nLFx/jsO2L/awFB/S/ZU= github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cAHbdG6o= github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= -github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= -github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y= +github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA= github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= +github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= +github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4/go.mod h1:I5sHm0Y0T1u5YjlyqC5GVArM7aNZRUYtTjmJ8mPJFds= -github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE= -github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I= +github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY= -github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= -github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= -github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/ethereum/go-ethereum v1.15.10 h1:UxqBhpsF2TNF1f7Z/k3RUUHEuLvDGAlHuh/lQ99ZA0w= +github.com/ethereum/go-ethereum v1.15.10/go.mod h1:+S9k+jFzlyVTNcYGvqFhzN/SFhI6vA+aOY4T5tLSPL0= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fergusstrange/embedded-postgres v1.19.0 h1:NqDufJHeA03U7biULlPHZ0pZ10/mDOMKPILEpT50Fyk= @@ -146,95 +147,89 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gernest/mention v2.0.0+incompatible h1:pTXnujBC6tqlw5awDkLojq92TXbt0F+4+8FBlQC+di8= github.com/gernest/mention v2.0.0+incompatible/go.mod h1:/z3Hb+4gaPF+vL8og/lj6Au5j8hh5EfU7/EknmDUuO4= github.com/git-chglog/git-chglog v0.15.4 h1:BwPDj7AghQTfpXO+UxG4mZM5MUTe9wfDuenF3jpyNf0= github.com/git-chglog/git-chglog v0.15.4/go.mod h1:BmWdTpqBVzPjKNrBTZGcQCrQV9zq6gFKurhWNnJbYDA= -github.com/go-pg/pg/v10 v10.13.0 h1:xMagDE57VP8Y2KvIf9PvrsOAIjX62XqaKmfEzB0c5eU= -github.com/go-pg/pg/v10 v10.13.0/go.mod h1:IXp9Ok9JNNW9yWedbQxxvKUv84XhoH5+tGd+68y+zDs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-pg/pg/v10 v10.14.0 h1:giXuPsJaWjzwzFJTxy39eBgGE44jpqH1jwv0uI3kBUU= +github.com/go-pg/pg/v10 v10.14.0/go.mod h1:6kizZh54FveJxw9XZdNg07x7DDBWNsQrSiJS04MLwO8= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= -github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= -github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= -github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241009165004-a3522334989c h1:NDovD0SMpBYXlE1zJmS1q55vWB/fUQBcPAqAboZSccA= -github.com/google/pprof v0.0.0-20241009165004-a3522334989c/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 h1:gD0vax+4I+mAj+jEChEf25Ia07Jq7kYOFO5PPhAxFl4= +github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 h1:iBt4Ew4XEGLfh6/bPk4rSYmuZJGizr6/x/AEizP0CQc= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8/go.mod h1:aiJI+PIApBRQG7FZTEBx5GiiX+HbOHilUdNxUZi4eV0= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= -github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= -github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= -github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -242,14 +237,15 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -261,14 +257,12 @@ github.com/kyokomi/emoji/v2 v2.2.13 h1:GhTfQa67venUUvmleTNFnb+bi7S3aocF7ZCXU9fSO github.com/kyokomi/emoji/v2 v2.2.13/go.mod h1:JUcn42DTdsXJo1SWanHh4HKDEyPaR5CqkmoirZZP9qE= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/goveralls v0.0.12 h1:PEEeF0k1SsTjOBQ8FOmrOAoCu4ytuMaWCnWe94zxbCg= @@ -280,17 +274,21 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= +github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU= github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4= -github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= -github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= +github.com/onflow/crypto v0.25.3 h1:XQ3HtLsw8h1+pBN+NQ1JYM9mS2mVXTyg55OldaAIF7U= +github.com/onflow/crypto v0.25.3/go.mod h1:+1igaXiK6Tjm9wQOBD1EGwW7bYWMUGKtwKJ/2QL/OWs= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -303,44 +301,50 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.120.1 h1:lK/3zr73guK9apbXTcnDnYrC0YCQ25V3CIULYz3k2xU= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.120.1/go.mod h1:01TvyaK8x640crO2iFwW/6CFCZgNsOvOGH3B5J239m0= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.120.1 h1:TCyOus9tym82PD1VYtthLKMVMlVyRwtDI4ck4SR2+Ok= +github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.120.1/go.mod h1:Z/S1brD5gU2Ntht/bHxBVnGxXKTvZDr0dNv/riUzPmY= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= -github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= +github.com/petermattis/goid v0.0.0-20250319124200-ccd6737f222a h1:S+AGcmAESQ0pXCUNnRH7V+bOUIgkSX5qVt2cNKCrm0Q= +github.com/petermattis/goid v0.0.0-20250319124200-ccd6737f222a/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3/go.mod h1:vl5+MqJ1nBINuSsUI2mGgH79UweUT/B5Fy8857PqyyI= github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0 h1:EjexnDlSIZoK/gMfQmKIqB7tYsI+SS5hqxmXd63RLb4= github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0/go.mod h1:9yEG60N97UVFGD/UKQUXoGVZh/t8KXx3JxEpxhKFlKY= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/secure-systems-lab/go-securesystemslib v0.8.0 h1:mr5An6X45Kb2nddcFlbmfHkLguCE9laoZCUzEEpIZXA= -github.com/secure-systems-lab/go-securesystemslib v0.8.0/go.mod h1:UH2VZVuJfCYR8WgMlCU1uFsOUU+KeyrTWcSS73NBOzU= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= +github.com/secure-systems-lab/go-securesystemslib v0.9.0 h1:rf1HIbL64nUpEIZnjLZ3mcNEL9NBPB0iuVjyxvq3LZc= +github.com/secure-systems-lab/go-securesystemslib v0.9.0/go.mod h1:DVHKMcZ+V4/woA/peqr+L0joiRXbPpQ042GgJckkFgw= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0 h1:Xuk8ma/ibJ1fOy4Ee11vHhUFHQNpHhrBneOCNHVXS5w= github.com/shibukawa/configdir v0.0.0-20170330084843-e180dbdc8da0/go.mod h1:7AwjWCpdPhkSmNAgUv5C7EJ4AbmjEB3r047r3DXWu3Y= +github.com/shirou/gopsutil/v4 v4.25.3 h1:SeA68lsu8gLggyMbmCn8cmp97V1TI9ld9sVzAUcKcKE= +github.com/shirou/gopsutil/v4 v4.25.3/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5BdjbJwf2fv4szA= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -348,25 +352,18 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= -github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -374,7 +371,6 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -389,8 +385,12 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/tinylib/msgp v1.2.2 h1:iHiBE1tJQwFI740SPEPkGE8cfhNfrqOYRlH450BnC/4= -github.com/tinylib/msgp v1.2.2/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= +github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= @@ -399,13 +399,14 @@ github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df h1:Y2l28Jr3 github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df/go.mod h1:pnyouUty/nBr/zm3GYwTIt+qFTLWbdjeLjZmJdzJOu8= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/unrolled/secure v1.16.0 h1:XgdAsS/Zl50ZfZPRJK6WpicFttfrsFYFd0+ONDBJubU= -github.com/unrolled/secure v1.16.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= -github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= -github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbWU= +github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= +github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= +github.com/urfave/cli/v2 v2.27.6/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= +github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= @@ -414,16 +415,62 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/collector/component v1.30.0 h1:HXjqBHaQ47/EEuWdnkjr4Y3kRWvmyWIDvqa1Q262Fls= +go.opentelemetry.io/collector/component v1.30.0/go.mod h1:vfM9kN+BM6oHBXWibquiprz8CVawxd4/aYy3nbhme3E= +go.opentelemetry.io/collector/component/componentstatus v0.120.0 h1:hzKjI9+AIl8A/saAARb47JqabWsge0kMp8NSPNiCNOQ= +go.opentelemetry.io/collector/component/componentstatus v0.120.0/go.mod h1:kbuAEddxvcyjGLXGmys3nckAj4jTGC0IqDIEXAOr3Ag= +go.opentelemetry.io/collector/component/componenttest v0.123.0 h1:h0B/kBj0URKq+i9iMbMuLhc6/dZ2GWL0y9L6tqHRNuA= +go.opentelemetry.io/collector/component/componenttest v0.123.0/go.mod h1:4Y6EMvsgE9fUNM98G0eW5+LFXfcxdhTHQDhaOxJrgN8= +go.opentelemetry.io/collector/consumer v1.26.0 h1:0MwuzkWFLOm13qJvwW85QkoavnGpR4ZObqCs9g1XAvk= +go.opentelemetry.io/collector/consumer v1.26.0/go.mod h1:I/ZwlWM0sbFLhbStpDOeimjtMbWpMFSoGdVmzYxLGDg= +go.opentelemetry.io/collector/consumer/consumertest v0.120.0 h1:iPFmXygDsDOjqwdQ6YZcTmpiJeQDJX+nHvrjTPsUuv4= +go.opentelemetry.io/collector/consumer/consumertest v0.120.0/go.mod h1:HeSnmPfAEBnjsRR5UY1fDTLlSrYsMsUjufg1ihgnFJ0= +go.opentelemetry.io/collector/consumer/xconsumer v0.120.0 h1:dzM/3KkFfMBIvad+NVXDV+mA+qUpHyu5c70TFOjDg68= +go.opentelemetry.io/collector/consumer/xconsumer v0.120.0/go.mod h1:eOf7RX9CYC7bTZQFg0z2GHdATpQDxI0DP36F9gsvXOQ= +go.opentelemetry.io/collector/featuregate v1.30.0 h1:mx7+iP/FQnY7KO8qw/xE3Qd1MQkWcU8VgcqLNrJ8EU8= +go.opentelemetry.io/collector/featuregate v1.30.0/go.mod h1:Y/KsHbvREENKvvN9RlpiWk/IGBK+CATBYzIIpU7nccc= +go.opentelemetry.io/collector/internal/telemetry v0.124.0 h1:kzd1/ZYhLj4bt2pDB529mL4rIRrRacemXodFNxfhdWk= +go.opentelemetry.io/collector/internal/telemetry v0.124.0/go.mod h1:ZjXjqV0dJ+6D4XGhTOxg/WHjnhdmXsmwmUSgALea66Y= +go.opentelemetry.io/collector/pdata v1.30.0 h1:j3jyq9um436r6WzWySzexP2nLnFdmL5uVBYAlyr9nDM= +go.opentelemetry.io/collector/pdata v1.30.0/go.mod h1:0Bxu1ktuj4wE7PIASNSvd0SdBscQ1PLtYasymJ13/Cs= +go.opentelemetry.io/collector/pdata/pprofile v0.124.0 h1:ZjL9wKqzP4BHj0/F1jfGxs1Va8B7xmYayipZeNVoWJE= +go.opentelemetry.io/collector/pdata/pprofile v0.124.0/go.mod h1:1EN3Gw5LSI4fSVma/Yfv/6nqeuYgRTm1/kmG5nE5Oyo= +go.opentelemetry.io/collector/pdata/testdata v0.120.0 h1:Zp0LBOv3yzv/lbWHK1oht41OZ4WNbaXb70ENqRY7HnE= +go.opentelemetry.io/collector/pdata/testdata v0.120.0/go.mod h1:PfezW5Rzd13CWwrElTZRrjRTSgMGUOOGLfHeBjj+LwY= +go.opentelemetry.io/collector/pipeline v0.124.0 h1:hKvhDyH2GPnNO8LGL34ugf36sY7EOXPjBvlrvBhsOdw= +go.opentelemetry.io/collector/pipeline v0.124.0/go.mod h1:TO02zju/K6E+oFIOdi372Wk0MXd+Szy72zcTsFQwXl4= +go.opentelemetry.io/collector/processor v0.120.0 h1:No+I65ybBLVy4jc7CxcsfduiBrm7Z6kGfTnekW3hx1A= +go.opentelemetry.io/collector/processor v0.120.0/go.mod h1:4zaJGLZCK8XKChkwlGC/gn0Dj4Yke04gQCu4LGbJGro= +go.opentelemetry.io/collector/processor/processortest v0.120.0 h1:R+VSVSU59W0/mPAcyt8/h1d0PfWN6JI2KY5KeMICXvo= +go.opentelemetry.io/collector/processor/processortest v0.120.0/go.mod h1:me+IVxPsj4IgK99I0pgKLX34XnJtcLwqtgTuVLhhYDI= +go.opentelemetry.io/collector/processor/xprocessor v0.120.0 h1:mBznj/1MtNqmu6UpcoXz6a63tU0931oWH2pVAt2+hzo= +go.opentelemetry.io/collector/processor/xprocessor v0.120.0/go.mod h1:Nsp0sDR3gE+GAhi9d0KbN0RhOP+BK8CGjBRn8+9d/SY= +go.opentelemetry.io/collector/semconv v0.124.0 h1:YTdo3UFwNyDQCh9DiSm2rbzAgBuwn/9dNZ0rv454goA= +go.opentelemetry.io/collector/semconv v0.124.0/go.mod h1:te6VQ4zZJO5Lp8dM2XIhDxDiL45mwX0YAQQWRQ0Qr9U= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -432,20 +479,15 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -453,13 +495,10 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -467,35 +506,29 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -504,6 +537,7 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -513,20 +547,19 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -534,15 +567,12 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -552,48 +582,36 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= -gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/DataDog/dd-trace-go.v1 v1.69.0 h1:zSY6DDsFRMQDNQYKWCv/AEwJXoPpDf1FfMyw7I1B7M8= -gopkg.in/DataDog/dd-trace-go.v1 v1.69.0/go.mod h1:U9AOeBHNAL95JXcd/SPf4a7O5GNeF/yD13sJtli/yaU= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/DataDog/dd-trace-go.v1 v1.72.2 h1:SLcih9LB+I1l76Wd7aUSpzISemewzjq6djntMnBnzkA= +gopkg.in/DataDog/dd-trace-go.v1 v1.72.2/go.mod h1:XqDhDqsLpThFnJc4z0FvAEItISIAUka+RHwmQ6EfN1U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -611,8 +629,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= +k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0= diff --git a/integration_testing/tools.go b/integration_testing/tools.go index 613a93947..9381a9f9f 100644 --- a/integration_testing/tools.go +++ b/integration_testing/tools.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/lib/block_view.go b/lib/block_view.go index c026437d5..b4e1dd617 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_atomic_txns_test.go b/lib/block_view_atomic_txns_test.go index 53d71c7e1..ead9a7049 100644 --- a/lib/block_view_atomic_txns_test.go +++ b/lib/block_view_atomic_txns_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_bitcoin_test.go b/lib/block_view_bitcoin_test.go index b68a18efd..0c00a1bdf 100644 --- a/lib/block_view_bitcoin_test.go +++ b/lib/block_view_bitcoin_test.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/btcsuite/btcd/wire" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_creator_coin_test.go b/lib/block_view_creator_coin_test.go index e25af8ac9..7e4a3a05b 100644 --- a/lib/block_view_creator_coin_test.go +++ b/lib/block_view_creator_coin_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_limit_order_test.go b/lib/block_view_dao_coin_limit_order_test.go index 07d214725..98034b5e0 100644 --- a/lib/block_view_dao_coin_limit_order_test.go +++ b/lib/block_view_dao_coin_limit_order_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_test.go b/lib/block_view_dao_coin_test.go index b006c3d7d..aa64707bd 100644 --- a/lib/block_view_dao_coin_test.go +++ b/lib/block_view_dao_coin_test.go @@ -6,7 +6,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_derived_key_test.go b/lib/block_view_derived_key_test.go index e9d079786..bc59dd3f9 100644 --- a/lib/block_view_derived_key_test.go +++ b/lib/block_view_derived_key_test.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_flush.go b/lib/block_view_flush.go index 6dc4701ce..e1f407aa3 100644 --- a/lib/block_view_flush.go +++ b/lib/block_view_flush.go @@ -5,7 +5,7 @@ import ( "reflect" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_follow_test.go b/lib/block_view_follow_test.go index fedb91a03..ef3823f63 100644 --- a/lib/block_view_follow_test.go +++ b/lib/block_view_follow_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_like_test.go b/lib/block_view_like_test.go index 01b715475..a757c7f70 100644 --- a/lib/block_view_like_test.go +++ b/lib/block_view_like_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_lockups.go b/lib/block_view_lockups.go index bf8e31f97..aaaaeeb81 100644 --- a/lib/block_view_lockups.go +++ b/lib/block_view_lockups.go @@ -8,7 +8,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 3ab91972a..571a4253c 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -7,7 +7,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_message_test.go b/lib/block_view_message_test.go index dbe204f99..a47c3d85c 100644 --- a/lib/block_view_message_test.go +++ b/lib/block_view_message_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_nft_test.go b/lib/block_view_nft_test.go index ba00d8ece..2f4cfd64c 100644 --- a/lib/block_view_nft_test.go +++ b/lib/block_view_nft_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_post.go b/lib/block_view_post.go index e76745fbb..d8237e3b1 100644 --- a/lib/block_view_post.go +++ b/lib/block_view_post.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/gernest/mention" "github.com/golang/glog" "github.com/pkg/errors" diff --git a/lib/block_view_post_test.go b/lib/block_view_post_test.go index e84eaa1c9..264df7d69 100644 --- a/lib/block_view_post_test.go +++ b/lib/block_view_post_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_profile_test.go b/lib/block_view_profile_test.go index 01101d005..c59e37cd5 100644 --- a/lib/block_view_profile_test.go +++ b/lib/block_view_profile_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_stake.go b/lib/block_view_stake.go index b411673af..95b7d1a86 100644 --- a/lib/block_view_stake.go +++ b/lib/block_view_stake.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_test.go b/lib/block_view_test.go index 0f835cc02..e2c125457 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -14,7 +14,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" "github.com/stretchr/testify/assert" diff --git a/lib/block_view_validator.go b/lib/block_view_validator.go index 0357d098c..edf8f6b3a 100644 --- a/lib/block_view_validator.go +++ b/lib/block_view_validator.go @@ -15,7 +15,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain.go b/lib/blockchain.go index 7c68dff6b..c84878f27 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -29,7 +29,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index 85817fb24..e0fe102a3 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -19,7 +19,7 @@ import ( chainlib "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/db_adapter.go b/lib/db_adapter.go index 4eea4d0da..8cff5d757 100644 --- a/lib/db_adapter.go +++ b/lib/db_adapter.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils.go b/lib/db_utils.go index 79fc455c9..f721815f5 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -27,7 +27,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils_test.go b/lib/db_utils_test.go index 095a1db44..eafab46e8 100644 --- a/lib/db_utils_test.go +++ b/lib/db_utils_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/event_manager.go b/lib/event_manager.go index 486f7390e..4fe654273 100644 --- a/lib/event_manager.go +++ b/lib/event_manager.go @@ -1,7 +1,7 @@ package lib import ( - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/google/uuid" ) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index d489f1f7d..1a368f471 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -18,7 +18,7 @@ import ( "time" "github.com/btcsuite/btcd/btcutil" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/gernest/mention" "github.com/btcsuite/btcd/btcec/v2" diff --git a/lib/notifier.go b/lib/notifier.go index a42ffb3ab..b6c88c37d 100644 --- a/lib/notifier.go +++ b/lib/notifier.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/gernest/mention" "github.com/go-pg/pg/v10" diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 81e49be49..80f18e72e 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_epoch.go b/lib/pos_epoch.go index fed62d90e..ef4e3c78d 100644 --- a/lib/pos_epoch.go +++ b/lib/pos_epoch.go @@ -3,7 +3,7 @@ package lib import ( "bytes" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index b2217c121..5580bffe2 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -10,7 +10,7 @@ import ( "sync/atomic" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index 43ee7c449..79327094d 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister_test.go b/lib/pos_mempool_persister_test.go index 6cd1ef25d..73887a01f 100644 --- a/lib/pos_mempool_persister_test.go +++ b/lib/pos_mempool_persister_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_mempool_test.go b/lib/pos_mempool_test.go index d04883919..8dced4464 100644 --- a/lib/pos_mempool_test.go +++ b/lib/pos_mempool_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_random_seed.go b/lib/pos_random_seed.go index 6777de6e6..9c739c41d 100644 --- a/lib/pos_random_seed.go +++ b/lib/pos_random_seed.go @@ -7,7 +7,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_cache.go b/lib/pos_snapshot_cache.go index ec495633c..f320e99b0 100644 --- a/lib/pos_snapshot_cache.go +++ b/lib/pos_snapshot_cache.go @@ -3,7 +3,7 @@ package lib import ( "sync" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_entries.go b/lib/pos_snapshot_entries.go index 02cb16ceb..c0cc9ff8c 100644 --- a/lib/pos_snapshot_entries.go +++ b/lib/pos_snapshot_entries.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/postgres.go b/lib/postgres.go index f4da6642c..7c41102b8 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" "github.com/golang/glog" diff --git a/lib/server.go b/lib/server.go index 8cb5494f4..0833b58d9 100644 --- a/lib/server.go +++ b/lib/server.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "net" "path/filepath" "reflect" diff --git a/lib/snapshot.go b/lib/snapshot.go index 8276f03d3..7ccaf23ec 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -15,7 +15,7 @@ import ( "github.com/cloudflare/circl/group" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" diff --git a/lib/txindex.go b/lib/txindex.go index 3fadfae8a..2bdbc4060 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v4" chainlib "github.com/btcsuite/btcd/blockchain" "github.com/golang/glog" From ac2416370fa3571e6705f6eeb66fc53f175854f6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 25 Apr 2025 15:48:44 -0400 Subject: [PATCH 122/131] bump to go 1.24 --- .github/workflows/ci.yml | 4 ++-- Dockerfile | 2 +- go.mod | 2 +- test.Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 711f41d53..7ac1a13b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: "1.23" + go-version: "1.24" - name: Checkout branch uses: actions/checkout@v3 @@ -52,7 +52,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: "1.23" + go-version: "1.24" - name: Checkout branch uses: actions/checkout@v3 diff --git a/Dockerfile b/Dockerfile index 17892106c..3faaad5ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM alpine:latest AS core RUN apk update && apk upgrade RUN apk add --update bash cmake g++ gcc git make vips vips-dev -COPY --from=golang:1.23-alpine /usr/local/go/ /usr/local/go/ +COPY --from=golang:1.24-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" WORKDIR /deso/src/core diff --git a/go.mod b/go.mod index ea56f7bf1..c5dc57d4f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/deso-protocol/core -go 1.23.7 +go 1.24 toolchain go1.24.1 diff --git a/test.Dockerfile b/test.Dockerfile index a2c29d66d..e72d707c6 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -4,7 +4,7 @@ RUN apk update RUN apk upgrade RUN apk add --update bash cmake g++ gcc git make vips vips-dev -COPY --from=golang:1.23-alpine /usr/local/go/ /usr/local/go/ +COPY --from=golang:1.24-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" WORKDIR /deso/src/core From 6909d096988629d5d809d641e64266b165ae7d94 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 25 Apr 2025 15:58:45 -0400 Subject: [PATCH 123/131] use ristretto v2 --- go.mod | 3 +-- go.sum | 2 -- lib/db_utils.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index c5dc57d4f..e3003e95d 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 github.com/dgraph-io/badger/v4 v4.7.0 - github.com/dgraph-io/ristretto v0.2.0 + github.com/dgraph-io/ristretto/v2 v2.2.0 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.15.10 github.com/fatih/color v1.18.0 @@ -83,7 +83,6 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.2 // indirect diff --git a/go.sum b/go.sum index 610e678b8..aaf4768c3 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,6 @@ github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cA github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y= github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA= -github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= -github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= diff --git a/lib/db_utils.go b/lib/db_utils.go index f721815f5..0b516be27 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -9,7 +9,7 @@ import ( "encoding/json" "fmt" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/ristretto/z" + "github.com/dgraph-io/ristretto/v2/z" "io" "log" "math" From 8f52c01d728ef0fa6368a85ec87bef5cfc661f96 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 25 Apr 2025 16:07:01 -0400 Subject: [PATCH 124/131] fix error.Wrapf issues --- lib/block_view_dao_coin.go | 6 +++--- lib/legacy_mempool.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/block_view_dao_coin.go b/lib/block_view_dao_coin.go index e399e43e0..da81205d2 100644 --- a/lib/block_view_dao_coin.go +++ b/lib/block_view_dao_coin.go @@ -385,9 +385,9 @@ func (bav *UtxoView) HelpConnectDAOCoinMint( if creatorProfileEntry.DAOCoinEntry.CoinsInCirculationNanos.Gt( uint256.NewInt(0).Sub(MaxUint256, &txMeta.CoinsToMintNanos)) { return 0, 0, nil, errors.Wrapf( - RuleErrorOverflowWhileMintingDAOCoins, fmt.Sprintf( - "_connectDAOCoin: Overflow while summing CoinsInCirculationNanos and CoinsToMinNanos: %v, %v", - creatorProfileEntry.DAOCoinEntry.CoinsInCirculationNanos, txMeta.CoinsToMintNanos)) + RuleErrorOverflowWhileMintingDAOCoins, + "_connectDAOCoin: Overflow while summing CoinsInCirculationNanos and CoinsToMinNanos: %v, %v", + creatorProfileEntry.DAOCoinEntry.CoinsInCirculationNanos, txMeta.CoinsToMintNanos) } // CoinsInCirculationNanos = CoinsInCirculationNanos + CoinsToMintNanos creatorProfileEntry.DAOCoinEntry.CoinsInCirculationNanos = *uint256.NewInt(0).Add( diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index 1a368f471..0793aed85 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -1138,7 +1138,7 @@ func (mp *DeSoMempool) tryAcceptTransaction( totalInput, totalOutput, txHash, hex.EncodeToString(txBytes)) glog.Error(errRet) mp.rebuildBackupView() - return nil, nil, errors.Wrapf(TxErrorInsufficientFeeMinFee, errRet.Error()) + return nil, nil, errors.Wrap(TxErrorInsufficientFeeMinFee, errRet.Error()) } // If the transaction is bigger than half the maximum allowable size, From 38029eb76b0f9075bef74890624add645ff77f1b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Fri, 25 Apr 2025 16:44:39 -0400 Subject: [PATCH 125/131] use true for isHeaderChain in LatestLocator --- lib/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/blockchain.go b/lib/blockchain.go index 7c68dff6b..0a645fe97 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -1257,7 +1257,7 @@ func (bc *Blockchain) LatestLocator(tip *BlockNode) ([]*BlockHash, []uint32) { var exists bool var err error - tip, exists, err = bc.GetBlockFromBestChainByHeight(uint64(height), false) + tip, exists, err = bc.GetBlockFromBestChainByHeight(uint64(height), true) if err != nil { glog.Errorf("LatestLocator: Problem getting block from best chain by height: %v - continuing", err) continue From 32fa1de80d6a17838ec6e70d14db2e375828606b Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 3 May 2025 13:37:08 -0400 Subject: [PATCH 126/131] upgrade glog dep --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 08d540d10..52e42ab18 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/ethereum/go-ethereum v1.15.10 github.com/gernest/mention v2.0.0+incompatible github.com/go-pg/pg/v10 v10.14.0 - github.com/golang/glog v1.2.4 + github.com/golang/glog v1.2.5 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index c2d2c14bf..ec8aca7e4 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= From cb76f531441a1e85ea79e1911d0a7d076a7c8e79 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sat, 3 May 2025 13:44:37 -0400 Subject: [PATCH 127/131] add oleiade/lane dep --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 52e42ab18..7c38ab284 100644 --- a/go.mod +++ b/go.mod @@ -123,6 +123,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oleiade/lane v1.0.1 // indirect github.com/onsi/ginkgo v1.15.0 // indirect github.com/onsi/gomega v1.10.5 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect diff --git a/go.sum b/go.sum index ec8aca7e4..77b33a87f 100644 --- a/go.sum +++ b/go.sum @@ -285,6 +285,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU= +github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4= github.com/onflow/crypto v0.25.3 h1:XQ3HtLsw8h1+pBN+NQ1JYM9mS2mVXTyg55OldaAIF7U= github.com/onflow/crypto v0.25.3/go.mod h1:+1igaXiK6Tjm9wQOBD1EGwW7bYWMUGKtwKJ/2QL/OWs= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= From b2e6d96d3c37011371df7a03ea9b7ebc9d216111 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 4 May 2025 13:54:21 -0400 Subject: [PATCH 128/131] downgrade badger to v3 --- cmd/node.go | 2 +- go.mod | 10 +- go.sum | 101 +++++++++++++++++++- integration_testing/tools.go | 2 +- lib/block_view.go | 2 +- lib/block_view_atomic_txns_test.go | 2 +- lib/block_view_bitcoin_test.go | 2 +- lib/block_view_creator_coin_test.go | 2 +- lib/block_view_dao_coin_limit_order_test.go | 2 +- lib/block_view_dao_coin_test.go | 2 +- lib/block_view_derived_key_test.go | 2 +- lib/block_view_flush.go | 2 +- lib/block_view_follow_test.go | 2 +- lib/block_view_like_test.go | 2 +- lib/block_view_lockups.go | 2 +- lib/block_view_lockups_test.go | 2 +- lib/block_view_message_test.go | 2 +- lib/block_view_nft_test.go | 2 +- lib/block_view_post.go | 2 +- lib/block_view_post_test.go | 2 +- lib/block_view_profile_test.go | 2 +- lib/block_view_stake.go | 2 +- lib/block_view_test.go | 2 +- lib/block_view_validator.go | 2 +- lib/blockchain.go | 2 +- lib/blockchain_test.go | 2 +- lib/db_adapter.go | 2 +- lib/db_utils.go | 2 +- lib/db_utils_test.go | 2 +- lib/event_manager.go | 2 +- lib/legacy_mempool.go | 2 +- lib/notifier.go | 2 +- lib/pos_blockchain.go | 2 +- lib/pos_epoch.go | 2 +- lib/pos_mempool.go | 2 +- lib/pos_mempool_persister.go | 2 +- lib/pos_mempool_persister_test.go | 2 +- lib/pos_mempool_test.go | 2 +- lib/pos_random_seed.go | 2 +- lib/pos_snapshot_cache.go | 2 +- lib/pos_snapshot_entries.go | 2 +- lib/postgres.go | 2 +- lib/server.go | 2 +- lib/snapshot.go | 2 +- lib/txindex.go | 2 +- 45 files changed, 147 insertions(+), 50 deletions(-) diff --git a/cmd/node.go b/cmd/node.go index bebf6b233..b3f32f011 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -19,7 +19,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/deso-protocol/core/lib" "github.com/deso-protocol/core/migrate" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/golang/glog" migrations "github.com/robinjoseph08/go-pg-migrations/v3" diff --git a/go.mod b/go.mod index 7c38ab284..62723af0a 100644 --- a/go.mod +++ b/go.mod @@ -18,16 +18,18 @@ require ( github.com/deso-protocol/go-deadlock v1.0.1 github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 - github.com/dgraph-io/badger/v4 v4.7.0 + github.com/dgraph-io/badger/v3 v3.2103.5 github.com/dgraph-io/ristretto/v2 v2.2.0 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.15.10 + github.com/fatih/color v1.18.0 github.com/gernest/mention v2.0.0+incompatible github.com/go-pg/pg/v10 v10.14.0 github.com/golang/glog v1.2.5 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mitchellh/go-homedir v1.1.0 + github.com/oleiade/lane v1.0.1 github.com/onflow/crypto v0.25.3 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 @@ -76,15 +78,16 @@ require ( github.com/andygrunwald/go-jira v1.16.0 // indirect github.com/btcsuite/btclog v0.0.0-20241017175713-3428138b75c7 // indirect github.com/bwesterb/go-ristretto v1.2.3 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.2 // indirect - github.com/fatih/color v1.18.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/git-chglog/git-chglog v0.15.4 // indirect @@ -95,6 +98,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect @@ -123,7 +127,6 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/oleiade/lane v1.0.1 // indirect github.com/onsi/ginkgo v1.15.0 // indirect github.com/onsi/gomega v1.10.5 // indirect github.com/outcaste-io/ristretto v0.2.3 // indirect @@ -157,6 +160,7 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/collector/component v1.30.0 // indirect go.opentelemetry.io/collector/featuregate v1.30.0 // indirect diff --git a/go.sum b/go.sum index 77b33a87f..3de0a2e66 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/appsec-internal-go v1.11.2 h1:Q00pPMQzqMIw7jT2ObaORIxBzSly+deS0Ely9OZ/Bj0= github.com/DataDog/appsec-internal-go v1.11.2/go.mod h1:9YppRCpElfGX+emXOKruShFYsdPq7WEPq/Fen4tYYpk= github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.64.3 h1:FNklG6UnKVA6uSWoMcNPsJJMfhlbp6iJxMFj7zZZYuM= @@ -47,9 +49,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ= github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -85,15 +90,24 @@ github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/ github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/bxcodec/faker v2.0.1+incompatible/go.mod h1:BNzfpVdTwnFJ6GtfYTcQu6l6rHShT+veBxNCnjCx5XM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -117,8 +131,10 @@ github.com/deso-protocol/go-merkle-tree v1.0.0 h1:9zkI5dQsITYy77s4kbTGPQmZnhQ+Ls github.com/deso-protocol/go-merkle-tree v1.0.0/go.mod h1:V/vbg/maaNv6G7zf9VVs645nLFx/jsO2L/awFB/S/ZU= github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cAHbdG6o= github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= -github.com/dgraph-io/badger/v4 v4.7.0 h1:Q+J8HApYAY7UMpL8d9owqiB+odzEc0zn/aqOD9jhc6Y= -github.com/dgraph-io/badger/v4 v4.7.0/go.mod h1:He7TzG3YBy3j4f5baj5B7Zl2XyfNe5bl4Udl0aPemVA= +github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= +github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -133,6 +149,10 @@ github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.15.10 h1:UxqBhpsF2TNF1f7Z/k3RUUHEuLvDGAlHuh/lQ99ZA0w= github.com/ethereum/go-ethereum v1.15.10/go.mod h1:+S9k+jFzlyVTNcYGvqFhzN/SFhI6vA+aOY4T5tLSPL0= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -170,32 +190,46 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= -github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -208,6 +242,7 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 h1:gD0vax+4I+mAj+jEChEf25Ia07Jq7kYOFO5PPhAxFl4= github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -221,6 +256,7 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= @@ -230,6 +266,7 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -244,6 +281,7 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -259,6 +297,7 @@ github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= @@ -274,6 +313,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c h1:cqn374mizHuIWj+OSJCajGr/phAmuMug9qIX3l9CflE= github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -309,6 +349,7 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= @@ -325,6 +366,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richardartoul/molecule v1.0.1-0.20240531184615-7ca0df43c0b3 h1:4+LEVOB87y175cLJC/mbsgKmoDOjrBldtXvioEy96WY= @@ -333,6 +375,7 @@ github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0 h1:EjexnDlSIZoK/gMfQmKIqB7tY github.com/robinjoseph08/go-pg-migrations/v3 v3.1.0/go.mod h1:9yEG60N97UVFGD/UKQUXoGVZh/t8KXx3JxEpxhKFlKY= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -352,16 +395,23 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -371,6 +421,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -399,6 +450,7 @@ github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df h1:Y2l28Jr3 github.com/tsuyoshiwada/go-gitcmd v0.0.0-20180205145712-5f1f5f9475df/go.mod h1:pnyouUty/nBr/zm3GYwTIt+qFTLWbdjeLjZmJdzJOu8= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbWU= github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli/v2 v2.27.6 h1:VdRdS98FNhKZ8/Az8B7MTyGQmpIr36O1EHybx/LaZ4g= @@ -415,6 +467,7 @@ github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAh github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -423,6 +476,9 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/collector/component v1.30.0 h1:HXjqBHaQ47/EEuWdnkjr4Y3kRWvmyWIDvqa1Q262Fls= @@ -482,12 +538,17 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -498,7 +559,10 @@ golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -506,6 +570,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -514,7 +579,10 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -523,10 +591,13 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -547,6 +618,7 @@ golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -572,7 +644,10 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -592,10 +667,22 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 h1:29cjnHVylHwTzH66WfFZqgSQgnxzvWE+jvBwpZCLRxY= google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -603,7 +690,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= @@ -612,6 +702,7 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.72.2 h1:SLcih9LB+I1l76Wd7aUSpzISemewzjq6djntM gopkg.in/DataDog/dd-trace-go.v1 v1.72.2/go.mod h1:XqDhDqsLpThFnJc4z0FvAEItISIAUka+RHwmQ6EfN1U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -629,6 +720,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= diff --git a/integration_testing/tools.go b/integration_testing/tools.go index 9381a9f9f..613a93947 100644 --- a/integration_testing/tools.go +++ b/integration_testing/tools.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" "github.com/stretchr/testify/require" diff --git a/lib/block_view.go b/lib/block_view.go index b4e1dd617..c026437d5 100644 --- a/lib/block_view.go +++ b/lib/block_view.go @@ -18,7 +18,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_atomic_txns_test.go b/lib/block_view_atomic_txns_test.go index ead9a7049..53d71c7e1 100644 --- a/lib/block_view_atomic_txns_test.go +++ b/lib/block_view_atomic_txns_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_bitcoin_test.go b/lib/block_view_bitcoin_test.go index 0c00a1bdf..b68a18efd 100644 --- a/lib/block_view_bitcoin_test.go +++ b/lib/block_view_bitcoin_test.go @@ -14,7 +14,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/btcsuite/btcd/wire" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_creator_coin_test.go b/lib/block_view_creator_coin_test.go index 7e4a3a05b..e25af8ac9 100644 --- a/lib/block_view_creator_coin_test.go +++ b/lib/block_view_creator_coin_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_limit_order_test.go b/lib/block_view_dao_coin_limit_order_test.go index 98034b5e0..07d214725 100644 --- a/lib/block_view_dao_coin_limit_order_test.go +++ b/lib/block_view_dao_coin_limit_order_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_dao_coin_test.go b/lib/block_view_dao_coin_test.go index aa64707bd..b006c3d7d 100644 --- a/lib/block_view_dao_coin_test.go +++ b/lib/block_view_dao_coin_test.go @@ -6,7 +6,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_derived_key_test.go b/lib/block_view_derived_key_test.go index bc59dd3f9..e9d079786 100644 --- a/lib/block_view_derived_key_test.go +++ b/lib/block_view_derived_key_test.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_flush.go b/lib/block_view_flush.go index e1f407aa3..6dc4701ce 100644 --- a/lib/block_view_flush.go +++ b/lib/block_view_flush.go @@ -5,7 +5,7 @@ import ( "reflect" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_follow_test.go b/lib/block_view_follow_test.go index ef3823f63..fedb91a03 100644 --- a/lib/block_view_follow_test.go +++ b/lib/block_view_follow_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_like_test.go b/lib/block_view_like_test.go index a757c7f70..01b715475 100644 --- a/lib/block_view_like_test.go +++ b/lib/block_view_like_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_lockups.go b/lib/block_view_lockups.go index aaaaeeb81..bf8e31f97 100644 --- a/lib/block_view_lockups.go +++ b/lib/block_view_lockups.go @@ -8,7 +8,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_lockups_test.go b/lib/block_view_lockups_test.go index 571a4253c..3ab91972a 100644 --- a/lib/block_view_lockups_test.go +++ b/lib/block_view_lockups_test.go @@ -7,7 +7,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_message_test.go b/lib/block_view_message_test.go index a47c3d85c..dbe204f99 100644 --- a/lib/block_view_message_test.go +++ b/lib/block_view_message_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_nft_test.go b/lib/block_view_nft_test.go index 2f4cfd64c..ba00d8ece 100644 --- a/lib/block_view_nft_test.go +++ b/lib/block_view_nft_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_post.go b/lib/block_view_post.go index d8237e3b1..e76745fbb 100644 --- a/lib/block_view_post.go +++ b/lib/block_view_post.go @@ -13,7 +13,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/golang/glog" "github.com/pkg/errors" diff --git a/lib/block_view_post_test.go b/lib/block_view_post_test.go index 264df7d69..e84eaa1c9 100644 --- a/lib/block_view_post_test.go +++ b/lib/block_view_post_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/block_view_profile_test.go b/lib/block_view_profile_test.go index c59e37cd5..01101d005 100644 --- a/lib/block_view_profile_test.go +++ b/lib/block_view_profile_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/block_view_stake.go b/lib/block_view_stake.go index 95b7d1a86..b411673af 100644 --- a/lib/block_view_stake.go +++ b/lib/block_view_stake.go @@ -7,7 +7,7 @@ import ( "sort" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/block_view_test.go b/lib/block_view_test.go index e2c125457..0f835cc02 100644 --- a/lib/block_view_test.go +++ b/lib/block_view_test.go @@ -14,7 +14,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/golang/glog" "github.com/stretchr/testify/assert" diff --git a/lib/block_view_validator.go b/lib/block_view_validator.go index edf8f6b3a..0357d098c 100644 --- a/lib/block_view_validator.go +++ b/lib/block_view_validator.go @@ -15,7 +15,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain.go b/lib/blockchain.go index 596cf2c5e..0a645fe97 100644 --- a/lib/blockchain.go +++ b/lib/blockchain.go @@ -29,7 +29,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/davecgh/go-spew/spew" merkletree "github.com/deso-protocol/go-merkle-tree" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/blockchain_test.go b/lib/blockchain_test.go index e0fe102a3..85817fb24 100644 --- a/lib/blockchain_test.go +++ b/lib/blockchain_test.go @@ -19,7 +19,7 @@ import ( chainlib "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/lib/db_adapter.go b/lib/db_adapter.go index 8cff5d757..4eea4d0da 100644 --- a/lib/db_adapter.go +++ b/lib/db_adapter.go @@ -4,7 +4,7 @@ import ( "bytes" "sort" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils.go b/lib/db_utils.go index 0b516be27..33d326da2 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -27,7 +27,7 @@ import ( "github.com/btcsuite/btcd/btcec/v2" "github.com/davecgh/go-spew/spew" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/db_utils_test.go b/lib/db_utils_test.go index eafab46e8..095a1db44 100644 --- a/lib/db_utils_test.go +++ b/lib/db_utils_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/btcsuite/btcd/btcec/v2" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/event_manager.go b/lib/event_manager.go index 4fe654273..486f7390e 100644 --- a/lib/event_manager.go +++ b/lib/event_manager.go @@ -1,7 +1,7 @@ package lib import ( - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/google/uuid" ) diff --git a/lib/legacy_mempool.go b/lib/legacy_mempool.go index 0793aed85..05bb053fb 100644 --- a/lib/legacy_mempool.go +++ b/lib/legacy_mempool.go @@ -18,7 +18,7 @@ import ( "time" "github.com/btcsuite/btcd/btcutil" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/btcsuite/btcd/btcec/v2" diff --git a/lib/notifier.go b/lib/notifier.go index b6c88c37d..a42ffb3ab 100644 --- a/lib/notifier.go +++ b/lib/notifier.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/gernest/mention" "github.com/go-pg/pg/v10" diff --git a/lib/pos_blockchain.go b/lib/pos_blockchain.go index 80f18e72e..81e49be49 100644 --- a/lib/pos_blockchain.go +++ b/lib/pos_blockchain.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/collections" "github.com/deso-protocol/core/consensus" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_epoch.go b/lib/pos_epoch.go index ef4e3c78d..fed62d90e 100644 --- a/lib/pos_epoch.go +++ b/lib/pos_epoch.go @@ -3,7 +3,7 @@ package lib import ( "bytes" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool.go b/lib/pos_mempool.go index 5580bffe2..b2217c121 100644 --- a/lib/pos_mempool.go +++ b/lib/pos_mempool.go @@ -10,7 +10,7 @@ import ( "sync/atomic" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister.go b/lib/pos_mempool_persister.go index 79327094d..43ee7c449 100644 --- a/lib/pos_mempool_persister.go +++ b/lib/pos_mempool_persister.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_mempool_persister_test.go b/lib/pos_mempool_persister_test.go index 73887a01f..6cd1ef25d 100644 --- a/lib/pos_mempool_persister_test.go +++ b/lib/pos_mempool_persister_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_mempool_test.go b/lib/pos_mempool_test.go index 8dced4464..d04883919 100644 --- a/lib/pos_mempool_test.go +++ b/lib/pos_mempool_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/lib/pos_random_seed.go b/lib/pos_random_seed.go index 9c739c41d..6777de6e6 100644 --- a/lib/pos_random_seed.go +++ b/lib/pos_random_seed.go @@ -7,7 +7,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_cache.go b/lib/pos_snapshot_cache.go index f320e99b0..ec495633c 100644 --- a/lib/pos_snapshot_cache.go +++ b/lib/pos_snapshot_cache.go @@ -3,7 +3,7 @@ package lib import ( "sync" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/pkg/errors" ) diff --git a/lib/pos_snapshot_entries.go b/lib/pos_snapshot_entries.go index c0cc9ff8c..02cb16ceb 100644 --- a/lib/pos_snapshot_entries.go +++ b/lib/pos_snapshot_entries.go @@ -9,7 +9,7 @@ import ( "github.com/deso-protocol/core/bls" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/golang/glog" "github.com/pkg/errors" ) diff --git a/lib/postgres.go b/lib/postgres.go index 7c41102b8..f4da6642c 100644 --- a/lib/postgres.go +++ b/lib/postgres.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/deso-protocol/uint256" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" "github.com/golang/glog" diff --git a/lib/server.go b/lib/server.go index c753ccb2f..2d272b7b8 100644 --- a/lib/server.go +++ b/lib/server.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "net" "path/filepath" "reflect" diff --git a/lib/snapshot.go b/lib/snapshot.go index 7ccaf23ec..8276f03d3 100644 --- a/lib/snapshot.go +++ b/lib/snapshot.go @@ -15,7 +15,7 @@ import ( "github.com/cloudflare/circl/group" "github.com/deso-protocol/go-deadlock" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" "github.com/fatih/color" "github.com/golang/glog" "github.com/google/uuid" diff --git a/lib/txindex.go b/lib/txindex.go index 2bdbc4060..3fadfae8a 100644 --- a/lib/txindex.go +++ b/lib/txindex.go @@ -9,7 +9,7 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v3" chainlib "github.com/btcsuite/btcd/blockchain" "github.com/golang/glog" From 4f5e29dc6048b7d8e6eb14048b8c9944db4a28af Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 4 May 2025 13:58:01 -0400 Subject: [PATCH 129/131] fix reference to ristretto --- go.mod | 4 ++-- go.sum | 2 -- lib/db_utils.go | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 62723af0a..7d34e3389 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 github.com/dgraph-io/badger/v3 v3.2103.5 - github.com/dgraph-io/ristretto/v2 v2.2.0 + github.com/dgraph-io/ristretto v0.1.1 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.15.10 github.com/fatih/color v1.18.0 @@ -84,7 +84,7 @@ require ( github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 // indirect github.com/ebitengine/purego v0.8.2 // indirect diff --git a/go.sum b/go.sum index 3de0a2e66..17483375a 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,6 @@ github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0 github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= -github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= diff --git a/lib/db_utils.go b/lib/db_utils.go index 33d326da2..79fc455c9 100644 --- a/lib/db_utils.go +++ b/lib/db_utils.go @@ -9,7 +9,7 @@ import ( "encoding/json" "fmt" "github.com/deso-protocol/core/collections" - "github.com/dgraph-io/ristretto/v2/z" + "github.com/dgraph-io/ristretto/z" "io" "log" "math" From c5d701410595a5af74966ee0bda870a802c80da2 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 4 May 2025 14:04:38 -0400 Subject: [PATCH 130/131] set ristretto version to 0.2.0 to match previous state --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7d34e3389..9234b343b 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/deso-protocol/go-merkle-tree v1.0.0 github.com/deso-protocol/uint256 v1.3.2 github.com/dgraph-io/badger/v3 v3.2103.5 - github.com/dgraph-io/ristretto v0.1.1 + github.com/dgraph-io/ristretto v0.2.0 github.com/emirpasic/gods v1.18.1 github.com/ethereum/go-ethereum v1.15.10 github.com/fatih/color v1.18.0 diff --git a/go.sum b/go.sum index 17483375a..2998ffe0d 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,9 @@ github.com/deso-protocol/uint256 v1.3.2 h1:nHwqfdCKgWimWLJbiN/9DV95qDJ5lZcf8n5cA github.com/deso-protocol/uint256 v1.3.2/go.mod h1:Wq2bibbApz3TsiL+VPUnzr+UkhG4eBeQ0DpbQcjQYcA= github.com/dgraph-io/badger/v3 v3.2103.5 h1:ylPa6qzbjYRQMU6jokoj4wzcaweHylt//CH0AKt0akg= github.com/dgraph-io/badger/v3 v3.2103.5/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= From 57e482726fbc42542288a43f42bdc72bddf8be0c Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Sun, 4 May 2025 15:13:03 -0400 Subject: [PATCH 131/131] use default options for legacy mempool --- lib/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/server.go b/lib/server.go index 2d272b7b8..f9ec1e99a 100644 --- a/lib/server.go +++ b/lib/server.go @@ -593,7 +593,7 @@ func NewServer( // blocks. _mempool := NewDeSoMempool(_chain, _rateLimitFeerateNanosPerKB, _minFeeRateNanosPerKB, _blockCypherAPIKey, _runReadOnlyUtxoViewUpdater, _dataDir, - _mempoolDumpDir, false) + _mempoolDumpDir, true) // Initialize the PoS mempool. We need to initialize a best-effort UtxoView based on the current // known state of the chain. This will all be overwritten as we process blocks later on.