From 5cca0d977577952786c59bc0f5c4e1f705b15e0b Mon Sep 17 00:00:00 2001 From: Reisen Date: Fri, 5 Jul 2024 10:05:15 +0000 Subject: [PATCH 01/49] feat(program): match version with pythnet --- Cargo.lock | 304 ++++++++++++++++++++++------------------ program/rust/Cargo.toml | 6 +- 2 files changed, 174 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 933b7125..354907ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,6 +598,7 @@ dependencies = [ "bitflags", "clap_lex", "indexmap", + "once_cell", "strsim 0.10.0", "termcolor", "textwrap 0.16.0", @@ -1013,6 +1014,12 @@ dependencies = [ "syn 0.15.44", ] +[[package]] +name = "eager" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" + [[package]] name = "ed25519" version = "1.5.3" @@ -1083,18 +1090,18 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eeac5c5edb79e4e39fe8439ef35207780a11f69c52cbe424ce3dfad4cb78de6" +checksum = "2953d1df47ac0eb70086ccabf0275aa8da8591a28bd358ee2b52bd9f9e3ff9e9" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" +checksum = "8958699f9359f0b04e691a13850d48b7de329138023876d07cbd024c2c820598" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", @@ -1377,8 +1384,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -1389,9 +1398,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "goblin" -version = "0.4.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32401e89c6446dcd28185931a01b1093726d0356820ac744023e6850689bf926" +checksum = "a7666983ed0dd8d21a6f6576ee00053ca0926fb281a5522577a4dbd0f1b54143" dependencies = [ "log", "plain", @@ -1419,9 +1428,9 @@ dependencies = [ [[package]] name = "hash32" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" dependencies = [ "byteorder", ] @@ -2034,12 +2043,11 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags", - "cc", "cfg-if", "libc", "memoffset 0.6.5", @@ -2229,13 +2237,15 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf9b1c4e9a6c4de793c632496fa490bdc0e1eea73f0c91394f7b6990935d22" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" dependencies = [ "async-trait", "crossbeam-channel", - "futures", + "futures-channel", + "futures-executor", + "futures-util", "js-sys", "lazy_static", "percent-encoding", @@ -2252,20 +2262,19 @@ checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" [[package]] name = "ouroboros" -version = "0.14.2" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71643f290d126e18ac2598876d01e1d57aed164afc78fdb6e2a0c6589a1f6662" +checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db" dependencies = [ "aliasable", "ouroboros_macro", - "stable_deref_trait", ] [[package]] name = "ouroboros_macro" -version = "0.14.2" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9a247206016d424fe8497bc611e510887af5c261fbbf977877c4bb55ca4d82" +checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" dependencies = [ "Inflector", "proc-macro-error", @@ -2333,9 +2342,9 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271779f35b581956db91a3e55737327a03aa051e90b1c47aeb189508533adfd7" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest 0.10.6", ] @@ -2663,7 +2672,6 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", - "rand_pcg", ] [[package]] @@ -2724,15 +2732,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -3020,22 +3019,22 @@ checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "scroll" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" dependencies = [ "scroll_derive", ] [[package]] name = "scroll_derive" -version = "0.10.5" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -3280,9 +3279,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bf996a6f6063f8dc883b27c1b166de10461b3653d3213165174682855b7f1c" +checksum = "fd1d1d16c04e7867408f2e0383a863e272dba2eda2c4b7095c70aa1a7ad4ff36" dependencies = [ "Inflector", "base64 0.13.1", @@ -3293,20 +3292,21 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "solana-address-lookup-table-program", "solana-config-program", "solana-sdk", "solana-vote-program", "spl-token", - "spl-token-2022", + "spl-token-2022 0.6.0", "thiserror", "zstd", ] [[package]] name = "solana-address-lookup-table-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d6428d35413a20aaf4342f9874df16b54e548b0c50aab807cc98275eada8cf" +checksum = "de353d486f6e4a20cd163fc0c8391d01ff52e0ce504dbce7a45433362218b6d7" dependencies = [ "bincode", "bytemuck", @@ -3325,9 +3325,9 @@ dependencies = [ [[package]] name = "solana-banks-client" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22905a42c1668ab479ac02213565f157b8313b23bcc3b3adefef33fc5703e0ab" +checksum = "96c4d210c247714a742fa27629c30c63eefccb3fa7565168649dd58c275cb0cc" dependencies = [ "borsh", "futures", @@ -3342,9 +3342,9 @@ dependencies = [ [[package]] name = "solana-banks-interface" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc46a0ee1a44de50833125c46c1d94a56805a046668e6ea39308e62c27be307" +checksum = "bce2b9e85d389592a02fd061966b548488f2cadc03efca029551365d2d92fa14" dependencies = [ "serde", "solana-sdk", @@ -3353,9 +3353,9 @@ dependencies = [ [[package]] name = "solana-banks-server" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29591409f677da82d3ff26a39bbce697ea75d0886402f4169a970ad3f057148f" +checksum = "80855bd840b4255224b84641ef89762ec886968af43f617e1a22c20d906ce1c0" dependencies = [ "bincode", "crossbeam-channel", @@ -3373,9 +3373,9 @@ dependencies = [ [[package]] name = "solana-bpf-loader-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a303734501f4cad1ea0ef86910bd63d6560473511035941f034e0edf75b4912b" +checksum = "685347b658b1dfb5adf9f42007c4608a4d1954b8b9e36dd1035fd8fcb0918c8b" dependencies = [ "bincode", "byteorder", @@ -3392,9 +3392,9 @@ dependencies = [ [[package]] name = "solana-bucket-map" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cbb57ac4d0aa833140dd244e053ec86db9f0a2357d5482f20cd0c5a85d17be2" +checksum = "ae7219df17935400ead19e838b0a3c583dd7735745c52fca77f7966d39d41100" dependencies = [ "log", "memmap2", @@ -3407,9 +3407,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76b0c923e89c64ed398964ec2a3ddecacb5a712664d19812ebbf65eefd599145" +checksum = "30c261007a3f9424c7793d9a23e478c615f31ebc24e3ba5e52904575db36b865" dependencies = [ "chrono", "clap 2.34.0", @@ -3425,9 +3425,9 @@ dependencies = [ [[package]] name = "solana-cli-config" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11154a239590693f0d6b0b4d69193dc98ca21b3e8cc4d1d8d9b326311f520a" +checksum = "86c13afaa04bef4814c6eac7b48c47118d31ecce3dc03a609e0405a42fbddc12" dependencies = [ "dirs-next", "lazy_static", @@ -3441,9 +3441,9 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aba6eeb10249c986b335dd1c684f7353c3e48eeec3259550fcd3b79685013f2" +checksum = "0e803c468b3609af59298721f3a66ad145fa68416bfa420775f190ed9cfc6355" dependencies = [ "async-mutex", "async-trait", @@ -3462,7 +3462,6 @@ dependencies = [ "jsonrpc-core", "lazy_static", "log", - "lru", "quinn", "quinn-proto", "rand 0.7.3", @@ -3485,7 +3484,7 @@ dependencies = [ "solana-transaction-status", "solana-version", "solana-vote-program", - "spl-token-2022", + "spl-token-2022 0.6.0", "thiserror", "tokio", "tokio-stream", @@ -3496,9 +3495,9 @@ dependencies = [ [[package]] name = "solana-compute-budget-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9d97b7459e4b3af93401416e301f2cc62a52e8b85db7ec3c12afc82ae2bf32" +checksum = "47c9af546a45bb12d281bc714a688a104d3d9bfe4a0472bd249a5ebacb658f3d" dependencies = [ "solana-program-runtime", "solana-sdk", @@ -3506,9 +3505,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f530aab2e7933539698196e9c2c41521355cdd72ee270e9ec4b3154556dc9e" +checksum = "877de8a7d14e47e948f969277396b759e5361de662fa404980df9fd69d562964" dependencies = [ "bincode", "chrono", @@ -3520,9 +3519,9 @@ dependencies = [ [[package]] name = "solana-faucet" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9aae7a9b16c3e81b6532c3a5d886c58ee0cb518ee6b417f28ea0cfee972ac60" +checksum = "1d202bbd0e7cbea5e291692efbc7b633b4d6d0f2de5aa0e466d6fa7bf40fd98b" dependencies = [ "bincode", "byteorder", @@ -3544,31 +3543,43 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80650015acc4634f2bbb9a6c1a76c740a6f12854539103f3c13345b1e285b86" +checksum = "f53e63c8f2aac07bc21167e7ede9b9d010ae25523fff5c01b171d9bab9a5a394" dependencies = [ + "ahash", + "blake3", + "block-buffer 0.9.0", "bs58", "bv", + "byteorder", + "cc", + "either", "generic-array", + "getrandom 0.1.16", + "hashbrown 0.12.3", "im", "lazy_static", "log", "memmap2", + "once_cell", + "rand_core 0.6.4", "rustc_version", "serde", "serde_bytes", "serde_derive", + "serde_json", "sha2 0.10.6", "solana-frozen-abi-macro", + "subtle", "thiserror", ] [[package]] name = "solana-frozen-abi-macro" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439f7e3b931ff5ec531164f01c3eb8897c0ae72f7d537294e67b81029fa8e89f" +checksum = "daeaaa2713c06a2fe4bcdcfe7e1af55ee8a89c4d6693860b4041997af667207a" dependencies = [ "proc-macro2 1.0.56", "quote 1.0.26", @@ -3578,9 +3589,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27888ef86f0e2ea9b55be9e94f08e785c4d810b89d9d6597c9f590fdf605cd7a" +checksum = "2b502866be84a799633c0744e1d72b819a256337149e9fb6c7eee4db84ec63f5" dependencies = [ "env_logger 0.9.3", "lazy_static", @@ -3589,9 +3600,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66772934ed60505434586cf6e1b7133c266a36501c201aca1d26fff3b54e84ef" +checksum = "098178fabb0f0be17ed45ca52aea2754e49d4c41a443be5f98e1bce99b5c12bb" dependencies = [ "log", "solana-sdk", @@ -3599,9 +3610,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef37aa2167fffa33c6c97f9aa804ba155b16cc6ae4aac89bf1a5aca00939bce5" +checksum = "01dcd00c029063c09f15a1e2632570f5a549052cb3647db3bb06c2062e8351c4" dependencies = [ "crossbeam-channel", "gethostname", @@ -3613,12 +3624,12 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3093c7701cc7d3603a32aec204e5a2e4935f71b37eabd7302d5b67bdb4ea0a02" +checksum = "088b41440725aab4858d7e614fe4bb2c930ea60bba494485ed7640ed2b908724" dependencies = [ "bincode", - "clap 2.34.0", + "clap 3.2.23", "crossbeam-channel", "log", "nix", @@ -3635,9 +3646,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f5821cb3d6b17319663d5344639a8ea5eba049fcf144b839ecb4673ad2d62" +checksum = "7e9460658e4e92257ead496f8f3e36d8ec6778e09b476b7be6a518121bf0bd74" dependencies = [ "ahash", "bincode", @@ -3662,9 +3673,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "792814be1ae8ea58a43ae3cd51578517ec0d9c42c6241c9c0dc71a83d7e0e314" +checksum = "d66c02ad6002fbe7903ec96edd16352fe7964d3ee43b02053112f5304529849f" dependencies = [ "base64 0.13.1", "bincode", @@ -3675,41 +3686,49 @@ dependencies = [ "bs58", "bv", "bytemuck", + "cc", "console_error_panic_hook", "console_log", "curve25519-dalek", - "getrandom 0.1.16", + "getrandom 0.2.9", "itertools", "js-sys", "lazy_static", + "libc", "libsecp256k1", "log", + "memoffset 0.6.5", "num-derive", "num-traits", "parking_lot 0.12.1", "rand 0.7.3", + "rand_chacha 0.2.2", "rustc_version", "rustversion", "serde", "serde_bytes", "serde_derive", + "serde_json", "sha2 0.10.6", "sha3 0.10.7", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-sdk-macro", "thiserror", + "tiny-bip39", "wasm-bindgen", + "zeroize", ] [[package]] name = "solana-program-runtime" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef724d482da6354e6f2519ae17847ad383ba1e55b525899503ab13e2c97bd740" +checksum = "7d57bc75db0cbfb8e0620cef48b4de3388ed1ea5fbdcc68769da0c2b1ccf69bc" dependencies = [ "base64 0.13.1", "bincode", + "eager", "enum-iterator", "itertools", "libc", @@ -3717,21 +3736,24 @@ dependencies = [ "log", "num-derive", "num-traits", + "rand 0.7.3", "rustc_version", "serde", "solana-frozen-abi", "solana-frozen-abi-macro", "solana-measure", + "solana-metrics", "solana-sdk", "thiserror", ] [[package]] name = "solana-program-test" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e05308753a4a6ee579b97ca913c945b5db54784e99aa65fe13934b9c73428635" +checksum = "c0c4b86a72a48ebb66b2182a692b449d8590cab3eb9626a1967cc704aeb488c4" dependencies = [ + "assert_matches", "async-trait", "base64 0.13.1", "bincode", @@ -3752,9 +3774,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b5e1e75dbbde7ac578bb8e712de8d0408e299a220b50c3316b08654f5fb628" +checksum = "b2093a3edf87c3753e9548ac00d01a03da479f7fd7859f2d375528d543ecd101" dependencies = [ "lazy_static", "num_cpus", @@ -3762,9 +3784,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11989d097d9d47ee433a85714ff8a9ca940847b441dffdc78f3d0e17147da208" +checksum = "1844aed08fd8fc006732cc84fef8f4e0188d52188d7cdc859a5893553063b197" dependencies = [ "console", "dialoguer", @@ -3781,9 +3803,9 @@ dependencies = [ [[package]] name = "solana-runtime" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b738f3a58d8720735a349d555afc30be757cc3cb4e2747220e2fff03af524bb" +checksum = "3460b7a188de4b92ce5379e82ff370139cc0174c210df849e4126e701ff6885c" dependencies = [ "arrayref", "bincode", @@ -3802,11 +3824,13 @@ dependencies = [ "itertools", "lazy_static", "log", + "lru", "lz4", "memmap2", "num-derive", "num-traits", "num_cpus", + "once_cell", "ouroboros", "rand 0.7.3", "rayon", @@ -3840,9 +3864,9 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e54361f5faf0696f6f41a8f57e49ec9d99e80c333e2f5466f8501bc67313a0f" +checksum = "60cbad77fa09d23fa5e05029dec6c88e4b784be76cf6ae390f82cc04b8089e73" dependencies = [ "assert_matches", "base64 0.13.1", @@ -3867,7 +3891,7 @@ dependencies = [ "memmap2", "num-derive", "num-traits", - "pbkdf2 0.10.1", + "pbkdf2 0.11.0", "qstring", "rand 0.7.3", "rand_chacha 0.2.2", @@ -3891,9 +3915,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9ca4f24a272f09377a0b594e6cfa809247c8f8e6ab188928fa0072b6803543" +checksum = "b73f54502e7d537472bf393ffce0c252c55b534f16797029a1614d79ec0209c9" dependencies = [ "bs58", "proc-macro2 1.0.56", @@ -3904,9 +3928,9 @@ dependencies = [ [[package]] name = "solana-send-transaction-service" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c2d73b9568e50bd371cd3e50f125955db79a6c5e5f743bb90436d524a761fa" +checksum = "7b4db241054803501f57820c467b712556722ff2248e58e22b5728f773bd4fdd" dependencies = [ "crossbeam-channel", "log", @@ -3919,9 +3943,9 @@ dependencies = [ [[package]] name = "solana-stake-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5d713deaa887c0a5d042ad1ffa4e9bfc8bbc190a38b1b3fd24e83b05983c95" +checksum = "780cfa177de42c88a523acd3368cf16852bc1b0c4a9e7f3c893382dd4c9c10cb" dependencies = [ "bincode", "log", @@ -3942,9 +3966,9 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fdc52c3945a4d4fd9012e6c0c3d1a95f00ec51555e09ce6fef4806d7259c9ba" +checksum = "1b7834c7b6281d1e9f6d8207d544da0095b7e562a95b99ecbb7461408cec56eb" dependencies = [ "crossbeam-channel", "futures-util", @@ -3971,9 +3995,9 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9287c2199bd71b577641f00d1e4023e4908f59050a83fb14a1fd19327537d289" +checksum = "feacea79beaefa2aec4ea02bd56573845bcf2a480858f7d666077138928fc259" dependencies = [ "Inflector", "base64 0.13.1", @@ -3986,6 +4010,7 @@ dependencies = [ "serde_derive", "serde_json", "solana-account-decoder", + "solana-address-lookup-table-program", "solana-measure", "solana-metrics", "solana-sdk", @@ -3993,15 +4018,15 @@ dependencies = [ "spl-associated-token-account", "spl-memo", "spl-token", - "spl-token-2022", + "spl-token-2022 0.6.0", "thiserror", ] [[package]] name = "solana-version" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0312c45b6b94e220c5cc9ec1de4fa0fedf21c33ccb17d932ff6191ac237f405" +checksum = "2301b32765f9c37786b1d76b46c0d21be9475ad39d2f0e0494cb9cfe06182149" dependencies = [ "log", "rustc_version", @@ -4015,9 +4040,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93db350f670ad166107e4a241a1e9f1deff65fc17402900b443275e2fd8834c" +checksum = "fa94c3c98a33f9825c83ea3d68db39fcbfa94b66772d9a8eb9e16e711966b453" dependencies = [ "bincode", "log", @@ -4036,9 +4061,9 @@ dependencies = [ [[package]] name = "solana-zk-token-proof-program" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb26d981d61ce9b9b75ac5f5b04dcdeb1211cf209ae3a3bb4842c3c84894e3e" +checksum = "c7ebd4f7b2ed789d3c122b40e6590c0fcdb34d1029a6eb7ebb463e96beb5db35" dependencies = [ "bytemuck", "getrandom 0.1.16", @@ -4051,9 +4076,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.13.3" +version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a148ec781994129fea0ede0628131a8f78bc1aaa28fd7d14d72aacdd065c1" +checksum = "b28c5ec36aa1393174f7ea18c0cb809af82c10977bc5b2e1240a6b4b048b8f24" dependencies = [ "aes-gcm-siv", "arrayref", @@ -4064,6 +4089,7 @@ dependencies = [ "cipher 0.4.4", "curve25519-dalek", "getrandom 0.1.16", + "itertools", "lazy_static", "merlin", "num-derive", @@ -4081,9 +4107,9 @@ dependencies = [ [[package]] name = "solana_rbpf" -version = "0.2.24" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e138f6d6d4eb6a65f8e9f01ca620bc9907d79648d5038a69dd3f07b6ed3f1f" +checksum = "80a28c5dfe7e8af38daa39d6561c8e8b9ed7a2f900951ebe7362ad6348d36c73" dependencies = [ "byteorder", "combine", @@ -4091,11 +4117,10 @@ dependencies = [ "hash32", "libc", "log", - "rand 0.7.3", + "rand 0.8.5", "rustc-demangle", "scroll", "thiserror", - "time 0.1.45", ] [[package]] @@ -4116,9 +4141,9 @@ dependencies = [ [[package]] name = "spl-associated-token-account" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a33ecc83137583902c3e13c02f34151c8b2f2b74120f9c2b3ff841953e083d" +checksum = "fbc000f0fdf1f12f99d77d398137c1751345b18c88258ce0f99b7872cf6c9bd6" dependencies = [ "assert_matches", "borsh", @@ -4126,7 +4151,7 @@ dependencies = [ "num-traits", "solana-program", "spl-token", - "spl-token-2022", + "spl-token-2022 0.5.0", "thiserror", ] @@ -4156,9 +4181,9 @@ dependencies = [ [[package]] name = "spl-token-2022" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a97cbf60b91b610c846ccf8eecca96d92a24a19ffbf9fe06cd0c84e76ec45e" +checksum = "0edb869dbe159b018f17fb9bfa67118c30f232d7f54a73742bc96794dff77ed8" dependencies = [ "arrayref", "bytemuck", @@ -4173,10 +4198,22 @@ dependencies = [ ] [[package]] -name = "stable_deref_trait" -version = "1.2.0" +name = "spl-token-2022" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "67fcd758e8d22c5fce17315015f5ff319604d1a6e57a73c72795639dba898890" +dependencies = [ + "arrayref", + "bytemuck", + "num-derive", + "num-traits", + "num_enum", + "solana-program", + "solana-zk-token-sdk", + "spl-memo", + "spl-token", + "thiserror", +] [[package]] name = "static_assertions" @@ -4288,9 +4325,9 @@ dependencies = [ [[package]] name = "tarpc" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b85d0a9369a919ba0db919b142a2b704cd207dfc676f7a43c2d105d0bc225487" +checksum = "1c38a012bed6fb9681d3bf71ffaa4f88f3b4b9ed3198cda6e4c8462d24d4bb80" dependencies = [ "anyhow", "fnv", @@ -4654,10 +4691,11 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.15.0" +version = "0.17.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599f388ecb26b28d9c1b2e4437ae019a7b336018b45ed911458cd9ebf91129f6" +checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" dependencies = [ + "once_cell", "opentelemetry", "tracing", "tracing-core", @@ -4666,9 +4704,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.25" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "sharded-slab", "thread_local", diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index df753755..c0beeafb 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -9,7 +9,7 @@ publish = false bindgen = "0.60.1" [dependencies] -solana-program = "=1.13.3" +solana-program = "=1.14.17" bytemuck = "1.11.0" thiserror = "1.0" num-derive = "0.3" @@ -20,8 +20,8 @@ strum = { version = "0.24.1", features = ["derive"], optional = true } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60144002053a93f424be70decd8a8ccb8d618d81"} [dev-dependencies] -solana-program-test = "=1.13.3" -solana-sdk = "=1.13.3" +solana-program-test = "=1.14.17" +solana-sdk = "=1.14.17" tokio = "1.14.1" hex = "0.3.1" quickcheck = "1" From a8e1d327a9d11a7c2a74461a5bd76b03eb6f81d5 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 5 Jul 2024 14:46:21 +0100 Subject: [PATCH 02/49] feat: validator access point (wip) --- program/rust/Cargo.toml | 3 ++- program/rust/src/lib.rs | 1 + program/rust/src/validator.rs | 40 +++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 program/rust/src/validator.rs diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index c0beeafb..5c94185f 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -18,6 +18,7 @@ byteorder = "1.4.3" serde = { version = "1.0", features = ["derive"], optional = true } strum = { version = "0.24.1", features = ["derive"], optional = true } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60144002053a93f424be70decd8a8ccb8d618d81"} +solana-sdk = { version = "=1.14.17", optional = true } [dev-dependencies] solana-program-test = "=1.14.17" @@ -37,7 +38,7 @@ csv = "1.1" [features] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] -library = [] +library = ["solana-sdk"] [lib] crate-type = ["cdylib", "lib"] diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index b0e3b877..d9862c9e 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -9,6 +9,7 @@ mod error; mod instruction; mod processor; mod utils; +pub mod validator; #[cfg(test)] mod tests; diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs new file mode 100644 index 00000000..9dc55512 --- /dev/null +++ b/program/rust/src/validator.rs @@ -0,0 +1,40 @@ +use { + crate::{ + accounts::{ + AccountHeader, + PriceAccount, + PythAccount, + }, + c_oracle_header::PC_MAGIC, + error::OracleError, + utils::pyth_assert, + }, + solana_sdk::program_error::ProgramError, + std::mem::size_of, +}; + +pub fn accumulate_price(price_account_info: &mut [u8]) -> Result<(), ProgramError> { + // TODO: don't return error on non-price account? + pyth_assert( + price_account_info.len() >= PriceAccount::MINIMUM_SIZE, + OracleError::AccountTooSmall.into(), + )?; + + { + let account_header = bytemuck::from_bytes::( + &price_account_info[0..size_of::()], + ); + + pyth_assert( + account_header.magic_number == PC_MAGIC + && account_header.account_type == PriceAccount::ACCOUNT_TYPE, + OracleError::InvalidAccountHeader.into(), + )?; + } + + let _price_info = bytemuck::from_bytes_mut::( + &mut price_account_info[0..size_of::()], + ); + + todo!() +} From 4245d374745d2519bc5342494d26130d91ddace7 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 5 Jul 2024 15:18:39 +0100 Subject: [PATCH 03/49] feat: add price account flags --- Cargo.lock | 89 ++++++++++++++------- program/rust/Cargo.toml | 6 ++ program/rust/src/accounts.rs | 2 +- program/rust/src/accounts/price.rs | 13 ++- program/rust/src/lib.rs | 2 + program/rust/src/processor/add_publisher.rs | 16 +++- program/rust/src/tests/test_sizes.rs | 2 +- 7 files changed, 94 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 354907ce..6b598efc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,7 +150,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.20", + "time 0.3.7", ] [[package]] @@ -272,7 +272,7 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cexpr", "clang-sys", "clap 3.2.23", @@ -295,6 +295,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "bytemuck", +] + [[package]] name = "bitmaps" version = "2.1.0" @@ -581,7 +590,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags", + "bitflags 1.3.2", "strsim 0.8.0", "textwrap 0.11.0", "unicode-width", @@ -595,7 +604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_lex", "indexmap", "once_cell", @@ -2047,7 +2056,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.6.5", @@ -2202,6 +2211,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -2519,6 +2537,7 @@ version = "2.26.0" dependencies = [ "bincode", "bindgen", + "bitflags 2.6.0", "bytemuck", "byteorder", "csv", @@ -2537,7 +2556,10 @@ dependencies = [ "strum", "test-generator", "thiserror", + "time 0.3.7", + "time-macros", "tokio", + "tracing-subscriber", ] [[package]] @@ -2771,7 +2793,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring", - "time 0.3.20", + "time 0.3.7", "yasna", ] @@ -2781,7 +2803,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2790,7 +2812,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2925,7 +2947,7 @@ version = "0.37.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3053,7 +3075,7 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -3679,7 +3701,7 @@ checksum = "d66c02ad6002fbe7903ec96edd16352fe7964d3ee43b02053112f5304529849f" dependencies = [ "base64 0.13.1", "bincode", - "bitflags", + "bitflags 1.3.2", "blake3", "borsh", "borsh-derive", @@ -3871,7 +3893,7 @@ dependencies = [ "assert_matches", "base64 0.13.1", "bincode", - "bitflags", + "bitflags 1.3.2", "borsh", "bs58", "bytemuck", @@ -4450,30 +4472,21 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" dependencies = [ "itoa", - "serde", - "time-core", + "libc", + "num_threads", "time-macros", ] -[[package]] -name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" -dependencies = [ - "time-core", -] +checksum = "25eb0ca3468fc0acc11828786797f6ef9aa1555e4a211a60d64cc8e4d1be47d6" [[package]] name = "tiny-bip39" @@ -4689,6 +4702,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" version = "0.17.4" @@ -4704,13 +4728,16 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "5cf865b5ddc38e503a29c41c4843e616a73028ae18c637bc3eb2afaef4909c84" dependencies = [ + "ansi_term", "sharded-slab", + "smallvec", "thread_local", "tracing-core", + "tracing-log", ] [[package]] @@ -5226,7 +5253,7 @@ dependencies = [ "oid-registry", "rusticata-macros", "thiserror", - "time 0.3.20", + "time 0.3.7", ] [[package]] @@ -5253,7 +5280,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.20", + "time 0.3.7", ] [[package]] diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 5c94185f..8ba4b929 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -19,6 +19,7 @@ serde = { version = "1.0", features = ["derive"], optional = true } strum = { version = "0.24.1", features = ["derive"], optional = true } pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60144002053a93f424be70decd8a8ccb8d618d81"} solana-sdk = { version = "=1.14.17", optional = true } +bitflags = { version = "2.6.0", features = ["bytemuck"] } [dev-dependencies] solana-program-test = "=1.14.17" @@ -35,6 +36,11 @@ serde_json = "1.0" test-generator = "0.3.1" csv = "1.1" +# Downgrade to be compatible with Rust 1.60 +tracing-subscriber = "=0.3.0" +time-macros = "=0.2.3" +time = "=0.3.7" + [features] check = [] # Skips make build in build.rs, use with cargo-clippy and cargo-check debug = [] diff --git a/program/rust/src/accounts.rs b/program/rust/src/accounts.rs index 5958fc88..f05e0685 100644 --- a/program/rust/src/accounts.rs +++ b/program/rust/src/accounts.rs @@ -34,7 +34,6 @@ use { std::borrow::BorrowMut, }; - mod mapping; mod permission; mod price; @@ -53,6 +52,7 @@ pub use { permission::PermissionAccount, price::{ PriceAccount, + PriceAccountFlags, PriceComponent, PriceCumulative, PriceEma, diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 82f2d62c..c708cc02 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -33,6 +33,7 @@ mod price_pythnet { }, error::OracleError, }, + bitflags::bitflags, }; /// Pythnet-only extended price account format. This extension is @@ -65,8 +66,9 @@ mod price_pythnet { pub message_sent_: u8, /// Configurable max latency in slots between send and receive pub max_latency_: u8, + /// Various flags + pub flags: PriceAccountFlags, /// Unused placeholder for alignment - pub unused_2_: i8, pub unused_3_: i32, /// Corresponding product account pub product_account: Pubkey, @@ -91,6 +93,15 @@ mod price_pythnet { pub price_cumulative: PriceCumulative, } + bitflags! { + #[repr(C)] + #[derive(Copy, Clone, Pod, Zeroable)] + pub struct PriceAccountFlags: u8 { + /// If set, the program doesn't do accumulation, but validator does. + const ACCUMULATOR_V2 = 1; + } + } + impl PriceAccountPythnet { pub fn as_price_feed_message(&self, key: &Pubkey) -> PriceFeedMessage { let (price, conf, publish_time) = if self.agg_.status_ == PC_STATUS_TRADING { diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index d9862c9e..ece6eb01 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -9,6 +9,8 @@ mod error; mod instruction; mod processor; mod utils; + +#[cfg(feature = "library")] pub mod validator; #[cfg(test)] diff --git a/program/rust/src/processor/add_publisher.rs b/program/rust/src/processor/add_publisher.rs index 841de3b9..0d895f20 100644 --- a/program/rust/src/processor/add_publisher.rs +++ b/program/rust/src/processor/add_publisher.rs @@ -2,6 +2,7 @@ use { crate::{ accounts::{ PriceAccount, + PriceAccountFlags, PriceComponent, PythAccount, }, @@ -33,6 +34,13 @@ use { std::mem::size_of, }; +const ENABLE_ACCUMULATOR_V2: [u8; 32] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, +]; +const DISABLE_ACCUMULATOR_V2: [u8; 32] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, +]; + /// Add publisher to symbol account // account[0] funding account [signer writable] // account[1] price account [signer writable] @@ -62,7 +70,6 @@ pub fn add_publisher( &cmd_args.header, )?; - let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // Use the call with the default pubkey (000..) as a trigger to sort the publishers as a @@ -71,6 +78,12 @@ pub fn add_publisher( let num_comps = try_convert::(price_data.num_)?; sort_price_comps(&mut price_data.comp_, num_comps)?; return Ok(()); + } else if cmd_args.publisher == Pubkey::from(ENABLE_ACCUMULATOR_V2) { + price_data.flags.insert(PriceAccountFlags::ACCUMULATOR_V2); + return Ok(()); + } else if cmd_args.publisher == Pubkey::from(DISABLE_ACCUMULATOR_V2) { + price_data.flags.remove(PriceAccountFlags::ACCUMULATOR_V2); + return Ok(()); } if price_data.num_ >= PC_NUM_COMP { @@ -224,7 +237,6 @@ mod test { let mut rust_std_sorted_comps = comps.get(..num_comps).unwrap().to_vec(); rust_std_sorted_comps.sort_by_key(|x| x.pub_); - assert_eq!(sort_price_comps(&mut comps, num_comps), Ok(())); assert_eq!(comps.get(..num_comps).unwrap(), rust_std_sorted_comps); } diff --git a/program/rust/src/tests/test_sizes.rs b/program/rust/src/tests/test_sizes.rs index 9ec1e37b..7a1d900d 100644 --- a/program/rust/src/tests/test_sizes.rs +++ b/program/rust/src/tests/test_sizes.rs @@ -112,7 +112,7 @@ fn test_offsets() { #[test] fn test_pubkey() { let default_pubkey = Pubkey::default(); - let zero_pubkey = Pubkey::new(&[0u8; 32]); + let zero_pubkey = Pubkey::from([0u8; 32]); let unique_pubkey = Pubkey::new_unique(); assert_eq!(default_pubkey, zero_pubkey); From 762ad5406bb78e9519cea30b3ff58ab396dd095f Mon Sep 17 00:00:00 2001 From: Reisen Date: Mon, 8 Jul 2024 08:25:57 +0000 Subject: [PATCH 04/49] feat(program): fill out validator aggregation code --- program/rust/src/processor/upd_price.rs | 2 +- program/rust/src/validator.rs | 68 +++++++++++++++++++++---- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 278673b4..0c6759bb 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -287,7 +287,7 @@ pub fn upd_price( /// to get the result faster if the list is sorted. If the list is not sorted, it falls back to /// a linear search. #[inline(always)] -fn find_publisher_index(comps: &[PriceComponent], key: &Pubkey) -> Option { +pub fn find_publisher_index(comps: &[PriceComponent], key: &Pubkey) -> Option { // Verify that publisher is authorized by initially binary searching // for the publisher's component in the price account. The binary // search might not work if the publisher list is not sorted; therefore diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 9dc55512..1792dd3a 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -4,21 +4,21 @@ use { AccountHeader, PriceAccount, PythAccount, - }, - c_oracle_header::PC_MAGIC, - error::OracleError, - utils::pyth_assert, + }, c_oracle_header::PC_MAGIC, error::OracleError, processor::{c_upd_aggregate, c_upd_twap}, utils::pyth_assert }, solana_sdk::program_error::ProgramError, std::mem::size_of, }; -pub fn accumulate_price(price_account_info: &mut [u8]) -> Result<(), ProgramError> { +// Attempts to validate and access the contents of an account as a PriceAccount. +fn validate_price_account( + price_account_info: &mut [u8], +) -> Option<&mut PriceAccount> { // TODO: don't return error on non-price account? pyth_assert( price_account_info.len() >= PriceAccount::MINIMUM_SIZE, OracleError::AccountTooSmall.into(), - )?; + ).ok()?; { let account_header = bytemuck::from_bytes::( @@ -29,12 +29,58 @@ pub fn accumulate_price(price_account_info: &mut [u8]) -> Result<(), ProgramErro account_header.magic_number == PC_MAGIC && account_header.account_type == PriceAccount::ACCOUNT_TYPE, OracleError::InvalidAccountHeader.into(), - )?; + ).ok()?; } - let _price_info = bytemuck::from_bytes_mut::( - &mut price_account_info[0..size_of::()], - ); + Some(bytemuck::from_bytes_mut::(&mut price_account_info[0..size_of::()])) +} + +fn update_aggregate( + slot: u64, + timestamp: i64, + price_account: &mut PriceAccount, +) -> Result<(), ProgramError> { + // NOTE: c_upd_aggregate must use a raw pointer to price data. We already + // have the exclusive mut reference so we can simply cast before calling + // the function. + // + // TODO: Verify the bytemuck::from_mut_bytes function is as safe as if + // we called `try_borrow_mut_data()`` + let updated = + unsafe { c_upd_aggregate(price_account as *mut PriceAccount as *mut u8, slot, timestamp) }; + + // If the aggregate was successfully updated, calculate the difference + // and update TWAP. + if !updated { + return Ok(()); + } + + let agg_diff = (slot as i64) - price_account.prev_slot_ as i64; - todo!() + // See comment on unsafe `c_upd_aggregate` call above for details. + unsafe { + c_upd_twap(price_account as *mut PriceAccount as *mut u8, agg_diff); + } + + // We want to send a message every time the aggregate price updates. However, during the migration, + // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag + // ensures that after every aggregate update, the next publisher who provides the accumulator accounts + // will send the message. + price_account.message_sent_ = 0; + price_account.update_price_cumulative()?; + + Ok(()) +} + +pub fn aggregate_price( + slot: u64, + timestamp: i64, + price_account_data: &mut [u8], +) -> Result { + if let Some(price_account) = validate_price_account(price_account_data) { + update_aggregate(slot, timestamp, price_account)?; + Ok(true) + } else { + Ok(false) + } } From 6eaf432c9b3e87d4dd76fe611641c7a729c77ba2 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Mon, 8 Jul 2024 13:46:16 +0100 Subject: [PATCH 05/49] feat: don't aggregate on v2, clear message buffer, report validator aggregation errors --- program/rust/src/accounts/price.rs | 5 +- program/rust/src/processor/add_publisher.rs | 4 +- program/rust/src/processor/upd_price.rs | 97 +++++++++++++-------- program/rust/src/validator.rs | 64 +++++++++----- 4 files changed, 110 insertions(+), 60 deletions(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index c708cc02..ecd154fa 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -98,7 +98,10 @@ mod price_pythnet { #[derive(Copy, Clone, Pod, Zeroable)] pub struct PriceAccountFlags: u8 { /// If set, the program doesn't do accumulation, but validator does. - const ACCUMULATOR_V2 = 1; + const ACCUMULATOR_V2 = 0b1; + /// If unset, the program will remove old messages from its message buffer account + /// and set this flag. + const MESSAGE_BUFFER_CLEARED = 0b10; } } diff --git a/program/rust/src/processor/add_publisher.rs b/program/rust/src/processor/add_publisher.rs index 0d895f20..ff2e9361 100644 --- a/program/rust/src/processor/add_publisher.rs +++ b/program/rust/src/processor/add_publisher.rs @@ -82,7 +82,9 @@ pub fn add_publisher( price_data.flags.insert(PriceAccountFlags::ACCUMULATOR_V2); return Ok(()); } else if cmd_args.publisher == Pubkey::from(DISABLE_ACCUMULATOR_V2) { - price_data.flags.remove(PriceAccountFlags::ACCUMULATOR_V2); + price_data + .flags + .remove(PriceAccountFlags::ACCUMULATOR_V2 | PriceAccountFlags::MESSAGE_BUFFER_CLEARED); return Ok(()); } diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 0c6759bb..37f673fd 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -2,6 +2,7 @@ use { crate::{ accounts::{ PriceAccount, + PriceAccountFlags, PriceComponent, PriceInfo, PythOracleSerialize, @@ -131,6 +132,7 @@ pub fn upd_price( let publisher_index: usize; let latest_aggregate_price: PriceInfo; + let flags: PriceAccountFlags; // The price_data borrow happens in a scope because it must be // dropped before we borrow again as raw data pointer for the C @@ -159,39 +161,43 @@ pub fn upd_price( && cmd_args.publishing_slot <= clock.slot), ProgramError::InvalidArgument, )?; - } - // Try to update the aggregate - #[allow(unused_variables)] - if clock.slot > latest_aggregate_price.pub_slot_ { - let updated = unsafe { - // NOTE: c_upd_aggregate must use a raw pointer to price - // data. Solana's `.borrow_*` methods require exclusive - // access, i.e. no other borrow can exist for the account. - c_upd_aggregate( - price_account.try_borrow_mut_data()?.as_mut_ptr(), - clock.slot, - clock.unix_timestamp, - ) - }; + flags = price_data.flags; + } - // If the aggregate was successfully updated, calculate the difference and update TWAP. - if updated { - let agg_diff = (clock.slot as i64) - - load_checked::(price_account, cmd_args.header.version)?.prev_slot_ - as i64; - // Encapsulate TWAP update logic in a function to minimize unsafe block scope. - unsafe { - c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); + if !flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + // Try to update the aggregate + #[allow(unused_variables)] + if clock.slot > latest_aggregate_price.pub_slot_ { + let updated = unsafe { + // NOTE: c_upd_aggregate must use a raw pointer to price + // data. Solana's `.borrow_*` methods require exclusive + // access, i.e. no other borrow can exist for the account. + c_upd_aggregate( + price_account.try_borrow_mut_data()?.as_mut_ptr(), + clock.slot, + clock.unix_timestamp, + ) + }; + + // If the aggregate was successfully updated, calculate the difference and update TWAP. + if updated { + let agg_diff = (clock.slot as i64) + - load_checked::(price_account, cmd_args.header.version)? + .prev_slot_ as i64; + // Encapsulate TWAP update logic in a function to minimize unsafe block scope. + unsafe { + c_upd_twap(price_account.try_borrow_mut_data()?.as_mut_ptr(), agg_diff); + } + let mut price_data = + load_checked::(price_account, cmd_args.header.version)?; + // We want to send a message every time the aggregate price updates. However, during the migration, + // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag + // ensures that after every aggregate update, the next publisher who provides the accumulator accounts + // will send the message. + price_data.message_sent_ = 0; + price_data.update_price_cumulative()?; } - let mut price_data = - load_checked::(price_account, cmd_args.header.version)?; - // We want to send a message every time the aggregate price updates. However, during the migration, - // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag - // ensures that after every aggregate update, the next publisher who provides the accumulator accounts - // will send the message. - price_data.message_sent_ = 0; - price_data.update_price_cumulative()?; } } @@ -199,7 +205,15 @@ pub fn upd_price( let mut price_data = load_checked::(price_account, cmd_args.header.version)?; // Feature-gated accumulator-specific code, used only on pythnet/pythtest - { + let need_message_buffer_update = if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + // We need to clear old messages. + !flags.contains(PriceAccountFlags::MESSAGE_BUFFER_CLEARED) + } else { + // V1 + true + }; + + if need_message_buffer_update { if let Some(accumulator_accounts) = maybe_accumulator_accounts { if price_data.message_sent_ == 0 { // Check that the oracle PDA is correctly configured for the program we are calling. @@ -232,12 +246,16 @@ pub fn upd_price( }, ]; - let message = vec![ - price_data - .as_price_feed_message(price_account.key) - .to_bytes(), - price_data.as_twap_message(price_account.key).to_bytes(), - ]; + let message = if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + vec![] + } else { + vec![ + price_data + .as_price_feed_message(price_account.key) + .to_bytes(), + price_data.as_twap_message(price_account.key).to_bytes(), + ] + }; // Append a TWAP message if available @@ -257,6 +275,11 @@ pub fn upd_price( invoke_signed(&create_inputs_ix, accounts, &[auth_seeds_with_bump])?; price_data.message_sent_ = 1; + if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + price_data + .flags + .insert(PriceAccountFlags::MESSAGE_BUFFER_CLEARED); + } } } } diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 1792dd3a..2e5a230f 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -1,24 +1,24 @@ use { crate::{ - accounts::{ - AccountHeader, - PriceAccount, - PythAccount, - }, c_oracle_header::PC_MAGIC, error::OracleError, processor::{c_upd_aggregate, c_upd_twap}, utils::pyth_assert + accounts::{AccountHeader, PriceAccount, PriceAccountFlags, PythAccount}, + c_oracle_header::PC_MAGIC, + error::OracleError, + processor::{c_upd_aggregate, c_upd_twap}, + utils::pyth_assert, }, - solana_sdk::program_error::ProgramError, std::mem::size_of, }; // Attempts to validate and access the contents of an account as a PriceAccount. fn validate_price_account( price_account_info: &mut [u8], -) -> Option<&mut PriceAccount> { +) -> Result<&mut PriceAccount, AggregationError> { // TODO: don't return error on non-price account? pyth_assert( price_account_info.len() >= PriceAccount::MINIMUM_SIZE, OracleError::AccountTooSmall.into(), - ).ok()?; + ) + .map_err(|_| AggregationError::NotPriceFeedAccount)?; { let account_header = bytemuck::from_bytes::( @@ -29,25 +29,38 @@ fn validate_price_account( account_header.magic_number == PC_MAGIC && account_header.account_type == PriceAccount::ACCOUNT_TYPE, OracleError::InvalidAccountHeader.into(), - ).ok()?; + ) + .map_err(|_| AggregationError::NotPriceFeedAccount)?; } - Some(bytemuck::from_bytes_mut::(&mut price_account_info[0..size_of::()])) + let data = bytemuck::from_bytes_mut::( + &mut price_account_info[0..size_of::()], + ); + if !data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + return Err(AggregationError::LegacyAggregationMode); + } + + Ok(data) } fn update_aggregate( slot: u64, timestamp: i64, price_account: &mut PriceAccount, -) -> Result<(), ProgramError> { +) -> Result<(), AggregationError> { // NOTE: c_upd_aggregate must use a raw pointer to price data. We already // have the exclusive mut reference so we can simply cast before calling // the function. // // TODO: Verify the bytemuck::from_mut_bytes function is as safe as if // we called `try_borrow_mut_data()`` - let updated = - unsafe { c_upd_aggregate(price_account as *mut PriceAccount as *mut u8, slot, timestamp) }; + let updated = unsafe { + c_upd_aggregate( + price_account as *mut PriceAccount as *mut u8, + slot, + timestamp, + ) + }; // If the aggregate was successfully updated, calculate the difference // and update TWAP. @@ -67,20 +80,29 @@ fn update_aggregate( // ensures that after every aggregate update, the next publisher who provides the accumulator accounts // will send the message. price_account.message_sent_ = 0; - price_account.update_price_cumulative()?; + price_account + .update_price_cumulative() + .map_err(|_| AggregationError::NotTradingStatus)?; Ok(()) } +#[derive(Debug, thiserror::Error)] +pub enum AggregationError { + #[error("NotPriceFeedAccount")] + NotPriceFeedAccount, + #[error("LegacyAggregationMode")] + LegacyAggregationMode, + #[error("NotTradingStatus")] + NotTradingStatus, +} + pub fn aggregate_price( slot: u64, timestamp: i64, price_account_data: &mut [u8], -) -> Result { - if let Some(price_account) = validate_price_account(price_account_data) { - update_aggregate(slot, timestamp, price_account)?; - Ok(true) - } else { - Ok(false) - } +) -> Result<(), AggregationError> { + let price_account = validate_price_account(price_account_data)?; + update_aggregate(slot, timestamp, price_account)?; + Ok(()) } From 27212eeda219b5d8d5d7c1e128fc2dd407861a8a Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Mon, 8 Jul 2024 14:32:12 +0100 Subject: [PATCH 06/49] feat: return price feed messages from validator aggregation --- program/rust/src/validator.rs | 60 +++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 2e5a230f..7c138525 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -1,11 +1,21 @@ use { crate::{ - accounts::{AccountHeader, PriceAccount, PriceAccountFlags, PythAccount}, + accounts::{ + AccountHeader, + PriceAccount, + PriceAccountFlags, + PythAccount, + PythOracleSerialize, + }, c_oracle_header::PC_MAGIC, error::OracleError, - processor::{c_upd_aggregate, c_upd_twap}, + processor::{ + c_upd_aggregate, + c_upd_twap, + }, utils::pyth_assert, }, + solana_sdk::pubkey::Pubkey, std::mem::size_of, }; @@ -37,17 +47,14 @@ fn validate_price_account( &mut price_account_info[0..size_of::()], ); if !data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { - return Err(AggregationError::LegacyAggregationMode); + return Err(AggregationError::V1AggregationMode); } Ok(data) } -fn update_aggregate( - slot: u64, - timestamp: i64, - price_account: &mut PriceAccount, -) -> Result<(), AggregationError> { +// Returns true if the price account data has been modified and should be committed. +fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) -> bool { // NOTE: c_upd_aggregate must use a raw pointer to price data. We already // have the exclusive mut reference so we can simply cast before calling // the function. @@ -65,7 +72,7 @@ fn update_aggregate( // If the aggregate was successfully updated, calculate the difference // and update TWAP. if !updated { - return Ok(()); + return false; } let agg_diff = (slot as i64) - price_account.prev_slot_ as i64; @@ -80,29 +87,42 @@ fn update_aggregate( // ensures that after every aggregate update, the next publisher who provides the accumulator accounts // will send the message. price_account.message_sent_ = 0; - price_account - .update_price_cumulative() - .map_err(|_| AggregationError::NotTradingStatus)?; + if price_account.update_price_cumulative().is_err() { + // Revert in case of a bad trading status. + return false; + } - Ok(()) + true } #[derive(Debug, thiserror::Error)] pub enum AggregationError { #[error("NotPriceFeedAccount")] NotPriceFeedAccount, - #[error("LegacyAggregationMode")] - LegacyAggregationMode, - #[error("NotTradingStatus")] - NotTradingStatus, + #[error("V1AggregationMode")] + V1AggregationMode, +} + +pub struct AggregationOutcome { + pub messages: [Vec; 2], + pub commit: bool, } pub fn aggregate_price( slot: u64, timestamp: i64, + price_account_pubkey: &Pubkey, price_account_data: &mut [u8], -) -> Result<(), AggregationError> { +) -> Result { let price_account = validate_price_account(price_account_data)?; - update_aggregate(slot, timestamp, price_account)?; - Ok(()) + let commit = update_aggregate(slot, timestamp, price_account); + let messages = [ + price_account + .as_price_feed_message(price_account_pubkey) + .to_bytes(), + price_account + .as_twap_message(price_account_pubkey) + .to_bytes(), + ]; + Ok(AggregationOutcome { messages, commit }) } From 847b3f52d64e04de23d17264d0b4393b55785697 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Wed, 10 Jul 2024 13:02:56 +0100 Subject: [PATCH 07/49] test: enable v2 aggregation --- program/rust/src/processor.rs | 6 +- program/rust/src/processor/add_publisher.rs | 4 +- program/rust/src/tests/mod.rs | 1 + program/rust/src/tests/test_aggregate_v2.rs | 101 ++++++++++++++++++++ 4 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 program/rust/src/tests/test_aggregate_v2.rs diff --git a/program/rust/src/processor.rs b/program/rust/src/processor.rs index b8902165..7018a948 100644 --- a/program/rust/src/processor.rs +++ b/program/rust/src/processor.rs @@ -30,7 +30,11 @@ mod upd_product; pub use { add_price::add_price, add_product::add_product, - add_publisher::add_publisher, + add_publisher::{ + add_publisher, + DISABLE_ACCUMULATOR_V2, + ENABLE_ACCUMULATOR_V2, + }, del_price::del_price, del_product::del_product, del_publisher::del_publisher, diff --git a/program/rust/src/processor/add_publisher.rs b/program/rust/src/processor/add_publisher.rs index ff2e9361..f5768447 100644 --- a/program/rust/src/processor/add_publisher.rs +++ b/program/rust/src/processor/add_publisher.rs @@ -34,10 +34,10 @@ use { std::mem::size_of, }; -const ENABLE_ACCUMULATOR_V2: [u8; 32] = [ +pub const ENABLE_ACCUMULATOR_V2: [u8; 32] = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, ]; -const DISABLE_ACCUMULATOR_V2: [u8; 32] = [ +pub const DISABLE_ACCUMULATOR_V2: [u8; 32] = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, ]; diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index 8208d153..ccd4f585 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -2,6 +2,7 @@ mod pyth_simulator; mod test_add_price; mod test_add_product; mod test_add_publisher; +mod test_aggregate_v2; mod test_aggregation; mod test_c_code; mod test_check_valid_signable_account_or_permissioned_funding_account; diff --git a/program/rust/src/tests/test_aggregate_v2.rs b/program/rust/src/tests/test_aggregate_v2.rs new file mode 100644 index 00000000..7e1997f3 --- /dev/null +++ b/program/rust/src/tests/test_aggregate_v2.rs @@ -0,0 +1,101 @@ +use { + crate::{ + accounts::{ + PermissionAccount, + PriceAccount, + PriceAccountFlags, + PythAccount, + }, + c_oracle_header::PC_VERSION, + deserialize::load_checked, + instruction::{ + AddPublisherArgs, + OracleCommand, + }, + processor::{ + process_instruction, + ENABLE_ACCUMULATOR_V2, + }, + tests::test_utils::AccountSetup, + }, + bytemuck::bytes_of, + solana_program::{ + pubkey::Pubkey, + rent::Rent, + }, +}; + +#[test] +fn test_aggregate_v2() { + let program_id = Pubkey::new_unique(); + let publisher = Pubkey::new_unique(); + + let mut cmd = AddPublisherArgs { + header: OracleCommand::AddPublisher.into(), + publisher, + }; + let mut instruction_data = bytes_of::(&cmd); + + let mut funding_setup = AccountSetup::new_funding(); + let funding_account = funding_setup.as_account_info(); + + let mut price_setup = AccountSetup::new::(&program_id); + let price_account = price_setup.as_account_info(); + PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); + + **price_account.try_borrow_mut_lamports().unwrap() = 100; + + let mut permissions_setup = AccountSetup::new_permission(&program_id); + let permissions_account = permissions_setup.as_account_info(); + + { + let mut permissions_account_data = + PermissionAccount::initialize(&permissions_account, PC_VERSION).unwrap(); + permissions_account_data.master_authority = *funding_account.key; + permissions_account_data.data_curation_authority = *funding_account.key; + permissions_account_data.security_authority = *funding_account.key; + } + + // Give the price account enough lamports to be rent exempt + **price_account.try_borrow_mut_lamports().unwrap() = + Rent::minimum_balance(&Rent::default(), PriceAccount::MINIMUM_SIZE); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + permissions_account.clone(), + ], + instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.num_, 1); + assert_eq!(price_data.header.size, PriceAccount::INITIAL_SIZE); + assert!(price_data.comp_[0].pub_ == publisher); + // Make sure that v2 aggregation is disabled + assert!(!price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); + } + + cmd.publisher = ENABLE_ACCUMULATOR_V2.into(); + instruction_data = bytes_of::(&cmd); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + permissions_account.clone(), + ], + instruction_data + ) + .is_ok()); + + // Make sure that v2 aggregation is enabled + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert!(price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); + } +} From c9aa1e3c6c9422565d1e005819172a69132c324e Mon Sep 17 00:00:00 2001 From: Reisen Date: Wed, 10 Jul 2024 12:29:36 +0000 Subject: [PATCH 08/49] test: test aggregation toggle --- program/rust/src/tests/test_aggregate_v2.rs | 187 ++++++++++++++------ 1 file changed, 132 insertions(+), 55 deletions(-) diff --git a/program/rust/src/tests/test_aggregate_v2.rs b/program/rust/src/tests/test_aggregate_v2.rs index 7e1997f3..cc4ba88b 100644 --- a/program/rust/src/tests/test_aggregate_v2.rs +++ b/program/rust/src/tests/test_aggregate_v2.rs @@ -6,96 +6,173 @@ use { PriceAccountFlags, PythAccount, }, - c_oracle_header::PC_VERSION, - deserialize::load_checked, + c_oracle_header::{ + PC_STATUS_TRADING, + PC_VERSION, + }, + deserialize::{ + load_checked, + load_mut, + }, instruction::{ AddPublisherArgs, OracleCommand, + UpdPriceArgs, }, processor::{ process_instruction, + DISABLE_ACCUMULATOR_V2, ENABLE_ACCUMULATOR_V2, }, - tests::test_utils::AccountSetup, + tests::test_utils::{ + update_clock_slot, + AccountSetup, + }, }, bytemuck::bytes_of, solana_program::{ pubkey::Pubkey, rent::Rent, }, + solana_sdk::account_info::AccountInfo, + std::mem::size_of, }; -#[test] -fn test_aggregate_v2() { - let program_id = Pubkey::new_unique(); - let publisher = Pubkey::new_unique(); - - let mut cmd = AddPublisherArgs { - header: OracleCommand::AddPublisher.into(), - publisher, - }; - let mut instruction_data = bytes_of::(&cmd); - - let mut funding_setup = AccountSetup::new_funding(); - let funding_account = funding_setup.as_account_info(); +struct Accounts { + program_id: Pubkey, + publisher_account: AccountSetup, + funding_account: AccountSetup, + price_account: AccountSetup, + permissions_account: AccountSetup, + clock_account: AccountSetup, +} - let mut price_setup = AccountSetup::new::(&program_id); - let price_account = price_setup.as_account_info(); - PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); +impl Accounts { + fn new() -> Self { + let program_id = Pubkey::new_unique(); + let publisher_account = AccountSetup::new_funding(); + let clock_account = AccountSetup::new_clock(); + let mut funding_account = AccountSetup::new_funding(); + let mut permissions_account = AccountSetup::new_permission(&program_id); + let mut price_account = AccountSetup::new::(&program_id); - **price_account.try_borrow_mut_lamports().unwrap() = 100; + PriceAccount::initialize(&price_account.as_account_info(), PC_VERSION).unwrap(); - let mut permissions_setup = AccountSetup::new_permission(&program_id); - let permissions_account = permissions_setup.as_account_info(); + { + let permissions_account_info = permissions_account.as_account_info(); + let mut permissions_account_data = + PermissionAccount::initialize(&permissions_account_info, PC_VERSION).unwrap(); + permissions_account_data.master_authority = *funding_account.as_account_info().key; + permissions_account_data.data_curation_authority = + *funding_account.as_account_info().key; + permissions_account_data.security_authority = *funding_account.as_account_info().key; + } - { - let mut permissions_account_data = - PermissionAccount::initialize(&permissions_account, PC_VERSION).unwrap(); - permissions_account_data.master_authority = *funding_account.key; - permissions_account_data.data_curation_authority = *funding_account.key; - permissions_account_data.security_authority = *funding_account.key; + Self { + program_id, + publisher_account, + funding_account, + price_account, + permissions_account, + clock_account, + } } +} - // Give the price account enough lamports to be rent exempt - **price_account.try_borrow_mut_lamports().unwrap() = - Rent::minimum_balance(&Rent::default(), PriceAccount::MINIMUM_SIZE); +fn add_publisher(accounts: &mut Accounts, publisher: Option) { + let args = AddPublisherArgs { + header: OracleCommand::AddPublisher.into(), + publisher: publisher.unwrap_or(*accounts.publisher_account.as_account_info().key), + }; assert!(process_instruction( - &program_id, + &accounts.program_id, &[ - funding_account.clone(), - price_account.clone(), - permissions_account.clone(), + accounts.funding_account.as_account_info(), + accounts.price_account.as_account_info(), + accounts.permissions_account.as_account_info(), ], - instruction_data + bytes_of::(&args) ) .is_ok()); +} - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert_eq!(price_data.num_, 1); - assert_eq!(price_data.header.size, PriceAccount::INITIAL_SIZE); - assert!(price_data.comp_[0].pub_ == publisher); - // Make sure that v2 aggregation is disabled - assert!(!price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); - } +fn update_price(accounts: &mut Accounts, price: i64, conf: u64, slot: u64) { + let instruction_data = &mut [0u8; size_of::()]; + let mut cmd = load_mut::(instruction_data).unwrap(); + cmd.header = OracleCommand::UpdPrice.into(); + cmd.status = PC_STATUS_TRADING; + cmd.price = price; + cmd.confidence = conf; + cmd.publishing_slot = slot; + cmd.unused_ = 0; - cmd.publisher = ENABLE_ACCUMULATOR_V2.into(); - instruction_data = bytes_of::(&cmd); - assert!(process_instruction( - &program_id, + let mut clock = accounts.clock_account.as_account_info(); + clock.is_signer = false; + clock.is_writable = false; + + process_instruction( + &accounts.program_id, &[ - funding_account.clone(), - price_account.clone(), - permissions_account.clone(), + accounts.publisher_account.as_account_info(), + accounts.price_account.as_account_info(), + clock, ], - instruction_data + instruction_data, ) - .is_ok()); + .unwrap(); +} + +#[test] +fn test_aggregate_v2_toggle() { + let accounts = &mut Accounts::new(); + + // Add an initial Publisher to test with. + add_publisher(accounts, None); + + // Update the price, no aggregation will happen on the first slot. + { + update_clock_slot(&mut accounts.clock_account.as_account_info(), 1); + update_price(accounts, 42, 2, 1); + let info = accounts.price_account.as_account_info(); + let price_data = load_checked::(&info, PC_VERSION).unwrap(); + assert_eq!(price_data.last_slot_, 0); + assert!(!price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); + } + + // Update again, component is now TRADING so aggregation should trigger. + { + update_clock_slot(&mut accounts.clock_account.as_account_info(), 2); + update_price(accounts, 42, 2, 2); + let info = accounts.price_account.as_account_info(); + let price_data = load_checked::(&info, PC_VERSION).unwrap(); + assert_eq!(price_data.last_slot_, 2); + assert!(!price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); + } - // Make sure that v2 aggregation is enabled + // Enable v2 Aggregation + add_publisher(accounts, Some(ENABLE_ACCUMULATOR_V2.into())); + + // Update again, with accumulator bit set, aggregation should not have + // happened, as its now the validators job. { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + update_clock_slot(&mut accounts.clock_account.as_account_info(), 3); + update_price(accounts, 42, 2, 3); + let info = accounts.price_account.as_account_info(); + let price_data = load_checked::(&info, PC_VERSION).unwrap(); + assert_eq!(price_data.last_slot_, 2); assert!(price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); } + + add_publisher(accounts, Some(DISABLE_ACCUMULATOR_V2.into())); + + // Confirm disabling v2 Aggregation re-enables the aggregation flow. + { + update_clock_slot(&mut accounts.clock_account.as_account_info(), 4); + update_price(accounts, 42, 2, 4); + let info = accounts.price_account.as_account_info(); + let price_data = load_checked::(&info, PC_VERSION).unwrap(); + assert_eq!(price_data.last_slot_, 4); + assert!(!price_data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2)); + } } From f0ac2678d738d659ea4ef94f1db81e7d7e439183 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Wed, 10 Jul 2024 17:22:02 +0100 Subject: [PATCH 09/49] fix: don't aggregate in validator if already aggregated in this slot --- program/rust/src/tests/test_aggregate_v2.rs | 6 +----- program/rust/src/validator.rs | 8 +++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/program/rust/src/tests/test_aggregate_v2.rs b/program/rust/src/tests/test_aggregate_v2.rs index cc4ba88b..1a8c8e11 100644 --- a/program/rust/src/tests/test_aggregate_v2.rs +++ b/program/rust/src/tests/test_aggregate_v2.rs @@ -30,11 +30,7 @@ use { }, }, bytemuck::bytes_of, - solana_program::{ - pubkey::Pubkey, - rent::Rent, - }, - solana_sdk::account_info::AccountInfo, + solana_program::pubkey::Pubkey, std::mem::size_of, }; diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 7c138525..e9bfb602 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -58,9 +58,6 @@ fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) // NOTE: c_upd_aggregate must use a raw pointer to price data. We already // have the exclusive mut reference so we can simply cast before calling // the function. - // - // TODO: Verify the bytemuck::from_mut_bytes function is as safe as if - // we called `try_borrow_mut_data()`` let updated = unsafe { c_upd_aggregate( price_account as *mut PriceAccount as *mut u8, @@ -101,6 +98,8 @@ pub enum AggregationError { NotPriceFeedAccount, #[error("V1AggregationMode")] V1AggregationMode, + #[error("AlreadyAggregated")] + AlreadyAggregated, } pub struct AggregationOutcome { @@ -115,6 +114,9 @@ pub fn aggregate_price( price_account_data: &mut [u8], ) -> Result { let price_account = validate_price_account(price_account_data)?; + if price_account.agg_.pub_slot_ == slot { + return Err(AggregationError::AlreadyAggregated); + } let commit = update_aggregate(slot, timestamp, price_account); let messages = [ price_account From 17a1cf6f2ff9a98a73bf85ff32eb2efcf6e92e25 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Thu, 11 Jul 2024 14:12:17 +0100 Subject: [PATCH 10/49] chore: reexport solana_program and add derive debug --- program/rust/src/lib.rs | 3 +++ program/rust/src/validator.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index ece6eb01..beb48639 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -13,6 +13,9 @@ mod utils; #[cfg(feature = "library")] pub mod validator; +#[cfg(feature = "library")] +pub use solana_program; + #[cfg(test)] mod tests; diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index e9bfb602..04a5ef73 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -102,6 +102,7 @@ pub enum AggregationError { AlreadyAggregated, } +#[derive(Debug)] pub struct AggregationOutcome { pub messages: [Vec; 2], pub commit: bool, From ce71807a1c64f21c866779fda43eca0417205a8c Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Thu, 11 Jul 2024 16:26:27 +0100 Subject: [PATCH 11/49] chore: add comments --- program/rust/src/processor/add_publisher.rs | 4 ++++ program/rust/src/validator.rs | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/program/rust/src/processor/add_publisher.rs b/program/rust/src/processor/add_publisher.rs index f5768447..eb68b667 100644 --- a/program/rust/src/processor/add_publisher.rs +++ b/program/rust/src/processor/add_publisher.rs @@ -79,6 +79,10 @@ pub fn add_publisher( sort_price_comps(&mut price_data.comp_, num_comps)?; return Ok(()); } else if cmd_args.publisher == Pubkey::from(ENABLE_ACCUMULATOR_V2) { + // Hack: we use add_publisher instruction to configure the `ACCUMULATOR_V2` flag. Using a new + // instruction would be cleaner but it would require more work in the tooling. + // These special cases can be removed along with the v1 aggregation code once the transition + // is complete. price_data.flags.insert(PriceAccountFlags::ACCUMULATOR_V2); return Ok(()); } else if cmd_args.publisher == Pubkey::from(DISABLE_ACCUMULATOR_V2) { diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 04a5ef73..44e1e018 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -104,10 +104,21 @@ pub enum AggregationError { #[derive(Debug)] pub struct AggregationOutcome { + /// Messages that should be included in the merkle tree + /// (a price feed message and a TWAP message). pub messages: [Vec; 2], + /// A flag indicating that the changes to `price_account_data` + /// should be saved to the price account. pub commit: bool, } +/// Attempts to read a price account and create a new price aggregate if v2 +/// aggregation is enabled on this price account. +/// `price_account_data` will be modified in case of successful aggregation. +/// However, the changes to `price_account_data` should not be stored in the +/// account if `commit = false` in the returned value or if an error is returned. +/// Note that the `messages` will be returned whenever possible, even if +/// aggregation fails for some reason. pub fn aggregate_price( slot: u64, timestamp: i64, @@ -116,6 +127,8 @@ pub fn aggregate_price( ) -> Result { let price_account = validate_price_account(price_account_data)?; if price_account.agg_.pub_slot_ == slot { + // Avoid v2 aggregation if v1 aggregation has happened in the same slot + // (this should normally happen only in the slot that contains the v1->v2 transition). return Err(AggregationError::AlreadyAggregated); } let commit = update_aggregate(slot, timestamp, price_account); From 9ee8058c4167930cff07708d1ad4c80a7a5963f2 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 10:16:02 +0100 Subject: [PATCH 12/49] fix: don't revert if c_upd_aggregate returns false --- program/rust/src/validator.rs | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 44e1e018..eb983883 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -66,27 +66,24 @@ fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) ) }; - // If the aggregate was successfully updated, calculate the difference - // and update TWAP. - if !updated { - return false; - } - - let agg_diff = (slot as i64) - price_account.prev_slot_ as i64; + // If the aggregate was successfully updated, calculate the difference and update TWAP. + if updated { + let agg_diff = (slot as i64) - price_account.prev_slot_ as i64; - // See comment on unsafe `c_upd_aggregate` call above for details. - unsafe { - c_upd_twap(price_account as *mut PriceAccount as *mut u8, agg_diff); - } + // See comment on unsafe `c_upd_aggregate` call above for details. + unsafe { + c_upd_twap(price_account as *mut PriceAccount as *mut u8, agg_diff); + } - // We want to send a message every time the aggregate price updates. However, during the migration, - // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag - // ensures that after every aggregate update, the next publisher who provides the accumulator accounts - // will send the message. - price_account.message_sent_ = 0; - if price_account.update_price_cumulative().is_err() { - // Revert in case of a bad trading status. - return false; + // We want to send a message every time the aggregate price updates. However, during the migration, + // not every publisher will necessarily provide the accumulator accounts. The message_sent_ flag + // ensures that after every aggregate update, the next publisher who provides the accumulator accounts + // will send the message. + price_account.message_sent_ = 0; + if price_account.update_price_cumulative().is_err() { + // Revert in case of a bad trading status. + return false; + } } true From 9f035448583f91158413b417ef1b8a84ce0577c6 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 12:06:21 +0100 Subject: [PATCH 13/49] fix: make update_price_cumulative infallible; don't generate v2 messages until v1 buffer is cleared --- program/rust/src/accounts/price.rs | 17 ++++-------- program/rust/src/processor/upd_price.rs | 2 +- program/rust/src/tests/test_twap.rs | 9 ++---- program/rust/src/validator.rs | 37 ++++++++++++------------- 4 files changed, 27 insertions(+), 38 deletions(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index ecd154fa..a7e5dce8 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -25,13 +25,10 @@ mod price_pythnet { use { super::*, - crate::{ - c_oracle_header::{ - PC_MAX_SEND_LATENCY, - PC_NUM_COMP_PYTHNET, - PC_STATUS_TRADING, - }, - error::OracleError, + crate::c_oracle_header::{ + PC_MAX_SEND_LATENCY, + PC_NUM_COMP_PYTHNET, + PC_STATUS_TRADING, }, bitflags::bitflags, }; @@ -125,7 +122,8 @@ mod price_pythnet { } } /// This function gets triggered when there's a succesful aggregation and updates the cumulative sums - pub fn update_price_cumulative(&mut self) -> Result<(), OracleError> { + pub fn update_price_cumulative(&mut self) { + debug_assert!(self.agg_.status_ == PC_STATUS_TRADING); if self.agg_.status_ == PC_STATUS_TRADING { self.price_cumulative.update( self.agg_.price_, @@ -133,9 +131,6 @@ mod price_pythnet { self.agg_.pub_slot_.saturating_sub(self.prev_slot_), self.max_latency_, ); // pub_slot should always be >= prev_slot, but we protect ourselves against underflow just in case - Ok(()) - } else { - Err(OracleError::NeedsSuccesfulAggregation) } } diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index 37f673fd..da42ce97 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -196,7 +196,7 @@ pub fn upd_price( // ensures that after every aggregate update, the next publisher who provides the accumulator accounts // will send the message. price_data.message_sent_ = 0; - price_data.update_price_cumulative()?; + price_data.update_price_cumulative(); } } } diff --git a/program/rust/src/tests/test_twap.rs b/program/rust/src/tests/test_twap.rs index ff3b7379..db554234 100644 --- a/program/rust/src/tests/test_twap.rs +++ b/program/rust/src/tests/test_twap.rs @@ -12,7 +12,6 @@ use { PC_STATUS_UNKNOWN, }, deserialize::load_account_as_mut, - error::OracleError, }, quickcheck::Arbitrary, quickcheck_macros::quickcheck, @@ -262,7 +261,7 @@ fn test_twap_with_price_account() { unused: 0, }; price_data.prev_slot_ = 3; - price_data.update_price_cumulative().unwrap(); + price_data.update_price_cumulative(); assert_eq!(price_data.price_cumulative.price, 1 - 2 * 10); assert_eq!(price_data.price_cumulative.conf, 2 + 2 * 5); @@ -291,10 +290,6 @@ fn test_twap_with_price_account() { pub_slot_: 6, }; price_data.prev_slot_ = 5; - assert_eq!( - price_data.update_price_cumulative(), - Err(OracleError::NeedsSuccesfulAggregation) - ); assert_eq!(price_data.price_cumulative.price, 1 - 2 * 10); assert_eq!(price_data.price_cumulative.conf, 2 + 2 * 5); @@ -302,7 +297,7 @@ fn test_twap_with_price_account() { // Back to normal behavior price_data.agg_.status_ = PC_STATUS_TRADING; - price_data.update_price_cumulative().unwrap(); + price_data.update_price_cumulative(); assert_eq!(price_data.price_cumulative.price, 1 - 2 * 10 + 1); assert_eq!(price_data.price_cumulative.conf, 2 + 2 * 5 + 2); diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index eb983883..c9470011 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -49,12 +49,19 @@ fn validate_price_account( if !data.flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { return Err(AggregationError::V1AggregationMode); } + if !data + .flags + .contains(PriceAccountFlags::MESSAGE_BUFFER_CLEARED) + { + // We make sure that we don't generate v2 messages while v1 messages are still + // in the message buffer. + return Err(AggregationError::V1AggregationMode); + } Ok(data) } -// Returns true if the price account data has been modified and should be committed. -fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) -> bool { +fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) { // NOTE: c_upd_aggregate must use a raw pointer to price data. We already // have the exclusive mut reference so we can simply cast before calling // the function. @@ -80,13 +87,8 @@ fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) // ensures that after every aggregate update, the next publisher who provides the accumulator accounts // will send the message. price_account.message_sent_ = 0; - if price_account.update_price_cumulative().is_err() { - // Revert in case of a bad trading status. - return false; - } + price_account.update_price_cumulative(); } - - true } #[derive(Debug, thiserror::Error)] @@ -110,32 +112,29 @@ pub struct AggregationOutcome { } /// Attempts to read a price account and create a new price aggregate if v2 -/// aggregation is enabled on this price account. -/// `price_account_data` will be modified in case of successful aggregation. -/// However, the changes to `price_account_data` should not be stored in the -/// account if `commit = false` in the returned value or if an error is returned. -/// Note that the `messages` will be returned whenever possible, even if -/// aggregation fails for some reason. +/// aggregation is enabled on this price account. Modifies `price_account_data` accordingly. +/// Returns messages that should be included in the merkle tree, unless v1 aggregation +/// is still in use. +/// Note that the `messages` may be returned even if aggregation fails for some reason. pub fn aggregate_price( slot: u64, timestamp: i64, price_account_pubkey: &Pubkey, price_account_data: &mut [u8], -) -> Result { +) -> Result<[Vec; 2], AggregationError> { let price_account = validate_price_account(price_account_data)?; if price_account.agg_.pub_slot_ == slot { // Avoid v2 aggregation if v1 aggregation has happened in the same slot // (this should normally happen only in the slot that contains the v1->v2 transition). return Err(AggregationError::AlreadyAggregated); } - let commit = update_aggregate(slot, timestamp, price_account); - let messages = [ + update_aggregate(slot, timestamp, price_account); + Ok([ price_account .as_price_feed_message(price_account_pubkey) .to_bytes(), price_account .as_twap_message(price_account_pubkey) .to_bytes(), - ]; - Ok(AggregationOutcome { messages, commit }) + ]) } From a59effe7d77cfe88488fd3c4c34f3d5e33719f96 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 12:15:01 +0100 Subject: [PATCH 14/49] chore: remove publisher sorting hack --- program/rust/src/processor/add_publisher.rs | 8 +---- program/rust/src/tests/test_add_publisher.rs | 33 -------------------- 2 files changed, 1 insertion(+), 40 deletions(-) diff --git a/program/rust/src/processor/add_publisher.rs b/program/rust/src/processor/add_publisher.rs index eb68b667..13c29ea9 100644 --- a/program/rust/src/processor/add_publisher.rs +++ b/program/rust/src/processor/add_publisher.rs @@ -72,13 +72,7 @@ pub fn add_publisher( let mut price_data = load_checked::(price_account, cmd_args.header.version)?; - // Use the call with the default pubkey (000..) as a trigger to sort the publishers as a - // migration step from unsorted list to sorted list. - if cmd_args.publisher == Pubkey::default() { - let num_comps = try_convert::(price_data.num_)?; - sort_price_comps(&mut price_data.comp_, num_comps)?; - return Ok(()); - } else if cmd_args.publisher == Pubkey::from(ENABLE_ACCUMULATOR_V2) { + if cmd_args.publisher == Pubkey::from(ENABLE_ACCUMULATOR_V2) { // Hack: we use add_publisher instruction to configure the `ACCUMULATOR_V2` flag. Using a new // instruction would be cleaner but it would require more work in the tooling. // These special cases can be removed along with the v1 aggregation code once the transition diff --git a/program/rust/src/tests/test_add_publisher.rs b/program/rust/src/tests/test_add_publisher.rs index 41dcfe67..57f2a346 100644 --- a/program/rust/src/tests/test_add_publisher.rs +++ b/program/rust/src/tests/test_add_publisher.rs @@ -157,37 +157,4 @@ fn test_add_publisher() { assert!(price_data.comp_[i as usize].pub_ > price_data.comp_[(i - 1) as usize].pub_); } } - - // Test sorting by reordering the publishers to be in reverse order - // and then adding the default (000...) publisher to trigger the sorting. - { - let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - price_data - .comp_ - .get_mut(..PC_NUM_COMP as usize) - .unwrap() - .reverse(); - } - - cmd.publisher = Pubkey::default(); - instruction_data = bytes_of::(&cmd); - assert!(process_instruction( - &program_id, - &[ - funding_account.clone(), - price_account.clone(), - permissions_account.clone(), - ], - instruction_data - ) - .is_ok()); - - // Make sure that publishers get sorted after adding the default publisher - { - let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); - assert!(price_data.num_ == PC_NUM_COMP); - for i in 1..PC_NUM_COMP { - assert!(price_data.comp_[i as usize].pub_ > price_data.comp_[(i - 1) as usize].pub_); - } - } } From 95232099c900c2adcac4e0c2b48e1756f48a3128 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 13:16:29 +0100 Subject: [PATCH 15/49] chore: remove AggregationOutcome --- program/rust/src/validator.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index c9470011..c882f5bb 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -101,16 +101,6 @@ pub enum AggregationError { AlreadyAggregated, } -#[derive(Debug)] -pub struct AggregationOutcome { - /// Messages that should be included in the merkle tree - /// (a price feed message and a TWAP message). - pub messages: [Vec; 2], - /// A flag indicating that the changes to `price_account_data` - /// should be saved to the price account. - pub commit: bool, -} - /// Attempts to read a price account and create a new price aggregate if v2 /// aggregation is enabled on this price account. Modifies `price_account_data` accordingly. /// Returns messages that should be included in the merkle tree, unless v1 aggregation From 064a17312c244c64146a76448275e9bae2217e8a Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 12 Jul 2024 13:37:33 +0100 Subject: [PATCH 16/49] fix: remove debug_assert --- program/rust/src/accounts/price.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index a7e5dce8..6958970f 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -123,7 +123,6 @@ mod price_pythnet { } /// This function gets triggered when there's a succesful aggregation and updates the cumulative sums pub fn update_price_cumulative(&mut self) { - debug_assert!(self.agg_.status_ == PC_STATUS_TRADING); if self.agg_.status_ == PC_STATUS_TRADING { self.price_cumulative.update( self.agg_.price_, From 70c9559198f1d31b3e5a026e3287400c13e88a84 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Tue, 16 Jul 2024 14:39:36 +0100 Subject: [PATCH 17/49] fix: allow clearing message buffers regardless of message_sent_ flag --- program/rust/src/processor/upd_price.rs | 128 ++++++++++++------------ 1 file changed, 63 insertions(+), 65 deletions(-) diff --git a/program/rust/src/processor/upd_price.rs b/program/rust/src/processor/upd_price.rs index da42ce97..525dd246 100644 --- a/program/rust/src/processor/upd_price.rs +++ b/program/rust/src/processor/upd_price.rs @@ -210,76 +210,74 @@ pub fn upd_price( !flags.contains(PriceAccountFlags::MESSAGE_BUFFER_CLEARED) } else { // V1 - true + price_data.message_sent_ == 0 }; if need_message_buffer_update { if let Some(accumulator_accounts) = maybe_accumulator_accounts { - if price_data.message_sent_ == 0 { - // Check that the oracle PDA is correctly configured for the program we are calling. - let oracle_auth_seeds: &[&[u8]] = &[ - UPD_PRICE_WRITE_SEED.as_bytes(), - &accumulator_accounts.program_id.key.to_bytes(), - ]; - let (expected_oracle_auth_pda, bump) = - Pubkey::find_program_address(oracle_auth_seeds, program_id); - pyth_assert( - expected_oracle_auth_pda == *accumulator_accounts.oracle_auth_pda.key, - OracleError::InvalidPda.into(), - )?; - - let account_metas = vec![ - AccountMeta { - pubkey: *accumulator_accounts.whitelist.key, - is_signer: false, - is_writable: false, - }, - AccountMeta { - pubkey: *accumulator_accounts.oracle_auth_pda.key, - is_signer: true, - is_writable: false, - }, - AccountMeta { - pubkey: *accumulator_accounts.message_buffer_data.key, - is_signer: false, - is_writable: true, - }, - ]; - - let message = if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { - vec![] - } else { - vec![ - price_data - .as_price_feed_message(price_account.key) - .to_bytes(), - price_data.as_twap_message(price_account.key).to_bytes(), - ] - }; - - // Append a TWAP message if available - - // anchor discriminator for "global:put_all" - let discriminator: [u8; 8] = [212, 225, 193, 91, 151, 238, 20, 93]; - let create_inputs_ix = Instruction::new_with_borsh( - *accumulator_accounts.program_id.key, - &(discriminator, price_account.key.to_bytes(), message), - account_metas, - ); - - let auth_seeds_with_bump: &[&[u8]] = &[ - UPD_PRICE_WRITE_SEED.as_bytes(), - &accumulator_accounts.program_id.key.to_bytes(), - &[bump], - ]; - - invoke_signed(&create_inputs_ix, accounts, &[auth_seeds_with_bump])?; - price_data.message_sent_ = 1; - if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + // Check that the oracle PDA is correctly configured for the program we are calling. + let oracle_auth_seeds: &[&[u8]] = &[ + UPD_PRICE_WRITE_SEED.as_bytes(), + &accumulator_accounts.program_id.key.to_bytes(), + ]; + let (expected_oracle_auth_pda, bump) = + Pubkey::find_program_address(oracle_auth_seeds, program_id); + pyth_assert( + expected_oracle_auth_pda == *accumulator_accounts.oracle_auth_pda.key, + OracleError::InvalidPda.into(), + )?; + + let account_metas = vec![ + AccountMeta { + pubkey: *accumulator_accounts.whitelist.key, + is_signer: false, + is_writable: false, + }, + AccountMeta { + pubkey: *accumulator_accounts.oracle_auth_pda.key, + is_signer: true, + is_writable: false, + }, + AccountMeta { + pubkey: *accumulator_accounts.message_buffer_data.key, + is_signer: false, + is_writable: true, + }, + ]; + + let message = if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + vec![] + } else { + vec![ price_data - .flags - .insert(PriceAccountFlags::MESSAGE_BUFFER_CLEARED); - } + .as_price_feed_message(price_account.key) + .to_bytes(), + price_data.as_twap_message(price_account.key).to_bytes(), + ] + }; + + // Append a TWAP message if available + + // anchor discriminator for "global:put_all" + let discriminator: [u8; 8] = [212, 225, 193, 91, 151, 238, 20, 93]; + let create_inputs_ix = Instruction::new_with_borsh( + *accumulator_accounts.program_id.key, + &(discriminator, price_account.key.to_bytes(), message), + account_metas, + ); + + let auth_seeds_with_bump: &[&[u8]] = &[ + UPD_PRICE_WRITE_SEED.as_bytes(), + &accumulator_accounts.program_id.key.to_bytes(), + &[bump], + ]; + + invoke_signed(&create_inputs_ix, accounts, &[auth_seeds_with_bump])?; + price_data.message_sent_ = 1; + if flags.contains(PriceAccountFlags::ACCUMULATOR_V2) { + price_data + .flags + .insert(PriceAccountFlags::MESSAGE_BUFFER_CLEARED); } } } From 5d1b09dab38b572170f3d6405cd87736ed62d543 Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Tue, 16 Jul 2024 14:40:13 +0100 Subject: [PATCH 18/49] test: add test_upd_price_with_validator --- program/rust/src/lib.rs | 2 +- program/rust/src/tests/mod.rs | 1 + .../tests/test_upd_price_with_validator.rs | 422 ++++++++++++++++++ program/rust/src/validator.rs | 2 +- 4 files changed, 425 insertions(+), 2 deletions(-) create mode 100644 program/rust/src/tests/test_upd_price_with_validator.rs diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index beb48639..e71b7c65 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -10,7 +10,7 @@ mod instruction; mod processor; mod utils; -#[cfg(feature = "library")] +#[cfg(any(test, feature = "library"))] pub mod validator; #[cfg(feature = "library")] diff --git a/program/rust/src/tests/mod.rs b/program/rust/src/tests/mod.rs index ccd4f585..33178ccd 100644 --- a/program/rust/src/tests/mod.rs +++ b/program/rust/src/tests/mod.rs @@ -24,6 +24,7 @@ mod test_upd_aggregate; mod test_upd_permissions; mod test_upd_price; mod test_upd_price_no_fail_on_error; +mod test_upd_price_with_validator; mod test_upd_product; mod test_utils; diff --git a/program/rust/src/tests/test_upd_price_with_validator.rs b/program/rust/src/tests/test_upd_price_with_validator.rs new file mode 100644 index 00000000..f25db775 --- /dev/null +++ b/program/rust/src/tests/test_upd_price_with_validator.rs @@ -0,0 +1,422 @@ +use { + crate::{ + accounts::{ + PriceAccount, + PriceAccountFlags, + PythAccount, + }, + c_oracle_header::{ + PC_STATUS_IGNORED, + PC_STATUS_TRADING, + PC_STATUS_UNKNOWN, + PC_VERSION, + }, + deserialize::{ + load_checked, + load_mut, + }, + instruction::{ + OracleCommand, + UpdPriceArgs, + }, + processor::process_instruction, + tests::test_utils::{ + update_clock_slot, + AccountSetup, + }, + validator::{ + self, + }, + }, + solana_program::{ + program_error::ProgramError, + pubkey::Pubkey, + }, + std::mem::size_of, +}; + +#[test] +fn test_upd_price_with_validator() { + // Similar to `test_upd_price` but uses validator aggregation instead of + // in-program aggregation. + let mut instruction_data = [0u8; size_of::()]; + populate_instruction(&mut instruction_data, 42, 2, 1); + + let program_id = Pubkey::new_unique(); + + let mut funding_setup = AccountSetup::new_funding(); + let funding_account = funding_setup.as_account_info(); + + let mut price_setup = AccountSetup::new::(&program_id); + let mut price_account = price_setup.as_account_info(); + price_account.is_signer = false; + PriceAccount::initialize(&price_account, PC_VERSION).unwrap(); + + { + let mut price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + price_data.num_ = 1; + price_data.comp_[0].pub_ = *funding_account.key; + price_data.flags.insert(PriceAccountFlags::ACCUMULATOR_V2); + price_data + .flags + .insert(PriceAccountFlags::MESSAGE_BUFFER_CLEARED); + } + + let mut clock_setup = AccountSetup::new_clock(); + let mut clock_account = clock_setup.as_account_info(); + clock_account.is_signer = false; + clock_account.is_writable = false; + + update_clock_slot(&mut clock_account, 1); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.agg_.pub_slot_, 0); + assert_eq!(price_data.agg_.price_, 0); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + + // a publisher's component pub_slot_ has to be strictly increasing -- get rejected + populate_instruction(&mut instruction_data, 43, 2, 1); + + assert_eq!( + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ), + Err(ProgramError::InvalidArgument) + ); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 42); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 1); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 0); + assert_eq!(price_data.agg_.pub_slot_, 0); + assert_eq!(price_data.agg_.price_, 0); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + + // We aggregate the price at the end of each slot now. + validator::aggregate_price( + 1, + 101, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 2); + + validator::aggregate_price( + 2, + 102, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 3); + // add next price in new slot triggering snapshot and aggregate calc + populate_instruction(&mut instruction_data, 81, 2, 2); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 2); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + // Slot values are 1 less than in `test_upd_price` because + // we aggregate at the end of the block now. + assert_eq!(price_data.valid_slot_, 1); + assert_eq!(price_data.agg_.pub_slot_, 2); + assert_eq!(price_data.agg_.price_, 42); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + + // next price doesn't change but slot does + populate_instruction(&mut instruction_data, 81, 2, 3); + validator::aggregate_price( + 3, + 103, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 4); + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 3); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 2); + assert_eq!(price_data.agg_.pub_slot_, 3); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + + // next price doesn't change and neither does aggregate but slot does + populate_instruction(&mut instruction_data, 81, 2, 4); + validator::aggregate_price( + 4, + 104, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 5); + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 3); + assert_eq!(price_data.agg_.pub_slot_, 4); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + + // try to publish back-in-time + populate_instruction(&mut instruction_data, 81, 2, 1); + update_clock_slot(&mut clock_account, 5); + + assert_eq!( + process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ), + Err(ProgramError::InvalidArgument) + ); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 81); + assert_eq!(price_data.comp_[0].latest_.conf_, 2); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 4); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 3); + assert_eq!(price_data.agg_.pub_slot_, 4); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + + populate_instruction(&mut instruction_data, 50, 20, 5); + validator::aggregate_price( + 5, + 105, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 6); + + // Publishing a wide CI results in a status of unknown. + + // check that someone doesn't accidentally break the test. + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + } + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 50); + assert_eq!(price_data.comp_[0].latest_.conf_, 20); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 5); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); + assert_eq!(price_data.valid_slot_, 4); + assert_eq!(price_data.agg_.pub_slot_, 5); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } + + // Crank one more time and aggregate should be unknown + populate_instruction(&mut instruction_data, 50, 20, 6); + + validator::aggregate_price( + 6, + 106, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 7); + + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, 50); + assert_eq!(price_data.comp_[0].latest_.conf_, 20); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 6); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_IGNORED); + assert_eq!(price_data.valid_slot_, 5); + assert_eq!(price_data.agg_.pub_slot_, 6); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + + // Negative prices are accepted + populate_instruction(&mut instruction_data, -100, 1, 7); + validator::aggregate_price( + 7, + 107, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 8); + + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, -100); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 7); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 6); + assert_eq!(price_data.agg_.pub_slot_, 7); + assert_eq!(price_data.agg_.price_, 81); + assert_eq!(price_data.agg_.status_, PC_STATUS_UNKNOWN); + } + + // Crank again for aggregate + populate_instruction(&mut instruction_data, -100, 1, 8); + validator::aggregate_price( + 8, + 108, + price_account.key, + &mut *price_account.data.borrow_mut(), + ) + .unwrap(); + update_clock_slot(&mut clock_account, 9); + + + assert!(process_instruction( + &program_id, + &[ + funding_account.clone(), + price_account.clone(), + clock_account.clone() + ], + &instruction_data + ) + .is_ok()); + + { + let price_data = load_checked::(&price_account, PC_VERSION).unwrap(); + assert_eq!(price_data.comp_[0].latest_.price_, -100); + assert_eq!(price_data.comp_[0].latest_.conf_, 1); + assert_eq!(price_data.comp_[0].latest_.pub_slot_, 8); + assert_eq!(price_data.comp_[0].latest_.status_, PC_STATUS_TRADING); + assert_eq!(price_data.valid_slot_, 7); + assert_eq!(price_data.agg_.pub_slot_, 8); + assert_eq!(price_data.agg_.price_, -100); + assert_eq!(price_data.agg_.status_, PC_STATUS_TRADING); + } +} + +// Create an upd_price instruction with the provided parameters +fn populate_instruction(instruction_data: &mut [u8], price: i64, conf: u64, pub_slot: u64) { + let mut cmd = load_mut::(instruction_data).unwrap(); + cmd.header = OracleCommand::UpdPrice.into(); + cmd.status = PC_STATUS_TRADING; + cmd.price = price; + cmd.confidence = conf; + cmd.publishing_slot = pub_slot; + cmd.unused_ = 0; +} diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index c882f5bb..48761b7b 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -91,7 +91,7 @@ fn update_aggregate(slot: u64, timestamp: i64, price_account: &mut PriceAccount) } } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, PartialEq, thiserror::Error)] pub enum AggregationError { #[error("NotPriceFeedAccount")] NotPriceFeedAccount, From 3c9feb8fed4b57648c6239ed512f8697f8607154 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 17:20:43 +0100 Subject: [PATCH 19/49] checkpoint --- Cargo.lock | 68 +++++++++++++++---- program/rust/Cargo.toml | 4 +- program/rust/src/accounts/price.rs | 8 +-- program/rust/src/lib.rs | 2 +- .../tests/test_upd_price_with_validator.rs | 4 +- program/rust/src/validator.rs | 40 ++++++++++- 6 files changed, 102 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b598efc..91ce9234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -358,18 +358,41 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" dependencies = [ - "borsh-derive", + "borsh-derive 0.9.3", "hashbrown 0.11.2", ] +[[package]] +name = "borsh" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" +dependencies = [ + "borsh-derive 0.10.3", + "hashbrown 0.12.3", +] + [[package]] name = "borsh-derive" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" dependencies = [ - "borsh-derive-internal", - "borsh-schema-derive-internal", + "borsh-derive-internal 0.9.3", + "borsh-schema-derive-internal 0.9.3", + "proc-macro-crate 0.1.5", + "proc-macro2 1.0.56", + "syn 1.0.109", +] + +[[package]] +name = "borsh-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" +dependencies = [ + "borsh-derive-internal 0.10.3", + "borsh-schema-derive-internal 0.10.3", "proc-macro-crate 0.1.5", "proc-macro2 1.0.56", "syn 1.0.109", @@ -386,6 +409,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb438156919598d2c7bad7e1c0adf3d26ed3840dbc010db1a882a65583ca2fb" +dependencies = [ + "proc-macro2 1.0.56", + "quote 1.0.26", + "syn 1.0.109", +] + [[package]] name = "borsh-schema-derive-internal" version = "0.9.3" @@ -397,6 +431,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "borsh-schema-derive-internal" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634205cc43f74a1b9046ef87c4540ebda95696ec0f315024860cad7c5b0f5ccd" +dependencies = [ + "proc-macro2 1.0.56", + "quote 1.0.26", + "syn 1.0.109", +] + [[package]] name = "brotli" version = "3.3.4" @@ -2564,11 +2609,10 @@ dependencies = [ [[package]] name = "pythnet-sdk" -version = "1.13.6" -source = "git+https://github.com/pyth-network/pyth-crosschain?rev=60144002053a93f424be70decd8a8ccb8d618d81#60144002053a93f424be70decd8a8ccb8d618d81" +version = "2.1.0" dependencies = [ "bincode", - "borsh", + "borsh 0.10.3", "bytemuck", "byteorder", "fast-math", @@ -3351,7 +3395,7 @@ version = "1.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96c4d210c247714a742fa27629c30c63eefccb3fa7565168649dd58c275cb0cc" dependencies = [ - "borsh", + "borsh 0.9.3", "futures", "solana-banks-interface", "solana-program", @@ -3703,8 +3747,8 @@ dependencies = [ "bincode", "bitflags 1.3.2", "blake3", - "borsh", - "borsh-derive", + "borsh 0.9.3", + "borsh-derive 0.9.3", "bs58", "bv", "bytemuck", @@ -3894,7 +3938,7 @@ dependencies = [ "base64 0.13.1", "bincode", "bitflags 1.3.2", - "borsh", + "borsh 0.9.3", "bs58", "bytemuck", "byteorder", @@ -4024,7 +4068,7 @@ dependencies = [ "Inflector", "base64 0.13.1", "bincode", - "borsh", + "borsh 0.9.3", "bs58", "lazy_static", "log", @@ -4168,7 +4212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbc000f0fdf1f12f99d77d398137c1751345b18c88258ce0f99b7872cf6c9bd6" dependencies = [ "assert_matches", - "borsh", + "borsh 0.9.3", "num-derive", "num-traits", "solana-program", diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 8ba4b929..6ba75495 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -17,7 +17,7 @@ num-traits = "0.2" byteorder = "1.4.3" serde = { version = "1.0", features = ["derive"], optional = true } strum = { version = "0.24.1", features = ["derive"], optional = true } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60144002053a93f424be70decd8a8ccb8d618d81"} +pythnet-sdk = { path = "../../../pyth-crosschain/pythnet/pythnet_sdk"} solana-sdk = { version = "=1.14.17", optional = true } bitflags = { version = "2.6.0", features = ["bytemuck"] } @@ -31,7 +31,7 @@ rand = "0.8.5" quickcheck_macros = "1" bincode = "1.3.3" serde = { version = "1.0", features = ["derive"] } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", rev="60144002053a93f424be70decd8a8ccb8d618d81", features = ["quickcheck"]} +pythnet-sdk = { path = "../../../pyth-crosschain/pythnet/pythnet_sdk", features = ["quickcheck"]} serde_json = "1.0" test-generator = "0.3.1" csv = "1.1" diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 6958970f..dbec1510 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -111,7 +111,7 @@ mod price_pythnet { }; PriceFeedMessage { - id: key.to_bytes(), + feed_id: key.to_bytes(), price, conf, exponent: self.exponent, @@ -141,7 +141,7 @@ mod price_pythnet { }; TwapMessage { - id: key.to_bytes(), + feed_id: key.to_bytes(), cumulative_price: self.price_cumulative.price, cumulative_conf: self.price_cumulative.conf, num_down_slots: self.price_cumulative.num_down_slots, @@ -270,7 +270,7 @@ impl PythOracleSerialize for PriceFeedMessage { bytes[i..i + 1].clone_from_slice(&[DISCRIMINATOR]); i += 1; - bytes[i..i + 32].clone_from_slice(&self.id[..]); + bytes[i..i + 32].clone_from_slice(&self.feed_id[..]); i += 32; bytes[i..i + 8].clone_from_slice(&self.price.to_be_bytes()); @@ -313,7 +313,7 @@ impl PythOracleSerialize for TwapMessage { bytes[i..i + 1].clone_from_slice(&[DISCRIMINATOR]); i += 1; - bytes[i..i + 32].clone_from_slice(&self.id[..]); + bytes[i..i + 32].clone_from_slice(&self.feed_id[..]); i += 32; bytes[i..i + 16].clone_from_slice(&self.cumulative_price.to_be_bytes()); diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index e71b7c65..8b4f9a35 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -1,4 +1,4 @@ -#![deny(warnings)] +// #![deny(warnings)] // Allow non upper case globals from C #![allow(non_upper_case_globals)] diff --git a/program/rust/src/tests/test_upd_price_with_validator.rs b/program/rust/src/tests/test_upd_price_with_validator.rs index f25db775..235b4d16 100644 --- a/program/rust/src/tests/test_upd_price_with_validator.rs +++ b/program/rust/src/tests/test_upd_price_with_validator.rs @@ -24,9 +24,7 @@ use { update_clock_slot, AccountSetup, }, - validator::{ - self, - }, + validator::{self,}, }, solana_program::{ program_error::ProgramError, diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 48761b7b..272de835 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -15,8 +15,19 @@ use { }, utils::pyth_assert, }, - solana_sdk::pubkey::Pubkey, - std::mem::size_of, + pythnet_sdk::messages::{ + PublisherCap, + PublisherCapsMessage, + }, + solana_sdk::{ + account::ReadableAccount, + pubkey::Pubkey, + transaction_context::TransactionAccount, + }, + std::{ + collections::HashMap, + mem::size_of, + }, }; // Attempts to validate and access the contents of an account as a PriceAccount. @@ -128,3 +139,28 @@ pub fn aggregate_price( .to_bytes(), ]) } + +pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { + let mut publisher_caps: HashMap = HashMap::new(); + for (pubkey, account) in accounts { + let price_account_data = account.data(); + let price_account: &PriceAccount = bytemuck::from_bytes(&price_account_data); + let cap = 1_000_000 / (price_account.num_ as u64); + for i in 0..(price_account.num_ as usize) { + publisher_caps + .entry(price_account.comp_[i].pub_) + .and_modify(|e| *e += cap) + .or_insert(cap); + } + } + + let message = PublisherCapsMessage { + publish_time: timestamp, + caps: publisher_caps + .into_iter() + .map(|(publisher, cap)| PublisherCap { publisher, cap }) + .collect(), + }; + + return message.to_bytes(); +} From b13edd79f497e6c5cb1a49e528a8559a60ea189f Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 17:29:39 +0100 Subject: [PATCH 20/49] fix import --- Cargo.lock | 1 + program/rust/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91ce9234..1ed8d5c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2610,6 +2610,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#959a43cfbf5547d783344af641c27d1bbb240e38" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 6ba75495..bdd744b9 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -17,7 +17,7 @@ num-traits = "0.2" byteorder = "1.4.3" serde = { version = "1.0", features = ["derive"], optional = true } strum = { version = "0.24.1", features = ["derive"], optional = true } -pythnet-sdk = { path = "../../../pyth-crosschain/pythnet/pythnet_sdk"} +pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", branch="feat/publisher-caps"} solana-sdk = { version = "=1.14.17", optional = true } bitflags = { version = "2.6.0", features = ["bytemuck"] } @@ -31,7 +31,7 @@ rand = "0.8.5" quickcheck_macros = "1" bincode = "1.3.3" serde = { version = "1.0", features = ["derive"] } -pythnet-sdk = { path = "../../../pyth-crosschain/pythnet/pythnet_sdk", features = ["quickcheck"]} +pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", branch="feat/publisher-caps", features = ["quickcheck"]} serde_json = "1.0" test-generator = "0.3.1" csv = "1.1" From 7d0958fee5e83581fbac14f5e6419bb59257419c Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 17:46:02 +0100 Subject: [PATCH 21/49] go --- Cargo.lock | 4 ++-- program/rust/src/accounts/price.rs | 19 +++++++++++++++++++ program/rust/src/validator.rs | 7 +++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ed8d5c1..d81917e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive 0.10.3", - "hashbrown 0.12.3", + "hashbrown 0.11.2", ] [[package]] @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#959a43cfbf5547d783344af641c27d1bbb240e38" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#37f40e733a584d5a8af98161d786da4a3ac7176d" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index dbec1510..2596208f 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -13,6 +13,7 @@ use { }, pythnet_sdk::messages::{ PriceFeedMessage, + PublisherCapsMessage, TwapMessage, }, solana_program::pubkey::Pubkey, @@ -340,3 +341,21 @@ impl PythOracleSerialize for TwapMessage { bytes.to_vec() } } + +impl PythOracleSerialize for PublisherCapsMessage { + fn to_bytes(self) -> Vec { + const DISCRIMINATOR: u8 = 2; + let mut result: Vec = Vec::new(); + + result.extend_from_slice(&[DISCRIMINATOR]); + result.extend_from_slice(&self.publish_time.to_be_bytes()); + result.extend_from_slice(&self.caps.len().to_be_bytes()); + + for cap in self.caps { + result.extend_from_slice(&cap.publisher); + result.extend_from_slice(&cap.cap.to_be_bytes()); + } + + return result; + } +} diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 272de835..f58e587e 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -142,7 +142,7 @@ pub fn aggregate_price( pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { let mut publisher_caps: HashMap = HashMap::new(); - for (pubkey, account) in accounts { + for (_, account) in accounts { let price_account_data = account.data(); let price_account: &PriceAccount = bytemuck::from_bytes(&price_account_data); let cap = 1_000_000 / (price_account.num_ as u64); @@ -158,7 +158,10 @@ pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64 publish_time: timestamp, caps: publisher_caps .into_iter() - .map(|(publisher, cap)| PublisherCap { publisher, cap }) + .map(|(publisher, cap)| PublisherCap { + publisher: publisher.to_bytes(), + cap, + }) .collect(), }; From afe2eda86d42cde615c8a4430aacc54c214b527d Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 19:09:52 +0100 Subject: [PATCH 22/49] try this --- program/rust/src/validator.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index f58e587e..ced1e6a7 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -20,7 +20,10 @@ use { PublisherCapsMessage, }, solana_sdk::{ - account::ReadableAccount, + account::{ + AccountSharedData, + ReadableAccount, + }, pubkey::Pubkey, transaction_context::TransactionAccount, }, @@ -140,7 +143,7 @@ pub fn aggregate_price( ]) } -pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { +pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { let mut publisher_caps: HashMap = HashMap::new(); for (_, account) in accounts { let price_account_data = account.data(); From fdcad70525993c87887925386d1fcd28e6ab597e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 19:15:05 +0100 Subject: [PATCH 23/49] go --- program/rust/src/validator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index ced1e6a7..cc27bf92 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -145,7 +145,7 @@ pub fn aggregate_price( pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { let mut publisher_caps: HashMap = HashMap::new(); - for (_, account) in accounts { + for account in accounts { let price_account_data = account.data(); let price_account: &PriceAccount = bytemuck::from_bytes(&price_account_data); let cap = 1_000_000 / (price_account.num_ as u64); From 881a2c070e437a2ef99430bc5c5638c0d006eff4 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Tue, 16 Jul 2024 19:18:34 +0100 Subject: [PATCH 24/49] try again --- program/rust/src/validator.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index cc27bf92..84f4fef0 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -143,11 +143,10 @@ pub fn aggregate_price( ]) } -pub fn compute_publisher_caps(accounts: &Vec, timestamp: i64) -> Vec { +pub fn compute_publisher_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { let mut publisher_caps: HashMap = HashMap::new(); for account in accounts { - let price_account_data = account.data(); - let price_account: &PriceAccount = bytemuck::from_bytes(&price_account_data); + let price_account: &PriceAccount = bytemuck::from_bytes(&account); let cap = 1_000_000 / (price_account.num_ as u64); for i in 0..(price_account.num_ as usize) { publisher_caps From 4d135473187b4af4bfa05cbe7f3f1a08aa6fbe90 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 18:03:29 +0100 Subject: [PATCH 25/49] Checkpoint --- program/rust/src/validator.rs | 48 ++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 84f4fef0..332b9c75 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -8,6 +8,7 @@ use { PythOracleSerialize, }, c_oracle_header::PC_MAGIC, + deserialize::load_account_as, error::OracleError, processor::{ c_upd_aggregate, @@ -24,11 +25,12 @@ use { AccountSharedData, ReadableAccount, }, + program_error::ProgramError, pubkey::Pubkey, transaction_context::TransactionAccount, }, std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, mem::size_of, }, }; @@ -143,16 +145,44 @@ pub fn aggregate_price( ]) } +fn validate_price_account_readonly(price_account_info: &[u8]) -> Option<&PriceAccount> { + pyth_assert( + price_account_info.len() >= PriceAccount::MINIMUM_SIZE, + OracleError::AccountTooSmall.into(), + ) + .ok()?; + + { + let account_header = bytemuck::from_bytes::( + &price_account_info[0..size_of::()], + ); + + pyth_assert( + account_header.magic_number == PC_MAGIC + && account_header.account_type == PriceAccount::ACCOUNT_TYPE, + OracleError::InvalidAccountHeader.into(), + ) + .ok()? + } + + Some(bytemuck::from_bytes::( + &price_account_info[0..size_of::()], + )) +} + +pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; + pub fn compute_publisher_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { - let mut publisher_caps: HashMap = HashMap::new(); + let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in accounts { - let price_account: &PriceAccount = bytemuck::from_bytes(&account); - let cap = 1_000_000 / (price_account.num_ as u64); - for i in 0..(price_account.num_ as usize) { - publisher_caps - .entry(price_account.comp_[i].pub_) - .and_modify(|e| *e += cap) - .or_insert(cap); + if let Some(price_account) = validate_price_account_readonly(account) { + let cap : u64 = PUBLISHER_CAPS_DENOMINATOR / u64::from(price_account.num_); + for i in 0..usize::try_from(price_account.num_).unwrap() { + publisher_caps + .entry(price_account.comp_[i].pub_) + .and_modify(|e: &mut u64| *e = e.saturating_add(cap)) + .or_insert(cap); + } } } From f6f4e3b489c9eb28470b35d6c7351c2d5198e683 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 19:39:23 +0100 Subject: [PATCH 26/49] go --- program/rust/src/validator.rs | 71 ++++++++++++++--------------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 332b9c75..e6358a3c 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -30,34 +30,39 @@ use { transaction_context::TransactionAccount, }, std::{ - collections::{BTreeMap, HashMap}, + collections::{ + BTreeMap, + HashMap, + }, mem::size_of, }, }; -// Attempts to validate and access the contents of an account as a PriceAccount. -fn validate_price_account( - price_account_info: &mut [u8], -) -> Result<&mut PriceAccount, AggregationError> { - // TODO: don't return error on non-price account? +// Checks that the account is a PriceAccount from the length and header. +fn check_price_account_header(price_account_info: &[u8]) -> Result<(), ProgramError> { pyth_assert( price_account_info.len() >= PriceAccount::MINIMUM_SIZE, OracleError::AccountTooSmall.into(), - ) - .map_err(|_| AggregationError::NotPriceFeedAccount)?; + )?; - { - let account_header = bytemuck::from_bytes::( - &price_account_info[0..size_of::()], - ); - - pyth_assert( - account_header.magic_number == PC_MAGIC - && account_header.account_type == PriceAccount::ACCOUNT_TYPE, - OracleError::InvalidAccountHeader.into(), - ) + let account_header = + bytemuck::from_bytes::(&price_account_info[0..size_of::()]); + + pyth_assert( + account_header.magic_number == PC_MAGIC + && account_header.account_type == PriceAccount::ACCOUNT_TYPE, + OracleError::InvalidAccountHeader.into(), + )?; + + Ok(()) +} + +// Attempts to validate and access the contents of an account as a PriceAccount. +fn validate_price_account( + price_account_info: &mut [u8], +) -> Result<&mut PriceAccount, AggregationError> { + check_price_account_header(price_account_info) .map_err(|_| AggregationError::NotPriceFeedAccount)?; - } let data = bytemuck::from_bytes_mut::( &mut price_account_info[0..size_of::()], @@ -145,26 +150,8 @@ pub fn aggregate_price( ]) } -fn validate_price_account_readonly(price_account_info: &[u8]) -> Option<&PriceAccount> { - pyth_assert( - price_account_info.len() >= PriceAccount::MINIMUM_SIZE, - OracleError::AccountTooSmall.into(), - ) - .ok()?; - - { - let account_header = bytemuck::from_bytes::( - &price_account_info[0..size_of::()], - ); - - pyth_assert( - account_header.magic_number == PC_MAGIC - && account_header.account_type == PriceAccount::ACCOUNT_TYPE, - OracleError::InvalidAccountHeader.into(), - ) - .ok()? - } - +fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount> { + check_price_account_header(price_account_info).ok()?; Some(bytemuck::from_bytes::( &price_account_info[0..size_of::()], )) @@ -175,9 +162,9 @@ pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; pub fn compute_publisher_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in accounts { - if let Some(price_account) = validate_price_account_readonly(account) { - let cap : u64 = PUBLISHER_CAPS_DENOMINATOR / u64::from(price_account.num_); - for i in 0..usize::try_from(price_account.num_).unwrap() { + if let Some(price_account) = checked_load_price_account(account) { + let cap: u64 = PUBLISHER_CAPS_DENOMINATOR / u64::from(price_account.num_); + for i in 0..(price_account.num_ as usize) { publisher_caps .entry(price_account.comp_[i].pub_) .and_modify(|e: &mut u64| *e = e.saturating_add(cap)) From fa751b44548808cb99f76f40e6e03d1f901460a8 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 19:41:43 +0100 Subject: [PATCH 27/49] rename --- program/rust/src/validator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index e6358a3c..ce890e5d 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -159,7 +159,7 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; -pub fn compute_publisher_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { +pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in accounts { if let Some(price_account) = checked_load_price_account(account) { From 2c78d8c16acee5bc34194627c84c67925d34e40f Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 20:39:10 +0100 Subject: [PATCH 28/49] add test --- program/rust/src/tests/test_message.rs | 33 +++++++++++++++++++++++++- program/rust/src/validator.rs | 3 ++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/program/rust/src/tests/test_message.rs b/program/rust/src/tests/test_message.rs index 9e09d98d..7dba6a7f 100644 --- a/program/rust/src/tests/test_message.rs +++ b/program/rust/src/tests/test_message.rs @@ -5,11 +5,20 @@ use { messages::{ Message, PriceFeedMessage, + PublisherCapsMessage, TwapMessage, }, - wire::from_slice, + wire::{ + from_slice, + Serializer, + }, + }, + quickcheck::{ + Gen, + QuickCheck, }, quickcheck_macros::quickcheck, + serde::Serialize, }; #[quickcheck] @@ -38,3 +47,25 @@ fn test_twap_message_roundtrip(input: TwapMessage) -> bool { _ => false, } } + + +fn prop_publisher_caps_message_roundtrip(input: PublisherCapsMessage) -> bool { + let reconstructed = from_slice::(&input.clone().to_bytes()).unwrap(); + + println!("Failed test case:"); + println!("{:?}", input); + println!("{:?}", reconstructed); + + match reconstructed { + Message::PublisherCapsMessage(reconstructed) => reconstructed == input, + _ => false, + } +} + +#[test] +fn test_publisher_caps_message_roundtrip() { + // Configure the size parameter for the generator + QuickCheck::new() + .gen(Gen::new(1024)) + .quickcheck(prop_publisher_caps_message_roundtrip as fn(PublisherCapsMessage) -> bool); +} diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index ce890e5d..08bde4d6 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -181,7 +181,8 @@ pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec publisher: publisher.to_bytes(), cap, }) - .collect(), + .collect::>() + .into(), }; return message.to_bytes(); From 80a3ccf2842b284360f9734c39dd35816b585f01 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 20:40:32 +0100 Subject: [PATCH 29/49] cargo lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d81917e1..14ddd569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive 0.10.3", - "hashbrown 0.11.2", + "hashbrown 0.12.3", ] [[package]] @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#37f40e733a584d5a8af98161d786da4a3ac7176d" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#3522f2940522e49aa56038e1000ea78f2e489d20" dependencies = [ "bincode", "borsh 0.10.3", From f853cd0364530e4746aa94a7551daf11c20d235b Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 20:40:51 +0100 Subject: [PATCH 30/49] go --- program/rust/src/accounts/price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 2596208f..82ce5a62 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -349,7 +349,7 @@ impl PythOracleSerialize for PublisherCapsMessage { result.extend_from_slice(&[DISCRIMINATOR]); result.extend_from_slice(&self.publish_time.to_be_bytes()); - result.extend_from_slice(&self.caps.len().to_be_bytes()); + result.extend_from_slice(&TryInto::::try_into(self.caps.as_ref().len()).unwrap().to_be_bytes()); for cap in self.caps { result.extend_from_slice(&cap.publisher); From 170afdec9f5434deb3c242765ca409b180f306f7 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 17 Jul 2024 22:12:48 +0100 Subject: [PATCH 31/49] go --- program/rust/src/validator.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 08bde4d6..05e0a76f 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -163,7 +163,9 @@ pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in accounts { if let Some(price_account) = checked_load_price_account(account) { - let cap: u64 = PUBLISHER_CAPS_DENOMINATOR / u64::from(price_account.num_); + let cap: u64 = PUBLISHER_CAPS_DENOMINATOR + .checked_div(u64::from(price_account.num_)) + .unwrap_or(0); for i in 0..(price_account.num_ as usize) { publisher_caps .entry(price_account.comp_[i].pub_) From ef90813faebd92d62c604760413dcf940158a52e Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 15:47:01 +0100 Subject: [PATCH 32/49] format --- program/rust/src/accounts/price.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 82ce5a62..ffb142d2 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -349,7 +349,11 @@ impl PythOracleSerialize for PublisherCapsMessage { result.extend_from_slice(&[DISCRIMINATOR]); result.extend_from_slice(&self.publish_time.to_be_bytes()); - result.extend_from_slice(&TryInto::::try_into(self.caps.as_ref().len()).unwrap().to_be_bytes()); + result.extend_from_slice( + &TryInto::::try_into(self.caps.as_ref().len()) + .unwrap() + .to_be_bytes(), + ); for cap in self.caps { result.extend_from_slice(&cap.publisher); From 6ad489389535e04b9a2d73fed560e8fb2efdfe75 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:08:18 +0100 Subject: [PATCH 33/49] rename --- Cargo.lock | 4 ++-- program/rust/src/accounts/price.rs | 4 ++-- program/rust/src/tests/test_message.rs | 8 ++++---- program/rust/src/validator.rs | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14ddd569..975e2380 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive 0.10.3", - "hashbrown 0.12.3", + "hashbrown 0.11.2", ] [[package]] @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#3522f2940522e49aa56038e1000ea78f2e489d20" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#4989828325d932bcbbf67bdc507301a8b23dcaee" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index ffb142d2..b4393044 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -13,7 +13,7 @@ use { }, pythnet_sdk::messages::{ PriceFeedMessage, - PublisherCapsMessage, + PublisherStakeCapsMessage, TwapMessage, }, solana_program::pubkey::Pubkey, @@ -342,7 +342,7 @@ impl PythOracleSerialize for TwapMessage { } } -impl PythOracleSerialize for PublisherCapsMessage { +impl PythOracleSerialize for PublisherStakeCapsMessage { fn to_bytes(self) -> Vec { const DISCRIMINATOR: u8 = 2; let mut result: Vec = Vec::new(); diff --git a/program/rust/src/tests/test_message.rs b/program/rust/src/tests/test_message.rs index 7dba6a7f..d9630e5c 100644 --- a/program/rust/src/tests/test_message.rs +++ b/program/rust/src/tests/test_message.rs @@ -5,7 +5,7 @@ use { messages::{ Message, PriceFeedMessage, - PublisherCapsMessage, + PublisherStakeCapsMessage, TwapMessage, }, wire::{ @@ -49,7 +49,7 @@ fn test_twap_message_roundtrip(input: TwapMessage) -> bool { } -fn prop_publisher_caps_message_roundtrip(input: PublisherCapsMessage) -> bool { +fn prop_publisher_caps_message_roundtrip(input: PublisherStakeCapsMessage) -> bool { let reconstructed = from_slice::(&input.clone().to_bytes()).unwrap(); println!("Failed test case:"); @@ -57,7 +57,7 @@ fn prop_publisher_caps_message_roundtrip(input: PublisherCapsMessage) -> bool { println!("{:?}", reconstructed); match reconstructed { - Message::PublisherCapsMessage(reconstructed) => reconstructed == input, + Message::PublisherStakeCapsMessage(reconstructed) => reconstructed == input, _ => false, } } @@ -67,5 +67,5 @@ fn test_publisher_caps_message_roundtrip() { // Configure the size parameter for the generator QuickCheck::new() .gen(Gen::new(1024)) - .quickcheck(prop_publisher_caps_message_roundtrip as fn(PublisherCapsMessage) -> bool); + .quickcheck(prop_publisher_caps_message_roundtrip as fn(PublisherStakeCapsMessage) -> bool); } diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 05e0a76f..8f347621 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -17,8 +17,8 @@ use { utils::pyth_assert, }, pythnet_sdk::messages::{ - PublisherCap, - PublisherCapsMessage, + PublisherStakeCap, + PublisherStakeCapsMessage, }, solana_sdk::{ account::{ @@ -175,15 +175,15 @@ pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec } } - let message = PublisherCapsMessage { + let message = PublisherStakeCapsMessage { publish_time: timestamp, caps: publisher_caps .into_iter() - .map(|(publisher, cap)| PublisherCap { + .map(|(publisher, cap)| PublisherStakeCap { publisher: publisher.to_bytes(), cap, }) - .collect::>() + .collect::>() .into(), }; From a0b470b121f7c8b885504a7db03aadeae01e6a29 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:18:12 +0100 Subject: [PATCH 34/49] bump again --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 975e2380..fed5f0fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2610,7 +2610,7 @@ dependencies = [ [[package]] name = "pythnet-sdk" version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#4989828325d932bcbbf67bdc507301a8b23dcaee" +source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#76a9a50e51eed23fa123af08db7b1f79d708544e" dependencies = [ "bincode", "borsh 0.10.3", From 14c719d9d033c406ccf76dcf7f142a3cdc450d82 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:22:27 +0100 Subject: [PATCH 35/49] add z --- program/rust/src/validator.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 8f347621..2591113b 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -30,11 +30,10 @@ use { transaction_context::TransactionAccount, }, std::{ - collections::{ + cmp::max, collections::{ BTreeMap, HashMap, - }, - mem::size_of, + }, mem::size_of }, }; @@ -159,12 +158,12 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; -pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64) -> Vec { +pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64, z : u64) -> Vec { let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in accounts { if let Some(price_account) = checked_load_price_account(account) { let cap: u64 = PUBLISHER_CAPS_DENOMINATOR - .checked_div(u64::from(price_account.num_)) + .checked_div(max(u64::from(price_account.num_), z)) .unwrap_or(0); for i in 0..(price_account.num_ as usize) { publisher_caps From 706a29c55b5114af303a1dbac684c50b262f72aa Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 18 Jul 2024 16:23:52 +0100 Subject: [PATCH 36/49] go --- program/rust/src/validator.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 2591113b..d8dac483 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -30,10 +30,12 @@ use { transaction_context::TransactionAccount, }, std::{ - cmp::max, collections::{ + cmp::max, + collections::{ BTreeMap, HashMap, - }, mem::size_of + }, + mem::size_of, }, }; @@ -158,9 +160,9 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; -pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64, z : u64) -> Vec { +pub fn compute_publisher_stake_caps(account_datas: Vec<&[u8]>, timestamp: i64, z: u64) -> Vec { let mut publisher_caps: BTreeMap = BTreeMap::new(); - for account in accounts { + for account in account_datas { if let Some(price_account) = checked_load_price_account(account) { let cap: u64 = PUBLISHER_CAPS_DENOMINATOR .checked_div(max(u64::from(price_account.num_), z)) @@ -174,7 +176,7 @@ pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64, z : u6 } } - let message = PublisherStakeCapsMessage { + PublisherStakeCapsMessage { publish_time: timestamp, caps: publisher_caps .into_iter() @@ -184,7 +186,5 @@ pub fn compute_publisher_stake_caps(accounts: Vec<&[u8]>, timestamp: i64, z : u6 }) .collect::>() .into(), - }; - - return message.to_bytes(); + }.to_bytes() } From d949c27e14010427ce654055827c1610e577ecec Mon Sep 17 00:00:00 2001 From: Pavel Strakhov Date: Fri, 19 Jul 2024 11:43:12 +0100 Subject: [PATCH 37/49] chore: add missing imports and fix warnings --- program/rust/src/accounts.rs | 4 +- program/rust/src/lib.rs | 2 + program/rust/src/processor.rs | 11 +-- .../tests/test_upd_price_with_validator.rs | 76 +++++-------------- 4 files changed, 27 insertions(+), 66 deletions(-) diff --git a/program/rust/src/accounts.rs b/program/rust/src/accounts.rs index f05e0685..8bc83ce5 100644 --- a/program/rust/src/accounts.rs +++ b/program/rust/src/accounts.rs @@ -43,6 +43,8 @@ mod product; #[cfg(feature = "strum")] pub use price::MessageType; #[cfg(test)] +pub use price::PriceCumulative; +#[cfg(test)] pub use product::{ account_has_key_values, create_pc_str_t, @@ -54,13 +56,11 @@ pub use { PriceAccount, PriceAccountFlags, PriceComponent, - PriceCumulative, PriceEma, PriceInfo, PythOracleSerialize, }, product::{ - read_pc_str_t, update_product_metadata, ProductAccount, }, diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index e71b7c65..eb1881ef 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -37,11 +37,13 @@ pub use accounts::{ MappingAccount, PermissionAccount, PriceAccount, + PriceAccountFlags, PriceComponent, PriceEma, PriceInfo, ProductAccount, PythAccount, + PythOracleSerialize, }; use { crate::error::OracleError, diff --git a/program/rust/src/processor.rs b/program/rust/src/processor.rs index 7018a948..b4104fc1 100644 --- a/program/rust/src/processor.rs +++ b/program/rust/src/processor.rs @@ -27,14 +27,15 @@ mod upd_permissions; mod upd_price; mod upd_product; +#[cfg(test)] +pub use add_publisher::{ + DISABLE_ACCUMULATOR_V2, + ENABLE_ACCUMULATOR_V2, +}; pub use { add_price::add_price, add_product::add_product, - add_publisher::{ - add_publisher, - DISABLE_ACCUMULATOR_V2, - ENABLE_ACCUMULATOR_V2, - }, + add_publisher::add_publisher, del_price::del_price, del_product::del_product, del_publisher::del_publisher, diff --git a/program/rust/src/tests/test_upd_price_with_validator.rs b/program/rust/src/tests/test_upd_price_with_validator.rs index f25db775..c3d5111a 100644 --- a/program/rust/src/tests/test_upd_price_with_validator.rs +++ b/program/rust/src/tests/test_upd_price_with_validator.rs @@ -24,9 +24,7 @@ use { update_clock_slot, AccountSetup, }, - validator::{ - self, - }, + validator, }, solana_program::{ program_error::ProgramError, @@ -122,22 +120,12 @@ fn test_upd_price_with_validator() { } // We aggregate the price at the end of each slot now. - validator::aggregate_price( - 1, - 101, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(1, 101, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 2); - validator::aggregate_price( - 2, - 102, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(2, 102, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 3); // add next price in new slot triggering snapshot and aggregate calc populate_instruction(&mut instruction_data, 81, 2, 2); @@ -168,13 +156,8 @@ fn test_upd_price_with_validator() { // next price doesn't change but slot does populate_instruction(&mut instruction_data, 81, 2, 3); - validator::aggregate_price( - 3, - 103, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(3, 103, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 4); assert!(process_instruction( &program_id, @@ -201,13 +184,8 @@ fn test_upd_price_with_validator() { // next price doesn't change and neither does aggregate but slot does populate_instruction(&mut instruction_data, 81, 2, 4); - validator::aggregate_price( - 4, - 104, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(4, 104, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 5); assert!(process_instruction( @@ -263,13 +241,8 @@ fn test_upd_price_with_validator() { } populate_instruction(&mut instruction_data, 50, 20, 5); - validator::aggregate_price( - 5, - 105, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(5, 105, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 6); // Publishing a wide CI results in a status of unknown. @@ -306,13 +279,8 @@ fn test_upd_price_with_validator() { // Crank one more time and aggregate should be unknown populate_instruction(&mut instruction_data, 50, 20, 6); - validator::aggregate_price( - 6, - 106, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(6, 106, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 7); @@ -341,13 +309,8 @@ fn test_upd_price_with_validator() { // Negative prices are accepted populate_instruction(&mut instruction_data, -100, 1, 7); - validator::aggregate_price( - 7, - 107, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(7, 107, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 8); @@ -376,13 +339,8 @@ fn test_upd_price_with_validator() { // Crank again for aggregate populate_instruction(&mut instruction_data, -100, 1, 8); - validator::aggregate_price( - 8, - 108, - price_account.key, - &mut *price_account.data.borrow_mut(), - ) - .unwrap(); + validator::aggregate_price(8, 108, price_account.key, *price_account.data.borrow_mut()) + .unwrap(); update_clock_slot(&mut clock_account, 9); From fe64764a92e7212fc9b737070b7d3e288bcc4730 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Fri, 19 Jul 2024 19:38:34 +0100 Subject: [PATCH 38/49] add m --- program/rust/src/validator.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index d8dac483..6345bb05 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -158,13 +158,16 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount )) } -pub const PUBLISHER_CAPS_DENOMINATOR: u64 = 1_000_000; - -pub fn compute_publisher_stake_caps(account_datas: Vec<&[u8]>, timestamp: i64, z: u64) -> Vec { +pub fn compute_publisher_stake_caps( + account_datas: Vec<&[u8]>, + timestamp: i64, + m: u64, + z: u64, +) -> Vec { let mut publisher_caps: BTreeMap = BTreeMap::new(); for account in account_datas { if let Some(price_account) = checked_load_price_account(account) { - let cap: u64 = PUBLISHER_CAPS_DENOMINATOR + let cap: u64 = m .checked_div(max(u64::from(price_account.num_), z)) .unwrap_or(0); for i in 0..(price_account.num_ as usize) { @@ -186,5 +189,6 @@ pub fn compute_publisher_stake_caps(account_datas: Vec<&[u8]>, timestamp: i64, z }) .collect::>() .into(), - }.to_bytes() + } + .to_bytes() } From 91b235be224f18644370971d11e90da231c0e7e8 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Mon, 22 Jul 2024 15:35:23 +0100 Subject: [PATCH 39/49] Clippy --- program/rust/src/accounts/price.rs | 2 +- program/rust/src/tests/test_message.rs | 6 +----- program/rust/src/validator.rs | 11 +---------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index b4393044..2587afc3 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -360,6 +360,6 @@ impl PythOracleSerialize for PublisherStakeCapsMessage { result.extend_from_slice(&cap.cap.to_be_bytes()); } - return result; + result } } diff --git a/program/rust/src/tests/test_message.rs b/program/rust/src/tests/test_message.rs index d9630e5c..427b106a 100644 --- a/program/rust/src/tests/test_message.rs +++ b/program/rust/src/tests/test_message.rs @@ -8,17 +8,13 @@ use { PublisherStakeCapsMessage, TwapMessage, }, - wire::{ - from_slice, - Serializer, - }, + wire::from_slice, }, quickcheck::{ Gen, QuickCheck, }, quickcheck_macros::quickcheck, - serde::Serialize, }; #[quickcheck] diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 6345bb05..dbb68919 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -8,7 +8,6 @@ use { PythOracleSerialize, }, c_oracle_header::PC_MAGIC, - deserialize::load_account_as, error::OracleError, processor::{ c_upd_aggregate, @@ -21,20 +20,12 @@ use { PublisherStakeCapsMessage, }, solana_sdk::{ - account::{ - AccountSharedData, - ReadableAccount, - }, program_error::ProgramError, pubkey::Pubkey, - transaction_context::TransactionAccount, }, std::{ cmp::max, - collections::{ - BTreeMap, - HashMap, - }, + collections::BTreeMap, mem::size_of, }, }; From b2320d9510d6f7385d019d349c21869b2a88889f Mon Sep 17 00:00:00 2001 From: guibescos <59208140+guibescos@users.noreply.github.com> Date: Wed, 24 Jul 2024 22:46:07 +0100 Subject: [PATCH 40/49] Update program/rust/src/accounts/price.rs Co-authored-by: Pavel Strakhov --- program/rust/src/accounts/price.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 2587afc3..7ee41675 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -345,9 +345,7 @@ impl PythOracleSerialize for TwapMessage { impl PythOracleSerialize for PublisherStakeCapsMessage { fn to_bytes(self) -> Vec { const DISCRIMINATOR: u8 = 2; - let mut result: Vec = Vec::new(); - - result.extend_from_slice(&[DISCRIMINATOR]); + let mut result = vec![DISCRIMINATOR]; result.extend_from_slice(&self.publish_time.to_be_bytes()); result.extend_from_slice( &TryInto::::try_into(self.caps.as_ref().len()) From 0783aa5af8d551ab13693b50957a54b934955b72 Mon Sep 17 00:00:00 2001 From: guibescos <59208140+guibescos@users.noreply.github.com> Date: Wed, 24 Jul 2024 22:46:12 +0100 Subject: [PATCH 41/49] Update program/rust/src/accounts/price.rs Co-authored-by: Pavel Strakhov --- program/rust/src/accounts/price.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index 7ee41675..da4d9f1b 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -348,7 +348,7 @@ impl PythOracleSerialize for PublisherStakeCapsMessage { let mut result = vec![DISCRIMINATOR]; result.extend_from_slice(&self.publish_time.to_be_bytes()); result.extend_from_slice( - &TryInto::::try_into(self.caps.as_ref().len()) + &u16::try_from(self.caps.as_ref().len()) .unwrap() .to_be_bytes(), ); From 02bbe210f51afd6796b72ddfa05f413f4c7334d8 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 24 Jul 2024 23:01:10 +0100 Subject: [PATCH 42/49] Remove vector --- program/rust/src/validator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index dbb68919..e0a1d04f 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -149,8 +149,8 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount )) } -pub fn compute_publisher_stake_caps( - account_datas: Vec<&[u8]>, +pub fn compute_publisher_stake_caps<'a>( + account_datas: impl IntoIterator, timestamp: i64, m: u64, z: u64, From d7fa02df4a6419d2ed5f0b051e550ada3d2150a3 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 17:50:54 +0100 Subject: [PATCH 43/49] fix: remove unwrap --- program/rust/src/accounts/price.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/program/rust/src/accounts/price.rs b/program/rust/src/accounts/price.rs index da4d9f1b..a569d054 100644 --- a/program/rust/src/accounts/price.rs +++ b/program/rust/src/accounts/price.rs @@ -17,7 +17,10 @@ use { TwapMessage, }, solana_program::pubkey::Pubkey, - std::mem::size_of, + std::{ + mem::size_of, + u16, + }, }; /// Pythnet-specific PriceAccount implementation @@ -349,7 +352,7 @@ impl PythOracleSerialize for PublisherStakeCapsMessage { result.extend_from_slice(&self.publish_time.to_be_bytes()); result.extend_from_slice( &u16::try_from(self.caps.as_ref().len()) - .unwrap() + .unwrap_or(u16::MAX) .to_be_bytes(), ); From dbbbc06d1ef133bc163f72f491184320d706fbbf Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 17:59:28 +0100 Subject: [PATCH 44/49] add comments --- program/rust/src/validator.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index e0a1d04f..cc615a8a 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -149,6 +149,13 @@ fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount )) } +/// Computes the stake caps for each publisher based on the oracle program accounts provided +/// - `account_datas` - the account datas of the oracle program accounts +/// - `timestamp` - the timestamp to include in the message +/// - `m` - m is the cap per symbol, it gets split among all publishers of the symbol +/// - `z` - when a symbol has less than `z` publishers, each publisher gets a cap of `m/z` (instead of `m/number_of_publishers`). This is to prevent a single publisher from getting a large cap when there are few publishers. +/// +/// The stake cap for a publisher is computed as the sum of `m/min(z, number_of_publishers)` for all the symbols the publisher is publishing. pub fn compute_publisher_stake_caps<'a>( account_datas: impl IntoIterator, timestamp: i64, From 546efd72435752878465e550c4f94e96f060b018 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 18:01:51 +0100 Subject: [PATCH 45/49] fix: undo this --- program/rust/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/lib.rs b/program/rust/src/lib.rs index 732ce0ae..eb1881ef 100644 --- a/program/rust/src/lib.rs +++ b/program/rust/src/lib.rs @@ -1,4 +1,4 @@ -// #![deny(warnings)] +#![deny(warnings)] // Allow non upper case globals from C #![allow(non_upper_case_globals)] From 486e558c1a541ed9d3b84b6b3a640a312bb1539c Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 18:04:49 +0100 Subject: [PATCH 46/49] another comment --- program/rust/src/validator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index cc615a8a..d0ccee96 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -142,6 +142,7 @@ pub fn aggregate_price( ]) } +/// Load a price account as read-only, returning `None` if it isn't a valid price account. fn checked_load_price_account(price_account_info: &[u8]) -> Option<&PriceAccount> { check_price_account_header(price_account_info).ok()?; Some(bytemuck::from_bytes::( From f6de9ed368a7aa0740298740c8585cf82273c9fc Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Thu, 25 Jul 2024 18:36:14 +0100 Subject: [PATCH 47/49] add comment --- program/rust/src/validator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index d0ccee96..8c718aa3 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -163,7 +163,7 @@ pub fn compute_publisher_stake_caps<'a>( m: u64, z: u64, ) -> Vec { - let mut publisher_caps: BTreeMap = BTreeMap::new(); + let mut publisher_caps: BTreeMap = BTreeMap::new(); // BTreeMap to ensure it will be sorted by publisher for account in account_datas { if let Some(price_account) = checked_load_price_account(account) { let cap: u64 = m From 632ee4598b81ebed193e1ecea3f760039bdb5019 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 31 Jul 2024 18:10:02 +0100 Subject: [PATCH 48/49] use checked indexing --- program/rust/src/validator.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/program/rust/src/validator.rs b/program/rust/src/validator.rs index 8c718aa3..17a3aa10 100644 --- a/program/rust/src/validator.rs +++ b/program/rust/src/validator.rs @@ -170,10 +170,12 @@ pub fn compute_publisher_stake_caps<'a>( .checked_div(max(u64::from(price_account.num_), z)) .unwrap_or(0); for i in 0..(price_account.num_ as usize) { - publisher_caps - .entry(price_account.comp_[i].pub_) - .and_modify(|e: &mut u64| *e = e.saturating_add(cap)) - .or_insert(cap); + if let Some(pub_) = price_account.comp_.get(i).map(|comp| &comp.pub_) { + publisher_caps + .entry(*pub_) + .and_modify(|e: &mut u64| *e = e.saturating_add(cap)) + .or_insert(cap); + } } } } From 8195a381fd73de3e59716895d2ecc285b871c544 Mon Sep 17 00:00:00 2001 From: Guillermo Bescos Date: Wed, 31 Jul 2024 19:59:12 +0100 Subject: [PATCH 49/49] fix: use crates version of pythnet-sdk --- Cargo.lock | 7 ++++--- program/rust/Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b43d7168..7cb47d6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive 0.10.3", - "hashbrown 0.11.2", + "hashbrown 0.12.3", ] [[package]] @@ -2609,8 +2609,9 @@ dependencies = [ [[package]] name = "pythnet-sdk" -version = "2.1.0" -source = "git+https://github.com/pyth-network/pyth-crosschain?branch=feat/publisher-caps#76a9a50e51eed23fa123af08db7b1f79d708544e" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f81d23a693463366640d1ba14c612841b5b07adc56ee20bbf9acaca2ad86b5" dependencies = [ "bincode", "borsh 0.10.3", diff --git a/program/rust/Cargo.toml b/program/rust/Cargo.toml index 05536b76..b755605e 100644 --- a/program/rust/Cargo.toml +++ b/program/rust/Cargo.toml @@ -17,7 +17,7 @@ num-traits = "0.2" byteorder = "1.4.3" serde = { version = "1.0", features = ["derive"], optional = true } strum = { version = "0.24.1", features = ["derive"], optional = true } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", branch="feat/publisher-caps"} +pythnet-sdk = "2.2.0" solana-sdk = { version = "=1.14.17", optional = true } bitflags = { version = "2.6.0", features = ["bytemuck"] } @@ -31,7 +31,7 @@ rand = "0.8.5" quickcheck_macros = "1" bincode = "1.3.3" serde = { version = "1.0", features = ["derive"] } -pythnet-sdk = { git = "https://github.com/pyth-network/pyth-crosschain", branch="feat/publisher-caps", features = ["quickcheck"]} +pythnet-sdk = { version = "2.2.0" , features = ["quickcheck"]} serde_json = "1.0" test-generator = "0.3.1" csv = "1.1"