Merge pull request #6034 from fdefelici/test/pinny-stacks-node

test: integrate pinny on stacks-node

Author: wileyj

CONTRIBUTING.md

@@ -361,17 +361,31 @@ A non-exhaustive list of examples of consensus-critical changes include:
 
 - Every consensus-critical change needs an integration test to verify that the feature activates only when the hard fork activates.
 
-PRs must include test coverage. However, if your PR includes large tests or tests which cannot run in parallel
+- PRs must include test coverage. However, if your PR includes large tests or tests which cannot run in parallel
 (which is the default operation of the `cargo test` command), these tests should be decorated with `#[ignore]`.
-
 A test should be marked `#[ignore]` if:
 
-1. It does not _always_ pass `cargo test` in a vanilla environment
+  1. It does not _always_ pass `cargo test` in a vanilla environment
    (i.e., it does not need to run with `--test-threads 1`).
 
-2. Or, it runs for over a minute via a normal `cargo test` execution
+  2. Or, it runs for over a minute via a normal `cargo test` execution
    (the `cargo test` command will warn if this is not the case).
 
+- **Integration tests need to be properly tagged** using [pinny-rs](https://github.com/BitcoinL2-Labs/pinny-rs/) crate. Tagging requires two fundamental steps:
+  1. Define allowed tags in the package `Cargo.toml` file (if needed).
+  2. Apply relevant tags to the tests, picking from the allowed set.
+  
+  Then it will be possible to run tests with filtering based on the tags using `cargo test` and `cargo nextest` runner.
+  > For more information and examples on how tagging works, refer to the [pinny-rs](https://github.com/BitcoinL2-Labs/pinny-rs/) readme.
+  
+  Below the tag set currently defined with related purpose:
+
+  | Tag             | Description                                  |
+  |-----------------|----------------------------------------------|
+  | `slow`          | tests running over a minute                  |
+  | `bitcoind`      | tests requiring bitcoin daemon               |
+  | `flaky`         | tests that exhibit flaky behavior            |
+
 ## Formatting
 
 PRs will be checked against `rustfmt` and will _fail_ if not properly formatted.

Cargo.lock

@@ -51,6 +51,7 @@ http-types = "2.12"
 tempfile = "3.3"
 mockito = "1.5"
 serial_test = "3.2.0"
+pinny = { git = "https://github.com/BitcoinL2-Labs/pinny-rs.git", rev = "54ba9d533a7b84525a5e65a3eae1a3ae76b9ea49" } #v0.0.2
 madhouse = { git = "https://github.com/stacks-network/madhouse-rs.git", rev = "fc651ddcbaf85e888b06d4a87aa788c4b7ba9309" }
 proptest = { git = "https://github.com/proptest-rs/proptest.git", rev = "c9bdf18c232665b2b740c667c81866b598d06dc7" }
 
@@ -68,3 +69,6 @@ slog_json = ["stacks/slog_json", "stacks-common/slog_json", "clarity/slog_json"]
 prod-genesis-chainstate = []
 default = []
 testing = []
+
+[package.metadata.pinny] 
+allowed = ["bitcoind", "flaky", "slow"]

testnet/stacks-node/Cargo.toml

@@ -13,6 +13,7 @@ use clarity::vm::types::{
 };
 use clarity::vm::{ClarityVersion, Value};
 use lazy_static::lazy_static;
+use pinny::tag;
 use reqwest;
 use serde_json::json;
 use stacks::burnchains::Address;
@@ -41,7 +42,7 @@ use stacks::net::api::getistraitimplemented::GetIsTraitImplementedResponse;
 use stacks_common::types::chainstate::{StacksAddress, StacksBlockId, VRFSeed};
 use stacks_common::util::hash::{hex_bytes, to_hex, Sha256Sum};
 
-use super::{ADDR_4, SK_1, SK_2, SK_3};
+use super::{new_test_conf, ADDR_4, SK_1, SK_2, SK_3};
 use crate::helium::RunLoop;
 
 const OTHER_CONTRACT: &str = "
@@ -162,10 +163,11 @@ lazy_static! {
     static ref HTTP_BINDING: Mutex<Option<String>> = Mutex::new(None);
 }
 
+#[tag(slow)]
 #[test]
 #[ignore]
 fn integration_test_get_info() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
     let spender_addr = to_addr(&StacksPrivateKey::from_hex(SK_3).unwrap()).into();
     let principal_sk = StacksPrivateKey::from_hex(SK_2).unwrap();
     let contract_sk = StacksPrivateKey::from_hex(SK_1).unwrap();
@@ -1086,9 +1088,10 @@ const FAUCET_CONTRACT: &str = "
       (print (as-contract (stx-transfer? u1 .faucet recipient)))))
 ";
 
+#[tag(slow)]
 #[test]
 fn contract_stx_transfer() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
 
     let contract_sk = StacksPrivateKey::from_hex(SK_1).unwrap();
     let sk_3 = StacksPrivateKey::from_hex(SK_3).unwrap();
@@ -1452,7 +1455,7 @@ fn contract_stx_transfer() {
 
 #[test]
 fn mine_transactions_out_of_order() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
 
     let sk = StacksPrivateKey::from_hex(SK_3).unwrap();
     let addr = to_addr(&sk);
@@ -1624,7 +1627,7 @@ fn mine_transactions_out_of_order() {
 ///   in the block it was processed for. Tests issue #1540
 #[test]
 fn mine_contract_twice() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
     let contract_sk = StacksPrivateKey::from_hex(SK_1).unwrap();
 
     conf.burnchain.commit_anchor_block_within = 1000;
@@ -1710,7 +1713,7 @@ fn mine_contract_twice() {
 
 #[test]
 fn bad_contract_tx_rollback() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
 
     let contract_sk = StacksPrivateKey::from_hex(SK_1).unwrap();
     let sk_2 = StacksPrivateKey::from_hex(SK_2).unwrap();
@@ -1994,7 +1997,7 @@ fn make_keys(seed: &str, count: u64) -> Vec<StacksPrivateKey> {
 
 #[test]
 fn block_limit_runtime_test() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
 
     conf.burnchain.epochs = Some(EpochList::new(&[
         StacksEpoch {
@@ -2179,7 +2182,7 @@ fn block_limit_runtime_test() {
 
 #[test]
 fn mempool_errors() {
-    let mut conf = super::new_test_conf();
+    let mut conf = new_test_conf();
 
     conf.burnchain.commit_anchor_block_within = 5000;
 

testnet/stacks-node/src/tests/integrations.rs

@@ -30,6 +30,7 @@ use libsigner::{
     BlockProposal, BlockProposalData, SignerSession, StackerDBSession, VERSION_STRING,
 };
 use madhouse::{execute_commands, prop_allof, scenario, Command};
+use pinny::tag;
 use proptest::prelude::Strategy;
 use rand::{thread_rng, Rng};
 use rusqlite::Connection;
@@ -1487,6 +1488,7 @@ pub fn wait_for_state_machine_update(
     })
 }
 
+#[tag(bitcoind)]
 #[test]
 #[ignore]
 /// Test that a signer can respond to an invalid block proposal
@@ -10420,6 +10422,7 @@ fn multiple_miners_empty_sortition() {
     miners.shutdown();
 }
 
+#[tag(bitcoind, flaky, slow)]
 #[test]
 #[ignore]
 /// This test spins up a single nakamoto node configured to mine.