Change to new, experimental cargo feature resolver

The [new cargo feature resolver](rust-lang/cargo#7820)
won't unify features across build deps, dev deps, and targets. This solves our
problem of needing to work around feature unification to avoid Clone
implementations on private key material.

This commit puts back our true dependency information into the
Cargo.tomls. Specifically, it adds dev-dependencies that enable features that
aren't enabled on normal dependencies. This will cause the feature unification
to happen when using the old resolver, but the build will be correct under the
new resolver.

In order to maintain correct builds in CI, this commit also changes CI to use
the nightly cargo with `-Z features=all` but still preserving the rustc
toolchain specified in `rustc-toolchain`. Local developer builds will likely
still use the `rustc-toolchain` version of cargo with the old resolver, but
this shouldn't cause any problems for development.

Author: metajack

.circleci/config.yml

@@ -15,14 +15,6 @@ executors:
     resource_class: small
 
 commands:
-  rust_setup:
-    description: Set rustc version
-    steps:
-      - run:
-          name: Set rustc version
-          command: |
-            rustup default stable
-            rustup update stable
   print_versions:
     description: Version Info
     steps:
@@ -32,6 +24,10 @@ commands:
   env_setup:
     description: Environment Setup
     steps:
+      - run:
+          name: Set RUST_NIGHTLY
+          command: |
+            echo 'export RUST_NIGHTLY=nightly-2020-03-18' >> $BASH_ENV
       - run:
           name: Setup Env
           command: |
@@ -40,6 +36,8 @@ commands:
             echo 'export LIBRA_DUMP_LOGS=1' >> $BASH_ENV
             echo 'export CARGO_INCREMENTAL=0' >> $BASH_ENV
             echo 'export CI_TIMEOUT="timeout 40m"' >> $BASH_ENV
+            echo 'export CARGO=$(rustup which --toolchain $RUST_NIGHTLY cargo)' >> $BASH_ENV
+            echo 'export CARGOFLAGS=-Zfeatures=all' >> $BASH_ENV
   install_deps:
     steps:
       - run:
@@ -48,14 +46,15 @@ commands:
             sudo apt-get update
             sudo apt-get install -y cmake curl clang llvm
             rustup component add clippy rustfmt
+            rustup toolchain install nightly
   install_code_coverage_deps:
     steps:
       - run:
           name: Install grcov and lcov
           command: |
             sudo apt-get update
             sudo apt-get install lcov
-            cargo install --force grcov
+            $(CARGO) install --force grcov
   install_docker_linter:
     steps:
       - run:
@@ -64,11 +63,6 @@ commands:
              export HADOLINT=${HOME}/hadolint
              export HADOLINT_VER=v1.17.4
              curl -sL -o ${HADOLINT} "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VER}/hadolint-$(uname -s)-$(uname -m)" && chmod 700 ${HADOLINT}
-  install_rust_nightly_toolchain:
-    steps:
-      - run:
-          name: Install nightly toolchain for features not in beta/stable
-          command: rustup install nightly
   find_dockerfile_changes:
     steps:
       - run:
@@ -82,7 +76,6 @@ commands:
   build_setup:
     steps:
       - checkout
-      - rust_setup
       - print_versions
       - env_setup
       - install_deps
@@ -104,37 +97,37 @@ jobs:
       - run:
           name: Linting
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo x lint
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xclippy --workspace --all-targets
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xfmt --check
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS x lint
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS xclippy --workspace --all-targets
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS xfmt --check
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $($CARGO $CARGOFLAGS guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
       - run:
           name: Build Release
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 cargo build -j 16 --release
+            [[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 --release
       - run:
           name: Build Dev
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-swarm
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cluster-test
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-fuzzer
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p language_benchmarks
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cost-synthesis
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p test-generation
       - run:
           name: Run All Non Flaky Unit Tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
+            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGS test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
       - run:
           name: Run Cryptography Unit Tests with the formally verified backend
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT cargo test --features='std fiat_u64_backend fuzzing' --no-default-features )
+            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT $CARGO $CARGOFLAGS test --features='std fiat_u64_backend fuzzing' --no-default-features )
       - run:
           name: Run All End to End Tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test --package testsuite -- --test-threads 1
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGSx test --package testsuite -- --test-threads 1
       - run:
           name: Run Quarantined Unit Tests 3 (consensus) times
           command: |
@@ -155,16 +148,16 @@ jobs:
     steps:
       - build_setup
       - run:
-          name: Install Cargo Audit
+          name: Install cargo-audit
           command: |
-            cargo install --force cargo-audit
+            $CARGO $CARGOFLAGS install --force cargo-audit
       - run:
           # NOTE ignored advisory rules
           # RUSTSEC-2018-0015 - term
           # RUSTSEC-2019-0031 - spin
           name: Audit crates
           command: |
-            cargo audit --deny-warnings \
+            $CARGO $CARGOFLAGS audit --deny-warnings \
               --ignore RUSTSEC-2018-0015 \
               --ignore RUSTSEC-2019-0031
       - build_teardown
@@ -174,7 +167,6 @@ jobs:
     steps:
       - build_setup
       - install_code_coverage_deps
-      - install_rust_nightly_toolchain
       - run:
           name: Setup code coverage output
           command: echo "export CODECOV_OUTPUT=codecov" >> $BASH_ENV

Cargo.lock

@@ -108,7 +108,8 @@ members = [
 
 # NOTE: These members should never include crates that require fuzzing
 # features or test features. These are the crates we want built with no extra
-# test-only code included.
+# test-only code included. These are essentially the main libra release
+# binaries.
 default-members = [
     "client/cli",
     "language/compiler",

Cargo.toml

@@ -32,6 +32,12 @@ serde_json = "1.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
+proptest = "0.9.4"
+libra-mempool = { path = "../../mempool", version = "0.1.0", features = ["fuzzing"] }
+libra-proptest-helpers = { path = "../../common/proptest-helpers" }
+libra-prost-ext = { path = "../../common/prost-ext", version = "0.1.0" }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
+storage-service = { path = "../../storage/storage-service" }
 vm-validator = { path = "../../vm-validator", version = "0.1.0" }
 
 [features]

admission_control/admission-control-service/Cargo.toml

@@ -42,6 +42,7 @@ transaction-builder = { path = "../../language/transaction-builder", version = "
 
 [dev-dependencies]
 proptest = "0.9.2"
+libra-crypto = { path = "../../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

client/cli/Cargo.toml

@@ -29,6 +29,9 @@ libra-logger = { path = "../common/logger", version = "0.1.0" }
 libra-temppath = { path = "../common/temppath", version = "0.1.0" }
 libra-types = { path = "../types", version = "0.1.0" }
 
+[dev-dependencies]
+libra-crypto = { path = "../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
+
 [features]
 default = []
 fuzzing = ["libra-crypto/fuzzing", "libra-types/fuzzing"]

config/Cargo.toml

@@ -31,7 +31,7 @@ prometheus = { version = "0.8.0", default-features = false }
 proptest = { version = "0.9.4", optional = true }
 
 channel = { path = "../common/channel", version = "0.1.0" }
-consensus-types = { path = "consensus-types", version = "0.1.0", default-features = false }
+consensus-types = { path = "consensus-types", version = "0.1.0" }
 crash-handler = { path = "../common/crash-handler", version = "0.1.0" }
 debug-interface = { path = "../common/debug-interface", version = "0.1.0" }
 executor = { path = "../execution/executor", version = "0.1.0" }
@@ -57,6 +57,9 @@ cached = "0.12.0"
 proptest = "0.9.4"
 tempfile = "3.1.0"
 
+consensus-types = { path = "consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../config", version = "0.1.0", features = ["fuzzing"] }
+libra-mempool = { path = "../mempool", version = "0.1.0", features = ["fuzzing"] }
 vm-genesis = { path = "../language/tools/vm-genesis", version = "0.1.0" }
 vm-validator = { path = "../vm-validator", version = "0.1.0" }
 

consensus/Cargo.toml

@@ -21,6 +21,7 @@ libra-types = { path = "../../types", version = "0.1.0" }
 
 [dev-dependencies]
 proptest = "0.9.4"
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

consensus/consensus-types/Cargo.toml

@@ -28,6 +28,8 @@ workspace-builder = { path = "../../common/workspace-builder", version = "0.1.0"
 criterion = "0.3"
 rand = { version = "0.6.5", default-features = false }
 tempfile = "3.1.0"
+consensus-types = { path = "../consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../../config", version = "0.1.0", features = ["fuzzing"] }
 
 [[bench]]
 name = "safety_rules"

consensus/safety-rules/Cargo.toml

@@ -44,7 +44,6 @@ struct SMRNode {
     commit_cb_receiver: mpsc::UnboundedReceiver<LedgerInfoWithSignatures>,
     storage: Arc<MockStorage<TestPayload>>,
     state_sync: mpsc::UnboundedReceiver<Vec<usize>>,
-    shared_mempool: MockSharedMempool,
 }
 
 impl SMRNode {
@@ -75,7 +74,7 @@ impl SMRNode {
         let (state_sync_client, state_sync) = mpsc::unbounded();
         let (commit_cb_sender, commit_cb_receiver) = mpsc::unbounded::<LedgerInfoWithSignatures>();
         let shared_mempool = MockSharedMempool::new(None);
-        let consensus_to_mempool_sender = shared_mempool.consensus_sender.clone();
+        let consensus_to_mempool_sender = shared_mempool.consensus_sender;
 
         let mut smr = ChainedBftSMR::new(
             network_sender,
@@ -101,7 +100,6 @@ impl SMRNode {
             commit_cb_receiver,
             storage,
             state_sync,
-            shared_mempool,
         }
     }