tests: Add kcov based coverage generation scripts

Samuel Ortiz · rbradford · commit 79bd1d59757c · 2019-06-13T11:03:39.000+01:00
And update/add the corresponding README sections.

Signed-off-by: Samuel Ortiz &lt;sameo@linux.intel.com&gt;
diff --git a/README.md b/README.md
@@ -13,16 +13,77 @@ cd linux-loader
 cargo build
 ```
 
-## How to run test for Elf loader
+## Tests
 
-```shell
-# Assuming your linux-loader is under $HOME
-$ cd linux-loader
-$ cargo test
-$ cargo test -- --nocapture
+Our Continuous Integration (CI) pipeline is implemented on top of
+[Buildkite](https://buildkite.com/).
+For the complete list of tests, check our
+[CI pipeline](https://buildkite.com/rust-vmm/vm-virtio-ci).
+
+Each individual test runs in a container. To reproduce a test locally, you can
+use the dev-container on both x86 and arm64.
+
+```bash
+docker run -it \
+           --security-opt seccomp=unconfined \
+           --volume $(pwd):/linux-loader \
+           fandree/rust-vmm-dev
+cd linux-loader/
+cargo test
 ```
 
-## How to run test for bzImage loader
+### Test Profiles
+
+The integration tests support two test profiles:
+- **devel**: this is the recommended profile for running the integration tests
+  on a local development machine.
+- **ci** (default option): this is the profile used when running the
+  integration tests as part of the the Continuous Integration (CI).
+
+The test profiles are applicable to tests that run using pytest. Currently only
+the [coverage test](tests/test_coverage.py) follows this model as all the other
+integration tests are run using the
+[Buildkite pipeline](https://buildkite.com/rust-vmm/vm-virtio-ci).
+
+The difference between is declaring tests as passed or failed:
+- with the **devel** profile the coverage test passes if the current coverage
+  is equal or higher than the upstream coverage value. In case the current
+  coverage is higher, the coverage file is updated to the new coverage value.
+- with the **ci** profile the coverage test passes only if the current coverage
+  is equal to the upstream coverage value.
+
+Further details about the coverage test can be found in the
+[Adaptive Coverage](#adaptive-coverage) section.
+
+### Adaptive Coverage
+
+The line coverage is saved in [tests/coverage](tests/coverage). To update the
+coverage before submitting a PR, run the coverage test:
+
+```bash
+docker run -it \
+           --security-opt seccomp=unconfined \
+           --volume $(pwd):/linux-loader \
+           fandree/rust-vmm-dev
+cd linux-loader/
+pytest --profile=devel tests/test_coverage.py
+```
+
+If the PR coverage is higher than the upstream coverage, the coverage file
+needs to be manually added to the commit before submitting the PR:
+
+```bash
+git add tests/coverage
+```
+
+Failing to do so will generate a fail on the CI pipeline when publishing the
+PR.
+
+**NOTE:** The coverage file is only updated in the `devel` test profile. In
+the `ci` profile the coverage test will fail if the current coverage is higher
+than the coverage reported in [tests/coverage](tests/coverage).
+
+### bzImage test
 
 As we don't want to distribute an entire kernel bzImage, the `load_bzImage` test is ignored by
 default. In order to test the bzImage support, one needs to locally build a bzImage, copy it
@@ -35,10 +96,10 @@ $ cd linux-stable
 $ make bzImage 
 $ cp linux-stable/arch/x86/boot/bzImage $LINUX_LOADER/linux-loader/src/loader/
 $ cd $LINUX_LOADER/linux-loader
+$ docker run -it \
+           --security-opt seccomp=unconfined \
+           --volume $(pwd):/linux-loader \
+           fandree/rust-vmm-dev
+$ cd linux-loader/
 $ cargo test -- --ignored
 ```
-
-## Platform Support
-- Arch: x86
-- OS: Linux/Unix
-
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,28 @@
+import pytest
+
+
+PROFILE_CI="ci"
+PROFILE_DEVEL="devel"
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--profile",
+        default=PROFILE_CI,
+        choices=[PROFILE_CI, PROFILE_DEVEL],
+        help="Profile for running the test: {} or {}".format(
+            PROFILE_CI,
+            PROFILE_DEVEL
+        )
+    )
+
+
+@pytest.fixture
+def profile(request):
+    return request.config.getoption("--profile")
+
+
+# This is used for defining global variables in pytest.
+def pytest_configure():
+    pytest.profile_ci = PROFILE_CI
+    pytest.profile_devel = PROFILE_DEVEL
diff --git a/tests/coverage b/tests/coverage
@@ -0,0 +1 @@
+63.0
diff --git a/tests/test_coverage.py b/tests/test_coverage.py
@@ -0,0 +1,108 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+"""Test the coverage and update the threshold when coverage is increased."""
+
+import os, re, shutil, subprocess
+import pytest
+
+def _get_current_coverage():
+    """Helper function that returns the coverage computed with kcov."""
+    kcov_ouput_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        "kcov_output"
+    )
+
+    # By default the build output for kcov and unit tests are both in the debug
+    # directory. This causes some linker errors that I haven't investigated.
+    # Error: error: linking with `cc` failed: exit code: 1
+    # An easy fix is to have separate build directories for kcov & unit tests.
+    kcov_build_dir = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        "kcov_build"
+    )
+
+    # Remove kcov output and build directory to be sure we are always working
+    # on a clean environment.
+    shutil.rmtree(kcov_ouput_dir, ignore_errors=True)
+    shutil.rmtree(kcov_build_dir, ignore_errors=True)
+
+    exclude_pattern = (
+        '${CARGO_HOME:-$HOME/.cargo/},'
+        'usr/lib/,'
+        'lib/'
+    )
+    exclude_region = "'mod tests {'"
+
+    kcov_cmd = "CARGO_TARGET_DIR={} cargo kcov --all " \
+               "--output {} -- " \
+               "--exclude-region={} " \
+               "--exclude-pattern={} " \
+               "--verify".format(
+        kcov_build_dir,
+        kcov_ouput_dir,
+        exclude_region,
+        exclude_pattern
+    )
+
+    subprocess.run(kcov_cmd, shell=True, check=True)
+
+    # Read the coverage reported by kcov.
+    coverage_file = os.path.join(kcov_ouput_dir, 'index.js')
+    with open(coverage_file) as cov_output:
+        coverage = float(re.findall(
+            r'"covered":"(\d+\.\d)"',
+            cov_output.read()
+        )[0])
+
+    # Remove coverage related directories.
+    shutil.rmtree(kcov_ouput_dir, ignore_errors=True)
+    shutil.rmtree(kcov_build_dir, ignore_errors=True)
+
+    return coverage
+
+
+def _get_previous_coverage():
+    """Helper function that returns the last reported coverage."""
+    coverage_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'coverage'
+    )
+
+    # The first and only line of the file contains the coverage.
+    with open(coverage_path) as f:
+        coverage = f.readline()
+    return float(coverage.strip())
+
+def _update_coverage(cov_value):
+    """Updates the coverage in the coverage file."""
+    coverage_path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        'coverage'
+    )
+
+    with open(coverage_path, "w") as f:
+        f.write(str(cov_value))
+
+def test_coverage(profile):
+    current_coverage = _get_current_coverage()
+    previous_coverage = _get_previous_coverage()
+    if previous_coverage < current_coverage:
+        if profile == pytest.profile_ci:
+            # In the CI Profile we expect the coverage to be manually updated.
+            assert False, "Coverage is increased from {} to {}. " \
+                          "Please update the coverage in " \
+                          "tests/coverage.".format(
+                previous_coverage,
+                current_coverage
+            )
+        elif profile == pytest.profile_devel:
+            _update_coverage(current_coverage)
+        else:
+            # This should never happen because pytest should only accept
+            # the valid test profiles specified with `choices` in
+            # `pytest_addoption`.
+            assert False, "Invalid test profile."
+    elif previous_coverage > current_coverage:
+        diff = float(previous_coverage - current_coverage)
+        assert False, "Coverage drops by {:.2f}%. Please add unit tests for" \
+                      "the uncovered lines.".format(diff)
