CP-46060/feature/py3: Support running tests locally before pushing to GitHub

.github/workflows/main.yml

@@ -15,6 +15,8 @@ jobs:
   python-test:
     name: Python tests
     runs-on: ubuntu-22.04
+    env:
+      PYTHONDEVMODE: yes
     strategy:
       fail-fast: false
       matrix:
@@ -45,40 +47,16 @@ jobs:
         env:
           SKIP: no-commit-to-branch
 
-      - name: Install dependencies only needed for python 2
-        if: ${{ matrix.python-version == '2.7' }}
-        run: pip install enum
-
-      - name: Install dependencies only needed for python 3
-        if: ${{ matrix.python-version != '2.7' }}
-        run: pip install pandas pytype toml wrapt
-
-      - name: Install common dependencies for Python ${{matrix.python-version}}
-        run: pip install future mock pytest-coverage pytest-mock
-
       - name: Run Pytest for python 2 and get code coverage for Codecov
         if: ${{ matrix.python-version == '2.7' }}
         run: >
+          pip install enum future mock pytest-coverage pytest-mock &&
           pytest
           --cov=ocaml/xcp-rrdd
           scripts/ ocaml/xcp-rrdd -vv -rA
           --junitxml=.git/pytest${{matrix.python-version}}.xml
           --cov-report term-missing
           --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
-
-      - name: Run Pytest for python 3 and get code coverage for Codecov
-        if: ${{ matrix.python-version != '2.7' }}
-        run: >
-          pytest
-          --cov=python3/
-          python3/tests -vv -rA
-          --junitxml=.git/pytest${{matrix.python-version}}.xml
-          --cov-report term-missing
-          --cov-report xml:.git/coverage${{matrix.python-version}}.xml
-        env:
-          PYTHONDEVMODE: yes
 
       - name: Upload Python ${{matrix.python-version}} coverage report to Codecov
         uses: codecov/codecov-action@v3
@@ -101,7 +79,7 @@ jobs:
 
       - name: Run pytype checks
         if: ${{ matrix.python-version != '2.7' }}
-        run: ./pytype_reporter.py
+        run: pip install pandas pytype toml && ./pytype_reporter.py
         env:
           PR_NUMBER: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -14,6 +14,8 @@
 # pre-commit run -av --hook-stage pre-push
 #
 default_stages: [commit, push]
+default_language_version:
+    python: python3.11
 repos:
 # Recommendation for a minimal git pre-commit hook:
 # https://github.com/pre-commit/pre-commit-hooks/blob/main/README.md:
@@ -29,6 +31,95 @@ repos:
     -   id: check-executables-have-shebangs
         exclude: ocaml
 
+
+# Improve Python formatting incrementally:
+# https://dev.to/akaihola/improving-python-code-incrementally-3f7a
+#
+# darker checks if staged python changes are formatted according using
+# the PEP8-aligned black formatter. It also checks if the imports are sorted.
+#
+# It is a good idea to run this before committing, and it is also run in the
+# GitHub Workflow.
+#
+# Note: darker only checks the changes in files ending in .py!
+# Python scripts that don't end in .py should be renamed to have the .py extension
+# when moving them to python3/bin.
+# (remove the .py extension in the Makefile when installing the file)
+#
+-   repo: https://github.com/akaihola/darker
+    rev: 1.7.3
+    hooks:
+    -   id: darker
+        files: python3/
+        name: check changes in Python3 tree using darker and isort
+        args: [--diff, --skip-string-normalization, --isort, -tpy36]
+        additional_dependencies: [isort]
+
+#
+# Run pytest and diff-cover to check that the new /python3 test suite in passes.
+# This hook uses a local venv containing the required dependencies. When adding
+# new dependencies, they should be added to the additional_dependencies below.
+#
+-   repo: local
+    hooks:
+    -   id: pytest
+        files: python3/
+        name: check that the Python3 test suite in passes
+        entry: env PYTHONDEVMODE=yes sh -c 'python3 -m pytest -vv &&
+            diff-cover --ignore-whitespace --compare-branch=origin/feature/py3
+            --show-uncovered --html-report .git/coverage-diff.html
+            --fail-under 50 .git/coverage3.11.xml'
+        require_serial: true
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies:
+        - coverage
+        - diff-cover
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest-coverage
+        - pytest-mock
+        - mock
+        - wrapt
+        - XenAPI
+
+
+-   repo: https://github.com/RobertCraigie/pyright-python
+    rev: v1.1.361
+    hooks:
+    -   id: pyright
+        name: check that python3 tree passes pyright/VSCode check
+        files: python3/
+        additional_dependencies:
+        - mock
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pytest
+        - pyudev
+        - XenAPI
+
+
+# Check that pylint passes for the changes in new /python3 code.
+-   repo: local
+    hooks:
+    -   id: pylint
+        files: python3/
+        stages: [push]
+        name: check that changes to python3 tree pass pylint
+        entry: diff-quality --violations=pylint
+            --ignore-whitespace --compare-branch=origin/feature/py3
+        pass_filenames: false
+        language: python
+        types: [python]
+        additional_dependencies: [diff-cover, pylint, pytest]
+
+
+# pre-push hook (it only runs if you install pre-commit as a pre-push hook):
+# It can be manually tested using: `pre-commit run -av --hook-stage push`
 # Recommendation for a minimal git pre-push hook:
 # While using pre-commit yields great results, it
 # is "not fast". Therefore only run it pre-push,
@@ -53,4 +144,12 @@ repos:
         # developers have such version installed, it can be configured here:
         # language_version: python3.11
         require_serial: true
-        additional_dependencies: [pandas, pytype]
+        additional_dependencies:
+        - future
+        - opentelemetry-api
+        - opentelemetry-exporter-zipkin-json
+        - opentelemetry-sdk
+        - pandas
+        - pytest
+        - pytype
+        files: python3/

pyproject.toml

@@ -27,6 +27,25 @@ repository = "https://github.com/xapi-project/xen-api"
 [tool.black]
 line-length = 88
 
+
+# -----------------------------------------------------------------------------
+# Coverage.py - https://coverage.readthedocs.io/en/coverage-5.5/config.html
+# -----------------------------------------------------------------------------
+
+[tool.coverage.report]
+# fail_under:    minimum code coverage percentage
+# skip_covered:  if fully covered files should be reported
+# exclude_lines: lines to not require to be covered
+fail_under = 50
+skip_covered = true
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+    "pass",
+    "raise NbdDeviceNotFound",  # python3/libexec/usb_scan.py
+]
+
+
 [tool.isort]
 line_length = 88
 py_version = 27
@@ -40,6 +59,8 @@ disable = [
     "missing-module-docstring",
     "missing-class-docstring",
     "consider-using-f-string",
+    "too-few-public-methods",
+    "too-many-public-methods",
     "too-many-branches",
     "too-many-arguments",
     "broad-exception-caught",
@@ -82,54 +103,94 @@ disallow_any_unimported = true
 disallow_subclassing_any = true
 
 
+# -----------------------------------------------------------------------------
+# Pyright static analysis and Pylance IDE language plugin of VSCode
+# https://microsoft.github.io/pyright/#/configuration
+# -----------------------------------------------------------------------------
+
+[tool.pyright]
+# include:          directories to include in checking
+# strict:           paths for which strict checking works
+# typeCheckingMode: set the standard type checking mode
+include              = ["python3", "ocaml/xcp-rrdd"]
+strict               = ["python3/tests/observer"]
+pythonPlatform       = "Linux"
+typeCheckingMode     = "standard"
+reportMissingImports = false
+pythonVersion        = "3.6"
+exclude              = [
+    "ocaml/xcp-rrdd/scripts/rrdd/rrdd.py",
+    "ocaml/xcp-rrdd/scripts/rrdd/rrdd-example.py",
+    "python3/packages/observer.py",
+]
+
+
+# -----------------------------------------------------------------------------
+# pytest: helps you write better programs - https://docs.pytest.org
+# -----------------------------------------------------------------------------
+
+[tool.pytest.ini_options]
+# -----------------------------------------------------------------------------
+# Options to enable for pytest by default:
+# -v                           show what happens
+# -rA                          show summary after running tests
+# --cov=python3                measure coverage of the python3 directory
+# --cov-fail-under             minimum coverage percentage
+# --cov-report=term-missing    show missing lines in the coverage report
+# --cov-report=html:<filename> generate an HTML coverage report(for viewing)
+# --cov-report=xml:<filename>  generate an XML coverage report(for upload)
+# -----------------------------------------------------------------------------
+addopts = """
+-v -rA --cov=python3 --cov=scripts --cov-fail-under=50
+--cov-report=html:.git/coverage --cov-report=term-missing
+--cov-report=xml:.git/coverage3.11.xml
+"""
+
+# -----------------------------------------------------------------------------
+# Other pytest config options:
+# log_cli:          show logger messages
+# log_cli_level:    log level to show
+# python_files:     pattern for test files
+# python_functions: pattern for test functions
+# testpaths:        directories to search for tests
+# minversion:       this config requires pytest>=7 to configure pythonpath
+# pythonpath:       path to stub files and typing stubs for tests
+# xfail_strict:     require to remove pytext.xfail marker when test is fixed
+# required_plugins: require that these plugins are installed before testing
+# -----------------------------------------------------------------------------
+testpaths = ["python3", "scripts", "ocaml/xcp-rrdd"]
+required_plugins = ["pytest-cov", "pytest-mock"]
+log_cli_level = "INFO"
+log_cli = true
+minversion = "7.0"
+pythonpath = "python3/stubs"
+python_files = ["test_*.py", "it_*.py"]
+python_functions = ["test_", "it_", "when_"]
+xfail_strict = true
+
+
 [tool.pytype_reporter]
-default_branch = "master"
+default_branch = "feature/py3"
 discard_messages_matching = [
     "Couldn't import pyi for 'xml.dom.minidom'",
     "No attribute '.*' on RRDContentHandler",
     "No attribute 'group' on None",
     "No Node.TEXT_NODE in module xml.dom.minidom, referenced from 'xml.dom.expatbuilder'"
 ]
-expected_to_fail = [
-    # Need 2to3 -w <file> and maybe a few other minor updates:
-    "scripts/hatests",
-    "scripts/backup-sr-metadata.py",
-    "scripts/restore-sr-metadata.py",
-    # SSLSocket.send() only accepts bytes, not unicode string as argument:
-    "scripts/examples/python/exportimport.py",
-    # Other fixes needed:
-    "scripts/examples/python/mini-xenrt.py",
-    "scripts/examples/python/XenAPI/XenAPI.py",
-    "scripts/examples/python/monitor-unwanted-domains.py",
-    "scripts/examples/python/shell.py",
-    "scripts/examples/smapiv2.py",
-    "scripts/static-vdis",
-    "scripts/plugins/extauth-hook-AD.py",
-]
+expected_to_fail = []
 
 
+# -----------------------------------------------------------------------------
+# pytype: Google's static type analyzer - https://google.github.io/pytype/
+# -----------------------------------------------------------------------------
+
 [tool.pytype]
 inputs = [
-    "scripts/static-vdis",
-    "scripts/Makefile",
-    "scripts/generate-iscsi-iqn",
-    "scripts/hatests",
-    "scripts/host-display",
-    "scripts/mail-alarm",
-    "scripts/print-custom-templates",
-    "scripts/probe-device-for-file",
-    "scripts/xe-reset-networking",
-    "scripts/xe-scsi-dev-map",
-    "scripts/examples/python",
-    "scripts/yum-plugins",
-    "scripts/*.py",
-    "python3/packages/*.py",
-
-    # Python 3
     "python3/bin/hfx_filename",
     "python3/bin/perfmon",
     "python3/bin/*.py",
     "python3/libexec/*.py",
+    "python3/packages/*.py",
 
     # To be added later,
     # when converted to Python3-compatible syntax:
@@ -138,12 +199,8 @@ inputs = [
     # "ocaml/xapi-storage/python",
 ]
 disable = [
-    "import-error",  # xenfsimage, xcp.bootloader. xcp.cmd
-    "ignored-abstractmethod",
+    "import-error",      # opentelemetry packages
     "ignored-metaclass",
-    # https://github.com/google/pytype/issues/1130,
-    # https://github.com/google/pytype/issues/1485:
-    "pyi-error",
 ]
 platform = "linux"
-pythonpath = "scripts/examples/python:.:scripts:scripts/plugins:scripts/examples"
+pythonpath = "python3/stubs:."

python3/bin/hfx_filename

@@ -14,8 +14,14 @@
 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+# pylint: disable=redefined-outer-name
+# pyright: reportFunctionMemberAccess=false
+# pyright: reportOptionalMemberAccess=false, reportAttributeAccessIssue=false
 
-import sys, socket, urllib.request, XenAPI
+import sys
+import socket
+
+import XenAPI
 
 db_url = "/remote_db_access"
 

python3/libexec/nbd_client_manager.py

@@ -136,7 +136,7 @@ def _find_unused_nbd_device():
             return nbd_device
 
     # If there are 1000 nbd devices (unlikely) and all are connected
-    raise NbdDeviceNotFound(nbd_device)
+    raise NbdDeviceNotFound(nbd_device)  # pyright:ignore[reportPossiblyUnboundVariable]
 
 def _wait_for_nbd_device(nbd_device, connected):
     deadline = datetime.now() + timedelta(minutes=MAX_DEVICE_WAIT_MINUTES)

python3/libexec/usb_reset.py

@@ -132,7 +132,7 @@ def load_device_ids(device):
         # ignore and continue
         log.warning("Failed to remove device ids: {}".format(str(e)))
 
-    return uid, gid
+    return uid, gid  # pyright: ignore[reportPossiblyUnboundVariable] # pragma: no cover
 
 
 # throw IOError, ValueError