Add option to add markers for unavailable drivers

This will allow users to run only scenarios that can be run on their
machines.

Initial implementation does validation only for delegated and
docker drivers.

For delegated driver, we assume current inventory is valid if we
find presence of zuul variable. This should protect againt accidental
execution of ansible tasks on localhost.

Author: ssbarnea

README.rst

@@ -48,6 +48,17 @@ executed inside a specific scenario.
 
 All tests are added the ``molecule`` marker.
 
+This plugin also adds a new pytest option named
+``--molecule-unavailable-driver=skip`` which can be used to tell it what to do
+when molecule drivers are not loading. Current default is ``skip`` but you
+can choose other marks like ``xfail`` or empty string if you want to disable
+this functionality.
+
+Current implementation of this feature validates only ``docker`` and
+``delegated`` drivers.  Due to the potential destructive nature of delegated
+driver, scenarios running it will run only if Ansible detects one of the
+following variables as defined: ``zuul``, ``use_for_testing``.
+
 Installation
 ------------
 

pytest_molecule/__init__.py

@@ -11,12 +11,95 @@
 from molecule.config import molecule_drivers
 
 
+def pytest_addoption(parser):
+    group = parser.getgroup("molecule")
+    help_msg = (
+        "What marker to add to molecule scenarios when driver is "
+        "unavailable. (ex: skip, xfail)"
+    )
+    default = "skip"
+    dest = "molecule_unavailable_driver"
+
+    group.addoption(
+        "--molecule-unavailable-driver",
+        action="store",
+        dest=dest,
+        default=default,
+        help=help_msg,
+    )
+
+    parser.addini(dest, help_msg, default=default)
+
+
 def pytest_configure(config):
 
+    config.option.molecule = {}
     for driver in molecule_drivers():
         config.addinivalue_line(
             "markers", "{0}: mark test to run only when {0} is available".format(driver)
         )
+        config.option.molecule[driver] = {"available": True}
+        # TODO(ssbarnea): extend molecule itself to allow it to report usable drivers
+        if driver == "docker":
+            try:
+                import docker
+
+                # validate docker connectivity
+                # Default docker value is 60s but we want to fail faster
+                # With parallel execution 5s proved to give errors.
+                c = docker.from_env(timeout=10, version="auto")
+                if not c.ping():
+                    raise Exception("Failed to ping docker server.")
+
+            except Exception as e:
+                msg = "Molecule {} driver is not available due to: {}.".format(
+                    driver, e
+                )
+                if config.option.molecule_unavailable_driver:
+                    msg += " We will tag scenarios using it with '{}' marker.".format(
+                        config.option.molecule_unavailable_driver
+                    )
+                logging.getLogger().warning(msg)
+                config.option.molecule[driver]["available"] = False
+
+        if driver == "delegated":
+            # To protect ourselves from case where a molecule scenario using
+            # `delegated` is accidentally altering the localhost on a developer
+            # machine, we verify run delegated tests only when ansible `zuul`
+            # or `use_for_testing` vars are defined.
+            cmd = [
+                "ansible",
+                "localhost",
+                "-e",
+                "ansible_connection=local" "-o",
+                "-m",
+                "shell",
+                "-a",
+                "exit {% if zuul is defined or use_for_testing is defined %}0{% else %}1{% endif %}",
+            ]
+            try:
+                p = subprocess.Popen(
+                    cmd,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                    universal_newlines=True,
+                )
+                p.wait()
+                if p.returncode != 0:
+                    raise Exception(
+                        "Error code %s returned by: %s" % (p.returncode, " ".join(cmd))
+                    )
+            except Exception:
+                msg = "Molecule {} driver was not enabled because missing zuul.build variable in current inventory.".format(
+                    driver
+                )
+                if config.option.molecule_unavailable_driver:
+                    msg += " We will tag scenarios using it with '{}' marker.".format(
+                        config.option.molecule_unavailable_driver
+                    )
+                logging.getLogger().warning(msg)
+                config.option.molecule[driver]["available"] = False
+
     config.addinivalue_line("markers", "molecule: mark used by all molecule scenarios")
 
     # validate selinux availability
@@ -65,6 +148,11 @@ def __init__(self, name, parent):
             )
             self.add_marker(p)
         self.add_marker("molecule")
+        if (
+            self.config.option.molecule_unavailable_driver
+            and not self.config.option.molecule[self.molecule_driver]["available"]
+        ):
+            self.add_marker(self.config.option.molecule_unavailable_driver)
 
     def runtest(self):
         folders = self.fspath.dirname.split(os.sep)
@@ -80,7 +168,6 @@ def runtest(self):
             cmd.extend(shlex.split(opts))
 
         print("running: %s (from %s)" % (" ".join(quote(arg) for arg in cmd), cwd))
-
         try:
             # Workaround for STDOUT/STDERR line ordering issue:
             # https://github.com/pytest-dev/pytest/issues/5449

tox.ini

@@ -18,16 +18,17 @@ deps =
     pytest-html>=1.21.0
     selinux>=0.1.5rc1
 commands =
+    ansible-inventory --graph
     pytest --collect-only
     # pytest already needs built wheel in dist/
     python -m pep517.build \
       --source \
       --binary \
       --out-dir {toxinidir}/dist/ {toxinidir}
-    pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} -k foo
+    pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} --molecule-unavailable-driver= -k foo
 setenv =
     ANSIBLE_FORCE_COLOR={env:ANSIBLE_FORCE_COLOR:1}
-    ANSIBLE_INVENTORY={toxinidir}/tests/hosts.ini
+    ANSIBLE_INVENTORY={env:ANSIBLE_INVENTORY:{toxinidir}/../zuul-infra/inventory.yml}
     ANSIBLE_CONFIG={toxinidir}/ansible.cfg
     ANSIBLE_NOCOWS=1
     ANSIBLE_RETRY_FILES_ENABLED=0