Inventory: Remove unnecessary lists around singular host vars #141 (#155)

Author: DouglasHeriot

.gitignore

@@ -2,3 +2,8 @@
 __pycache__/
 *.pyc
 tests/output/*
+venv/
+.vscode/
+
+# https://github.com/ansible/ansible/issues/68499
+# ansible_collections/

.travis.yml

@@ -18,7 +18,7 @@ jobs:
     # Previous release of Netbox, latest release of Ansible
     - name: "Python 3.6 - Netbox 2.6 - Latest PyPi Ansible"
       python: 3.6
-      env: PYTHON_VER=3.6 VERSION=v2.6
+      env: PYTHON_VER=3.6 VERSION=v2.6 INTEGRATION_TESTS=v2.6
       install:
         - cd ..
         # Setup netbox container for integration testing
@@ -27,15 +27,16 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        # setuptools <46.1.3 has issues with ansible-test executables https://github.com/ansible/ansible/issues/68963
+        - pip install -U pip "setuptools>=46.1.3"
         # coverage 5.0 is not compatible with ansible-test https://github.com/ansible/ansible/issues/65907
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography codecov jmespath ansible
+        - pip install pynetbox cryptography codecov jmespath jsondiff ansible
 
-    # Latest release of Netbox and Ansible
     - name: "Python 3.6 - Netbox 2.7 - Latest PyPi Ansible"
       python: 3.6
-      env: PYTHON_VER=3.6 VERSION=v2.7
+      env:
+      - PYTHON_VER=3.6 VERSION=v2.7 INTEGRATION_TESTS=latest
       install:
         - cd ..
         # Setup netbox container for integration testing
@@ -44,16 +45,35 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        - pip install -U pip "setuptools>=46.1.3"
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography codecov jmespath ansible
+        - pip install pynetbox cryptography codecov jmespath jsondiff ansible
+
+    # Stick to python 3.7 instead of 3.8, as ansible-test sanity is not compatible with 3.7
+    # https://github.com/ansible/ansible/issues/67118
+
+    - name: "Python 3.7 - Netbox 2.8 - Latest PyPi Ansible"
+      python: 3.7
+      env:
+      - PYTHON_VER=3.7 VERSION=v2.8 INTEGRATION_TESTS=latest
+      install:
+        - cd ..
+        # Setup netbox container for integration testing
+        - git clone https://github.com/netbox-community/netbox-docker.git
+        - cd netbox-docker
+        - docker-compose pull
+        - docker-compose up -d
+        - cd ..
+        - pip install -U pip "setuptools>=46.1.3"
+        - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
+        - pip install pynetbox cryptography codecov jmespath jsondiff ansible
 
     # Latest development versions of Netbox and Ansible, newest Python
     # This may be broken sometimes by changes in the netbox & ansible projects
     # Failures will be allowed in this build
-    - name: "Python 3.8 - Netbox develop-2.8 - Ansible Devel"
+    - name: "Python 3.8 - Netbox develop branch (snapshot) - Ansible Devel"
       python: 3.8
-      env: PYTHON_VER=3.8 VERSION=develop-2.8
+      env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
       install:
         - cd ..
         # Setup netbox container for integration testing
@@ -62,16 +82,16 @@ jobs:
         - docker-compose pull
         - docker-compose up -d
         - cd ..
-        - pip install -U pip
+        - pip install -U pip "setuptools>=46.1.3"
         - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
-        - pip install pynetbox cryptography jmespath
+        - pip install pynetbox cryptography jmespath jsondiff
         - git clone https://github.com/ansible/ansible.git
         - cd ansible
         - source hacking/env-setup
         - cd  ..
   allow_failures:
     # When testing against dev netbox and dev ansible, allow failures
-    - env: PYTHON_VER=3.8 VERSION=develop-2.8
+    - env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
 
 before_script:
   - mkdir -p ~/ansible_collections/$COLLECTION_NAMESPACE
@@ -80,31 +100,47 @@ before_script:
   - ansible-galaxy collection build .
   - ansible-galaxy collection install $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz -p /home/travis/.ansible/collections
 
+  # Run all further tests from within the installed directory
+  # Required to resolve imports of other collections
+  - cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
+
+  # Set runme.sh execute permissions stripped by ansible-galaxy. Should be fixed in Ansible 2.10
+  # https://github.com/ansible/ansible/issues/68415
+  - chmod +x tests/integration/targets/inventory/runme.sh
+  - chmod +x tests/integration/targets/inventory/compare_inventory_json.py
+
 script:
-  # Perform unit tests on collection from within the installed directory, not the source directory
-  # Required for imports of other collections (ie. ansible.netcommon) to work correctly
-  - (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test units --coverage --python $PYTHON_VER -v)
 
   # Check python syntax
   - black . --check --diff
 
+  # Sanity tests
+  # --requirements - install pip packages as necessary
+  # Skip pep8 as we use black instead
+  # Skip validate-modules until the issues can be resolved #180
+  - ansible-test sanity -v --requirements --python $PYTHON_VER --skip-test pep8 --skip-test validate-modules
+
+  # Unit tests, with code coverage
+  - ansible-test units -v --coverage --python $PYTHON_VER
+
   # Wait for Netbox containers to be running
   - timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:32768)" != "200" ]]; do echo "waiting for Netbox"; sleep 5; done' || false
 
   # Prepare data in Netbox
-  - python tests/integration/netbox-deploy.py
+  - ./tests/integration/netbox-deploy.py
 
   # Run regression and integration tests
-  - ansible-playbook tests/integration/regression-tests.yml -vvvv
-  - ansible-playbook tests/integration/$VERSION/main.yml -vvvv
-  - ansible-inventory -i tests/integration/test-inventory.yml --list
+  # Run the inventory test first, in case any of the other tests modify the data.
+  - ansible-test integration -v --coverage --python $PYTHON_VER inventory
+  - ansible-test integration -v --coverage --python $PYTHON_VER regression
+  - ansible-test integration -v --coverage --python $PYTHON_VER $INTEGRATION_TESTS
 
   # Report code coverage
-  - (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test coverage report)
+  - ansible-test coverage report --all --omit "tests/*,hacking/*" --show-missing
 
 deploy:
   provider: script
   skip_cleanup: true
-  script: ansible-galaxy collection publish $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz --api-key="$GALAXY_API_TOKEN"
+  script: ansible-galaxy collection publish ~/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME/$COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz --api-key="$GALAXY_API_TOKEN"
   on:
     tags: true

CONTRIBUTING.md

@@ -1,5 +1,7 @@
 # Contributing
 
+## Modules
+
 The structure of the Netbox modules attempts to follow the layout of the Netbox API by having a module_util for each application (`dcim, ipam, tenancy, etc`) that inherits from a base module (`NetboxModule - netbox_utils.py`) and then implements the specific endpoints within the correct application module.
 
 e.g. Add logic for adding devices under netbox_dcim.py or ip addresses under netbox_ipam.py
@@ -257,7 +259,7 @@ Copying an existing module that has close to the same options is typically the p
 - Change the author: `Copyright: (c) 2018, Mikhail Yohman (@FragmentedPacket) <mikhail.yohman@gmail.com>`
 - Update the **DOCUMENTATION**/**EXAMPLES**/**RETURN** string with the necessary information
   - Main things are module, descriptions, author, version and the sub options under data
-  - The **RETURN** should return the singluar of the endpoint name (done dynamically, but needs to be documented correctly)
+  - The **RETURN** should return the singular of the endpoint name (done dynamically, but needs to be documented correctly)
 - Update the module_util, module, and endpoint variable for the endpoint
 
   ```python
@@ -296,3 +298,50 @@ Copying an existing module that has close to the same options is typically the p
 - Run `black .` within the base directory for black formatting as it's required for tests to pass
 - Run `ansible-lint integration-tests.yml` it's required for tests to pass
 - Check necessary dependencies defined within `.travis.yml` for now if you're wanting to test locally
+
+
+## Inventory
+
+Integration tests are run by comparing `ansible-inventory --list` against known output, for a few different inventory files with different options.
+
+When the inventory plugin is updated in a way that changes this output (or `netbox-deploy.py` is changed), the test data used for comparison will need to be updated. There is a script `./hacking/update_test_inventories.sh` to do this.
+
+Run from the root project working directory, and make sure you have a clean test instance of Netbox set up with test data loaded into it from `./tests/integration/netbox-deploy.py`
+
+```
+./hacking/update_test_inventories.sh
+```
+
+After running the script, to prevent introducing a regression you'll need to manually read through the diff to verify it looks correct.
+
+
+# Setting up a local dev/test environment
+
+You can see the specific details of what happens in CI in  `.travis.yml`. An overview of what you need to do is:
+
+* Run a test instance of Netbox on port 32768. The `netbox-docker` project makes this easy.
+
+```
+git clone git@github.com:netbox-community/netbox-docker.git
+cd netbox-docker
+export VERSION=snapshot
+docker-compose pull
+docker-compose up -d
+```
+
+* Check out this repo to a directory named `netbox` in a directory tree `ansible_collections/netbox/netbox` - running some `ansible-test` commands will complain if there's no `ansible_collections` in the current directory path, so you'd have to build/install the collection every time you want to run tests.
+
+```
+git clone git@github.com:netbox-community/ansible_modules.git ansible_collections/netbox/netbox
+```
+
+* Recommended - set up a Python virtualenv, in a directory above the collection. If you create a virtualenv inside the `ansible_modules` working directory `ansible-galaxy collection build` will include it as part of the build (until Ansible 2.10 where `build_ignore` is supported)
+
+```
+cd ../../..
+python3 -m venv venv
+source venv/bin/activate
+cd ansible_collections/netbox/netbox
+```
+
+* Install required python packages - see `.travis.yml` for the latest `pip install` list in different environments.

ansible.cfg

@@ -1,2 +1,5 @@
+[defaults]
+force_valid_group_names = always
+
 [inventory]
-enable_plugins =  netbox.netbox.nb_inventory, auto, host_list, yaml, ini, toml, script
+enable_plugins = netbox.netbox.nb_inventory, auto, host_list, yaml, ini, toml, script

galaxy.yml

@@ -46,8 +46,12 @@ tags:
 # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
 # range specifiers can be set and are separated by ','
 dependencies:
+  # Required for ipaddress python library
   ansible.netcommon: "*"
 
+  # Required for json_query used in lookup plugin integration tests
+  community.general: "*"
+
 # The URL of the originating SCM repository
 repository: https://github.com/netbox-community/ansible_modules
 
@@ -59,3 +63,13 @@ homepage:
 
 # The URL to the collection issue tracker
 issues: https://github.com/netbox-community/ansible_modules/issues
+
+# Files to ignore when building collection. Only supported when building with Ansible 2.10
+# https://github.com/ansible/ansible/issues/67543
+# https://docs.ansible.com/ansible/devel/dev_guide/developing_collections.html#ignoring-files-and-folders
+build_ignore:
+- venv
+- ansible_collections
+- tests/output
+- .pytest_cache
+- .vscode

hacking/black.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# Usage: ./hacking/black.sh
+
+# Run black to reformat all python code
+black . --exclude "ansible_collections|venv"

hacking/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Usage: ./hacking/build.sh
+
+# galaxy.yml build_ignore is not implemented in Ansible 2.9, so we have to delete files we don't want
+
+# Remove previous test installs
+rm -r ansible_collections
+
+# Remove old built versions
+rm netbox-netbox-*.tar.gz
+
+# Remove test output
+rm -r tests/output
+rm -r .pytest_cache
+
+ansible-galaxy collection build --force --verbose .
+
+# To verify what files have been included in the build:
+# tar --gunzip --list --file netbox-netbox-*.tar.gz

scripts/get_inventory_query_parameters.py renamed to hacking/get_inventory_query_parameters.py

@@ -1,10 +1,15 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 # Usage: ./get_inventory_query_filters.py https://netbox/api/docs/?format=openapi
 
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
 import sys
 import json
 import urllib.request
+from ansible.module_utils.urls import open_url
 
 
 def get_parameters(data, path):
@@ -23,7 +28,7 @@ def get_parameters(data, path):
 
 print("Getting from %s" % url, file=sys.stderr)
 
-response = urllib.request.urlopen(url)
+response = open_url(url)
 data = json.load(response)
 
 print("ALLOWED_DEVICE_QUERY_PARAMETERS = (")

hacking/local-test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Usage: ./hacking/local-test.sh
+
+# Run build, which will remove previously installed versions
+./hacking/build.sh
+
+# Install new built version
+ansible-galaxy collection install netbox-netbox-*.tar.gz -p .
+
+# You can now cd into the installed version and run tests
+(cd ansible_collections/netbox/netbox/ && ansible-test units -v --python 3.6)

hacking/update_test_inventories.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Stop on failure - if unit tests fail the inventory will not be updated
+set -e
+
+# Usage overview (see Contributing.md for more detail):
+# - Run netbox with the netbox-docker project
+# - Fill with test data, run ./tests/integration/netbox-deploy.py
+# - Run ./hacking/update_test_inventories.sh
+# - Manually verify the diff of new JSON is correct, to prevent introducing a regression.
+
+# Install locally
+export ANSIBLE_COLLECTIONS_PATHS=.
+export OUTPUT_INVENTORY_JSON=tests/integration/targets/inventory/files
+
+# Clean and install the built collection
+./hacking/local-test.sh
+
+# Run the same script used by integration tests, but save the results
+./tests/integration/targets/inventory/runme.sh