Skip to content

Commit 6b42638

Browse files
Inventory: Remove unnecessary lists around singular host vars #141 (#155)
1 parent d035cda commit 6b42638

File tree

175 files changed

+3047
-9621
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

175 files changed

+3047
-9621
lines changed

.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,8 @@
22
__pycache__/
33
*.pyc
44
tests/output/*
5+
venv/
6+
.vscode/
7+
8+
# https://github.com/ansible/ansible/issues/68499
9+
# ansible_collections/

.travis.yml

Lines changed: 57 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
# Previous release of Netbox, latest release of Ansible
1919
- name: "Python 3.6 - Netbox 2.6 - Latest PyPi Ansible"
2020
python: 3.6
21-
env: PYTHON_VER=3.6 VERSION=v2.6
21+
env: PYTHON_VER=3.6 VERSION=v2.6 INTEGRATION_TESTS=v2.6
2222
install:
2323
- cd ..
2424
# Setup netbox container for integration testing
@@ -27,15 +27,16 @@ jobs:
2727
- docker-compose pull
2828
- docker-compose up -d
2929
- cd ..
30-
- pip install -U pip
30+
# setuptools <46.1.3 has issues with ansible-test executables https://github.com/ansible/ansible/issues/68963
31+
- pip install -U pip "setuptools>=46.1.3"
3132
# coverage 5.0 is not compatible with ansible-test https://github.com/ansible/ansible/issues/65907
3233
- pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
33-
- pip install pynetbox cryptography codecov jmespath ansible
34+
- pip install pynetbox cryptography codecov jmespath jsondiff ansible
3435

35-
# Latest release of Netbox and Ansible
3636
- name: "Python 3.6 - Netbox 2.7 - Latest PyPi Ansible"
3737
python: 3.6
38-
env: PYTHON_VER=3.6 VERSION=v2.7
38+
env:
39+
- PYTHON_VER=3.6 VERSION=v2.7 INTEGRATION_TESTS=latest
3940
install:
4041
- cd ..
4142
# Setup netbox container for integration testing
@@ -44,16 +45,35 @@ jobs:
4445
- docker-compose pull
4546
- docker-compose up -d
4647
- cd ..
47-
- pip install -U pip
48+
- pip install -U pip "setuptools>=46.1.3"
4849
- pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
49-
- pip install pynetbox cryptography codecov jmespath ansible
50+
- pip install pynetbox cryptography codecov jmespath jsondiff ansible
51+
52+
# Stick to python 3.7 instead of 3.8, as ansible-test sanity is not compatible with 3.7
53+
# https://github.com/ansible/ansible/issues/67118
54+
55+
- name: "Python 3.7 - Netbox 2.8 - Latest PyPi Ansible"
56+
python: 3.7
57+
env:
58+
- PYTHON_VER=3.7 VERSION=v2.8 INTEGRATION_TESTS=latest
59+
install:
60+
- cd ..
61+
# Setup netbox container for integration testing
62+
- git clone https://github.com/netbox-community/netbox-docker.git
63+
- cd netbox-docker
64+
- docker-compose pull
65+
- docker-compose up -d
66+
- cd ..
67+
- pip install -U pip "setuptools>=46.1.3"
68+
- pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
69+
- pip install pynetbox cryptography codecov jmespath jsondiff ansible
5070

5171
# Latest development versions of Netbox and Ansible, newest Python
5272
# This may be broken sometimes by changes in the netbox & ansible projects
5373
# Failures will be allowed in this build
54-
- name: "Python 3.8 - Netbox develop-2.8 - Ansible Devel"
74+
- name: "Python 3.8 - Netbox develop branch (snapshot) - Ansible Devel"
5575
python: 3.8
56-
env: PYTHON_VER=3.8 VERSION=develop-2.8
76+
env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
5777
install:
5878
- cd ..
5979
# Setup netbox container for integration testing
@@ -62,16 +82,16 @@ jobs:
6282
- docker-compose pull
6383
- docker-compose up -d
6484
- cd ..
65-
- pip install -U pip
85+
- pip install -U pip "setuptools>=46.1.3"
6686
- pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
67-
- pip install pynetbox cryptography jmespath
87+
- pip install pynetbox cryptography jmespath jsondiff
6888
- git clone https://github.com/ansible/ansible.git
6989
- cd ansible
7090
- source hacking/env-setup
7191
- cd ..
7292
allow_failures:
7393
# When testing against dev netbox and dev ansible, allow failures
74-
- env: PYTHON_VER=3.8 VERSION=develop-2.8
94+
- env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
7595

7696
before_script:
7797
- mkdir -p ~/ansible_collections/$COLLECTION_NAMESPACE
@@ -80,31 +100,47 @@ before_script:
80100
- ansible-galaxy collection build .
81101
- ansible-galaxy collection install $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz -p /home/travis/.ansible/collections
82102

103+
# Run all further tests from within the installed directory
104+
# Required to resolve imports of other collections
105+
- cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
106+
107+
# Set runme.sh execute permissions stripped by ansible-galaxy. Should be fixed in Ansible 2.10
108+
# https://github.com/ansible/ansible/issues/68415
109+
- chmod +x tests/integration/targets/inventory/runme.sh
110+
- chmod +x tests/integration/targets/inventory/compare_inventory_json.py
111+
83112
script:
84-
# Perform unit tests on collection from within the installed directory, not the source directory
85-
# Required for imports of other collections (ie. ansible.netcommon) to work correctly
86-
- (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test units --coverage --python $PYTHON_VER -v)
87113

88114
# Check python syntax
89115
- black . --check --diff
90116

117+
# Sanity tests
118+
# --requirements - install pip packages as necessary
119+
# Skip pep8 as we use black instead
120+
# Skip validate-modules until the issues can be resolved #180
121+
- ansible-test sanity -v --requirements --python $PYTHON_VER --skip-test pep8 --skip-test validate-modules
122+
123+
# Unit tests, with code coverage
124+
- ansible-test units -v --coverage --python $PYTHON_VER
125+
91126
# Wait for Netbox containers to be running
92127
- timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:32768)" != "200" ]]; do echo "waiting for Netbox"; sleep 5; done' || false
93128

94129
# Prepare data in Netbox
95-
- python tests/integration/netbox-deploy.py
130+
- ./tests/integration/netbox-deploy.py
96131

97132
# Run regression and integration tests
98-
- ansible-playbook tests/integration/regression-tests.yml -vvvv
99-
- ansible-playbook tests/integration/$VERSION/main.yml -vvvv
100-
- ansible-inventory -i tests/integration/test-inventory.yml --list
133+
# Run the inventory test first, in case any of the other tests modify the data.
134+
- ansible-test integration -v --coverage --python $PYTHON_VER inventory
135+
- ansible-test integration -v --coverage --python $PYTHON_VER regression
136+
- ansible-test integration -v --coverage --python $PYTHON_VER $INTEGRATION_TESTS
101137

102138
# Report code coverage
103-
- (cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME && ansible-test coverage report)
139+
- ansible-test coverage report --all --omit "tests/*,hacking/*" --show-missing
104140

105141
deploy:
106142
provider: script
107143
skip_cleanup: true
108-
script: ansible-galaxy collection publish $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz --api-key="$GALAXY_API_TOKEN"
144+
script: ansible-galaxy collection publish ~/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME/$COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz --api-key="$GALAXY_API_TOKEN"
109145
on:
110146
tags: true

CONTRIBUTING.md

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Contributing
22

3+
## Modules
4+
35
The structure of the Netbox modules attempts to follow the layout of the Netbox API by having a module_util for each application (`dcim, ipam, tenancy, etc`) that inherits from a base module (`NetboxModule - netbox_utils.py`) and then implements the specific endpoints within the correct application module.
46

57
e.g. Add logic for adding devices under netbox_dcim.py or ip addresses under netbox_ipam.py
@@ -257,7 +259,7 @@ Copying an existing module that has close to the same options is typically the p
257259
- Change the author: `Copyright: (c) 2018, Mikhail Yohman (@FragmentedPacket) <mikhail.yohman@gmail.com>`
258260
- Update the **DOCUMENTATION**/**EXAMPLES**/**RETURN** string with the necessary information
259261
- Main things are module, descriptions, author, version and the sub options under data
260-
- The **RETURN** should return the singluar of the endpoint name (done dynamically, but needs to be documented correctly)
262+
- The **RETURN** should return the singular of the endpoint name (done dynamically, but needs to be documented correctly)
261263
- Update the module_util, module, and endpoint variable for the endpoint
262264
263265
```python
@@ -296,3 +298,50 @@ Copying an existing module that has close to the same options is typically the p
296298
- Run `black .` within the base directory for black formatting as it's required for tests to pass
297299
- Run `ansible-lint integration-tests.yml` it's required for tests to pass
298300
- Check necessary dependencies defined within `.travis.yml` for now if you're wanting to test locally
301+
302+
303+
## Inventory
304+
305+
Integration tests are run by comparing `ansible-inventory --list` against known output, for a few different inventory files with different options.
306+
307+
When the inventory plugin is updated in a way that changes this output (or `netbox-deploy.py` is changed), the test data used for comparison will need to be updated. There is a script `./hacking/update_test_inventories.sh` to do this.
308+
309+
Run from the root project working directory, and make sure you have a clean test instance of Netbox set up with test data loaded into it from `./tests/integration/netbox-deploy.py`
310+
311+
```
312+
./hacking/update_test_inventories.sh
313+
```
314+
315+
After running the script, to prevent introducing a regression you'll need to manually read through the diff to verify it looks correct.
316+
317+
318+
# Setting up a local dev/test environment
319+
320+
You can see the specific details of what happens in CI in `.travis.yml`. An overview of what you need to do is:
321+
322+
* Run a test instance of Netbox on port 32768. The `netbox-docker` project makes this easy.
323+
324+
```
325+
git clone git@github.com:netbox-community/netbox-docker.git
326+
cd netbox-docker
327+
export VERSION=snapshot
328+
docker-compose pull
329+
docker-compose up -d
330+
```
331+
332+
* Check out this repo to a directory named `netbox` in a directory tree `ansible_collections/netbox/netbox` - running some `ansible-test` commands will complain if there's no `ansible_collections` in the current directory path, so you'd have to build/install the collection every time you want to run tests.
333+
334+
```
335+
git clone git@github.com:netbox-community/ansible_modules.git ansible_collections/netbox/netbox
336+
```
337+
338+
* Recommended - set up a Python virtualenv, in a directory above the collection. If you create a virtualenv inside the `ansible_modules` working directory `ansible-galaxy collection build` will include it as part of the build (until Ansible 2.10 where `build_ignore` is supported)
339+
340+
```
341+
cd ../../..
342+
python3 -m venv venv
343+
source venv/bin/activate
344+
cd ansible_collections/netbox/netbox
345+
```
346+
347+
* Install required python packages - see `.travis.yml` for the latest `pip install` list in different environments.

ansible.cfg

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
1+
[defaults]
2+
force_valid_group_names = always
3+
14
[inventory]
2-
enable_plugins = netbox.netbox.nb_inventory, auto, host_list, yaml, ini, toml, script
5+
enable_plugins = netbox.netbox.nb_inventory, auto, host_list, yaml, ini, toml, script

galaxy.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,12 @@ tags:
4646
# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
4747
# range specifiers can be set and are separated by ','
4848
dependencies:
49+
# Required for ipaddress python library
4950
ansible.netcommon: "*"
5051

52+
# Required for json_query used in lookup plugin integration tests
53+
community.general: "*"
54+
5155
# The URL of the originating SCM repository
5256
repository: https://github.com/netbox-community/ansible_modules
5357

@@ -59,3 +63,13 @@ homepage:
5963

6064
# The URL to the collection issue tracker
6165
issues: https://github.com/netbox-community/ansible_modules/issues
66+
67+
# Files to ignore when building collection. Only supported when building with Ansible 2.10
68+
# https://github.com/ansible/ansible/issues/67543
69+
# https://docs.ansible.com/ansible/devel/dev_guide/developing_collections.html#ignoring-files-and-folders
70+
build_ignore:
71+
- venv
72+
- ansible_collections
73+
- tests/output
74+
- .pytest_cache
75+
- .vscode

hacking/black.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage: ./hacking/black.sh
4+
5+
# Run black to reformat all python code
6+
black . --exclude "ansible_collections|venv"

hacking/build.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage: ./hacking/build.sh
4+
5+
# galaxy.yml build_ignore is not implemented in Ansible 2.9, so we have to delete files we don't want
6+
7+
# Remove previous test installs
8+
rm -r ansible_collections
9+
10+
# Remove old built versions
11+
rm netbox-netbox-*.tar.gz
12+
13+
# Remove test output
14+
rm -r tests/output
15+
rm -r .pytest_cache
16+
17+
ansible-galaxy collection build --force --verbose .
18+
19+
# To verify what files have been included in the build:
20+
# tar --gunzip --list --file netbox-netbox-*.tar.gz

scripts/get_inventory_query_parameters.py renamed to hacking/get_inventory_query_parameters.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
1-
#!/usr/bin/env python3
1+
#!/usr/bin/env python
22

33
# Usage: ./get_inventory_query_filters.py https://netbox/api/docs/?format=openapi
44

5+
from __future__ import absolute_import, division, print_function
6+
7+
__metaclass__ = type
8+
59
import sys
610
import json
711
import urllib.request
12+
from ansible.module_utils.urls import open_url
813

914

1015
def get_parameters(data, path):
@@ -23,7 +28,7 @@ def get_parameters(data, path):
2328

2429
print("Getting from %s" % url, file=sys.stderr)
2530

26-
response = urllib.request.urlopen(url)
31+
response = open_url(url)
2732
data = json.load(response)
2833

2934
print("ALLOWED_DEVICE_QUERY_PARAMETERS = (")

hacking/local-test.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
3+
# Usage: ./hacking/local-test.sh
4+
5+
# Run build, which will remove previously installed versions
6+
./hacking/build.sh
7+
8+
# Install new built version
9+
ansible-galaxy collection install netbox-netbox-*.tar.gz -p .
10+
11+
# You can now cd into the installed version and run tests
12+
(cd ansible_collections/netbox/netbox/ && ansible-test units -v --python 3.6)

hacking/update_test_inventories.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
# Stop on failure - if unit tests fail the inventory will not be updated
4+
set -e
5+
6+
# Usage overview (see Contributing.md for more detail):
7+
# - Run netbox with the netbox-docker project
8+
# - Fill with test data, run ./tests/integration/netbox-deploy.py
9+
# - Run ./hacking/update_test_inventories.sh
10+
# - Manually verify the diff of new JSON is correct, to prevent introducing a regression.
11+
12+
# Install locally
13+
export ANSIBLE_COLLECTIONS_PATHS=.
14+
export OUTPUT_INVENTORY_JSON=tests/integration/targets/inventory/files
15+
16+
# Clean and install the built collection
17+
./hacking/local-test.sh
18+
19+
# Run the same script used by integration tests, but save the results
20+
./tests/integration/targets/inventory/runme.sh

0 commit comments

Comments
 (0)