Skip to content

Commit 3524cc3

Browse files
Huge PR that gets collection working with nb2.9 and updates testing (#345)
1 parent c6dd432 commit 3524cc3

File tree

114 files changed

+11268
-141
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

114 files changed

+11268
-141
lines changed

.travis.yml

Lines changed: 14 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@ env:
1515

1616
jobs:
1717
include:
18-
- name: "Python 3.6 - Netbox 2.7 - Latest PyPi Ansible"
18+
- name: "Python 3.6 - Netbox 2.8 - Latest PyPi Ansible"
1919
python: 3.6
2020
env:
21-
- PYTHON_VER=3.6 VERSION=v2.7 INTEGRATION_TESTS=latest
21+
- PYTHON_VER=3.6 VERSION=v2.8 INTEGRATION_TESTS=v2.8
2222
install:
2323
- cd ..
2424
# Setup netbox container for integration testing
@@ -37,10 +37,10 @@ jobs:
3737
# Stick to python 3.7 instead of 3.8, as ansible-test sanity is not compatible with 3.7
3838
# https://github.com/ansible/ansible/issues/67118
3939

40-
- name: "Python 3.7 - Netbox 2.8 - Latest PyPi Ansible"
40+
- name: "Python 3.7 - Netbox 2.9 - Latest PyPi Ansible"
4141
python: 3.7
4242
env:
43-
- PYTHON_VER=3.7 VERSION=v2.8 INTEGRATION_TESTS=latest
43+
- PYTHON_VER=3.7 VERSION=v2.9 INTEGRATION_TESTS=latest
4444
install:
4545
- cd ..
4646
# Setup netbox container for integration testing
@@ -58,9 +58,9 @@ jobs:
5858
# Latest development versions of Netbox and Ansible, newest Python
5959
# This may be broken sometimes by changes in the netbox & ansible projects
6060
# Failures will be allowed in this build
61-
- name: "Python 3.8 - Netbox develop branch (snapshot) - Ansible Devel"
62-
python: 3.8
63-
env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
61+
- name: "Python 3.6 - Netbox develop branch (snapshot) - Ansible Develop"
62+
python: 3.6
63+
env: PYTHON_VER=3.6 VERSION=snapshot INTEGRATION_TESTS=latest
6464
install:
6565
- cd ..
6666
# Setup netbox container for integration testing
@@ -79,30 +79,9 @@ jobs:
7979
- source hacking/env-setup
8080
- cd ..
8181

82-
# Commenting out, but keeping for next PR to work on sanity issues
83-
#- name: "Ansible 2.10 Preparation"
84-
# python: 3.8
85-
# env: INTEGRATION_TESTS=latest
86-
# install:
87-
# - pip install -U pip "setuptools>=46.1.3"
88-
# - pip install pytest==4.6.5 pytest-mock pytest-xdist jinja2 PyYAML black==19.10b0 "coverage<5"
89-
# - pip install pynetbox cryptography jmespath jsondiff ansible-base
90-
# before_script:
91-
# - ls
92-
# - cd ..
93-
# - mkdir -p ~/ansible_collections/$COLLECTION_NAMESPACE
94-
# - cp -R ansible_modules ~/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
95-
# - cd ~/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
96-
# - ansible-galaxy collection build .
97-
# - ansible-galaxy collection install $COLLECTION_NAMESPACE-$COLLECTION_NAME-$COLLECTION_VERSION.tar.gz -p /home/travis/.ansible/collections
98-
# Run all further tests from within the installed directory
99-
# Required to resolve imports of other collections
100-
#- cd /home/travis/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE/$COLLECTION_NAME
101-
#script:
102-
# - ansible-test sanity --docker -v --exclude tests/ --exclude docs/ --skip-test pep8
10382
allow_failures:
10483
# When testing against dev netbox and dev ansible, allow failures
105-
- env: PYTHON_VER=3.8 VERSION=snapshot INTEGRATION_TESTS=latest
84+
- env: PYTHON_VER=3.6 VERSION=snapshot INTEGRATION_TESTS=latest
10685

10786
before_script:
10887
- mkdir -p ~/ansible_collections/$COLLECTION_NAMESPACE
@@ -117,13 +96,13 @@ before_script:
11796

11897
# Set runme.sh execute permissions stripped by ansible-galaxy. Should be fixed in Ansible 2.10
11998
# https://github.com/ansible/ansible/issues/68415
120-
- chmod +x tests/integration/targets/inventory/runme.sh
121-
- chmod +x tests/integration/targets/inventory/compare_inventory_json.py
99+
- chmod +x tests/integration/targets/inventory-$INTEGRATION_TESTS/runme.sh
100+
- chmod +x tests/integration/targets/inventory-$INTEGRATION_TESTS/compare_inventory_json.py
122101
- chmod +x tests/integration/render_config.sh
123102

124103
# Run render_config.sh to pass environment variables to integration tests
125104
# https://www.ansible.com/blog/adding-integration-tests-to-ansible-content-collections
126-
- tests/integration/render_config.sh tests/integration/targets/inventory/runme_config.template > tests/integration/targets/inventory/runme_config
105+
- tests/integration/render_config.sh tests/integration/targets/inventory/runme_config.template > tests/integration/targets/inventory-$INTEGRATION_TESTS/runme_config
127106

128107
script:
129108
# Check python syntax
@@ -145,12 +124,12 @@ script:
145124

146125
# Run regression and integration tests
147126
# Run the inventory test first, in case any of the other tests modify the data.
148-
- ansible-test integration -v --coverage --python $PYTHON_VER inventory
149-
- ansible-test integration -v --coverage --python $PYTHON_VER regression
127+
- ansible-test integration -v --coverage --python $PYTHON_VER inventory-$INTEGRATION_TESTS
128+
- ansible-test integration -v --coverage --python $PYTHON_VER regression-$INTEGRATION_TESTS
150129
- ansible-test integration -v --coverage --python $PYTHON_VER $INTEGRATION_TESTS
151130

152131
# Report code coverage
153-
- ansible-test coverage report --all --omit "tests/*,hacking/*" --show-missing
132+
- ansible-test coverage report --all --omit "tests/*,hacking/*,docs/*" --show-missing
154133

155134
deploy:
156135
provider: script

plugins/module_utils/netbox_ipam.py

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,28 @@ def _handle_state_new_present(self, nb_app, nb_endpoint, endpoint_name, name, da
5353
def _ensure_ip_in_prefix_present_on_netif(
5454
self, nb_app, nb_endpoint, data, endpoint_name
5555
):
56-
""""""
57-
if not data.get("interface") or not data.get("prefix"):
58-
self._handle_errors("A prefix and interface are required")
56+
query_params = {
57+
"parent": data["prefix"],
58+
}
59+
60+
if self.version < 2.9:
61+
if not data.get("interface") or not data.get("prefix"):
62+
self._handle_errors("A prefix and interface is required")
63+
data_intf_key = "interface"
64+
65+
else:
66+
if not data.get("assigned_object_id") or not data.get("prefix"):
67+
self._handle_errors("A prefix and assigned_object is required")
68+
data_intf_key = "assigned_object_id"
69+
70+
intf_obj_type = data.get("assigned_object_type", "dcim.interface")
71+
if intf_obj_type == "virtualization.vminterface":
72+
intf_type = "vminterface_id"
73+
else:
74+
intf_type = "interface_id"
75+
76+
query_params.update({intf_type: data[data_intf_key]})
5977

60-
query_params = {"interface_id": data["interface"], "parent": data["prefix"]}
6178
if data.get("vrf"):
6279
query_params["vrf_id"] = data["vrf"]
6380

plugins/module_utils/netbox_utils.py

Lines changed: 131 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
"sites",
7070
"virtual_chassis",
7171
],
72-
extras=[],
72+
extras=["tags"],
7373
ipam=[
7474
"aggregates",
7575
"ip_addresses",
@@ -133,6 +133,7 @@
133133

134134
# Specifies keys within data that need to be converted to ID and the endpoint to be used when queried
135135
CONVERT_TO_ID = {
136+
"assigned_object": "assigned_object",
136137
"circuit": "circuits",
137138
"circuit_type": "circuit_types",
138139
"circuit_termination": "circuit_terminations",
@@ -180,6 +181,7 @@
180181
"rir": "rirs",
181182
"services": "services",
182183
"site": "sites",
184+
"tags": "tags",
183185
"tagged_vlans": "vlans",
184186
"tenant": "tenants",
185187
"tenant_group": "tenant_groups",
@@ -250,6 +252,7 @@
250252

251253
ALLOWED_QUERY_PARAMS = {
252254
"aggregate": set(["prefix", "rir"]),
255+
"assigned_object": set(["name", "device", "virtual_machine"]),
253256
"circuit": set(["cid"]),
254257
"circuit_type": set(["slug"]),
255258
"circuit_termination": set(["circuit", "term_side"]),
@@ -309,6 +312,7 @@
309312
"role": set(["slug"]),
310313
"services": set(["device", "virtual_machine", "name", "port", "protocol"]),
311314
"site": set(["slug"]),
315+
"tags": set(["slug"]),
312316
"tagged_vlans": set(["name", "site", "vid", "vlan_group", "tenant"]),
313317
"tenant": set(["slug"]),
314318
"tenant_group": set(["slug"]),
@@ -369,6 +373,7 @@
369373

370374
# This is used to map non-clashing keys to Netbox API compliant keys to prevent bad logic in code for similar keys but different modules
371375
CONVERT_KEYS = {
376+
"assigned_object": "assigned_object_id",
372377
"circuit_type": "type",
373378
"cluster_type": "type",
374379
"cluster_group": "group",
@@ -400,6 +405,7 @@
400405
"rirs",
401406
"roles",
402407
"sites",
408+
"tags",
403409
"tenants",
404410
"tenant_groups",
405411
"manufacturers",
@@ -557,6 +563,9 @@ def _convert_identical_keys(self, data):
557563
if self.endpoint == "power_panels" and key == "rack_group":
558564
temp_dict[key] = data[key]
559565
elif key in CONVERT_KEYS:
566+
# This will keep the original key for assigned_object, but also convert to assigned_object_id
567+
if key == "assigned_object":
568+
temp_dict[key] = data[key]
560569
new_key = CONVERT_KEYS[key]
561570
temp_dict[new_key] = data[key]
562571
else:
@@ -570,7 +579,10 @@ def _remove_arg_spec_default(self, data):
570579
"""
571580
new_dict = dict()
572581
for k, v in data.items():
573-
if v is not None:
582+
if isinstance(v, dict):
583+
v = self._remove_arg_spec_default(v)
584+
new_dict[k] = v
585+
elif v is not None:
574586
new_dict[k] = v
575587

576588
return new_dict
@@ -649,7 +661,18 @@ def _build_query_params(
649661
elif parent == "prefix" and module_data.get("parent"):
650662
query_dict.update({"prefix": module_data["parent"]})
651663

652-
elif parent == "ip_addreses":
664+
# This is for netbox_ipam and netbox_ip_address module
665+
elif parent == "ip_address" and module_data.get("assigned_object_type"):
666+
if module_data["assigned_object_type"] == "virtualization.vminterface":
667+
query_dict.update(
668+
{"vminterface_id": module_data.get("assigned_object_id")}
669+
)
670+
elif module_data["assigned_object_type"] == "dcim.interface":
671+
query_dict.update(
672+
{"interface_id": module_data.get("assigned_object_id")}
673+
)
674+
675+
elif parent == "ip_addresses":
653676
if isinstance(module_data["device"], int):
654677
query_dict.update({"device_id": module_data["device"]})
655678
else:
@@ -739,10 +762,14 @@ def _find_ids(self, data, user_query_params):
739762
"""
740763
for k, v in data.items():
741764
if k in CONVERT_TO_ID:
765+
if self.version < 2.9 and k == "tags":
766+
continue
742767
if k == "termination_a":
743768
endpoint = CONVERT_TO_ID[data.get("termination_a_type")]
744769
elif k == "termination_b":
745770
endpoint = CONVERT_TO_ID[data.get("termination_b_type")]
771+
elif k == "assigned_object":
772+
endpoint = "interfaces"
746773
else:
747774
endpoint = CONVERT_TO_ID[k]
748775
search = v
@@ -751,17 +778,23 @@ def _find_ids(self, data, user_query_params):
751778
nb_endpoint = getattr(nb_app, endpoint)
752779

753780
if isinstance(v, dict):
754-
if k == "interface" and v.get("virtual_machine"):
781+
if (k == "interface" or k == "assigned_object") and v.get(
782+
"virtual_machine"
783+
):
755784
nb_app = getattr(self.nb, "virtualization")
756785
nb_endpoint = getattr(nb_app, endpoint)
757786
query_params = self._build_query_params(k, data, child=v)
758787
query_id = self._nb_endpoint_get(nb_endpoint, query_params, k)
759-
760788
elif isinstance(v, list):
761789
id_list = list()
762790
for list_item in v:
763-
norm_data = self._normalize_data(list_item)
764-
temp_dict = self._build_query_params(k, data, child=norm_data)
791+
if k == "tags" and isinstance(list_item, str):
792+
temp_dict = {"slug": self._to_slug(list_item)}
793+
elif isinstance(list_item, dict):
794+
norm_data = self._normalize_data(list_item)
795+
temp_dict = self._build_query_params(
796+
k, data, child=norm_data
797+
)
765798
query_id = self._nb_endpoint_get(nb_endpoint, temp_dict, k)
766799
if query_id:
767800
id_list.append(query_id.id)
@@ -833,6 +866,14 @@ def _normalize_data(self, data):
833866
if k == "mac_address":
834867
data[k] = v.upper()
835868

869+
# We need to assign the correct type for the assigned object so the user doesn't have to worry about this.
870+
# We determine it by whether or not they pass in a device or virtual_machine
871+
if data.get("assigned_object"):
872+
if data["assigned_object"].get("device"):
873+
data["assigned_object_type"] = "dcim.interface"
874+
if data["assigned_object"].get("virtual_machine"):
875+
data["assigned_object_type"] = "virtualization.vminterface"
876+
836877
return data
837878

838879
def _create_netbox_object(self, nb_endpoint, data):
@@ -978,14 +1019,51 @@ def __init__(
9781019
argument_spec,
9791020
bypass_checks=False,
9801021
no_log=False,
981-
mutually_exclusive=None,
982-
required_together=None,
1022+
mutually_exclusive=mutually_exclusive,
1023+
required_together=required_together,
9831024
required_one_of=required_one_of,
9841025
add_file_common_args=False,
9851026
supports_check_mode=supports_check_mode,
9861027
required_if=required_if,
9871028
)
9881029

1030+
def _check_mutually_exclusive(self, spec, param=None):
1031+
if param is None:
1032+
param = self.params
1033+
1034+
try:
1035+
self.check_mutually_exclusive(spec, param)
1036+
except TypeError as e:
1037+
msg = to_native(e)
1038+
if self._options_context:
1039+
msg += " found in %s" % " -> ".join(self._options_context)
1040+
self.fail_json(msg=msg)
1041+
1042+
def check_mutually_exclusive(self, terms, module_parameters):
1043+
"""Check mutually exclusive terms against argument parameters
1044+
Accepts a single list or list of lists that are groups of terms that should be
1045+
mutually exclusive with one another
1046+
:arg terms: List of mutually exclusive module parameters
1047+
:arg module_parameters: Dictionary of module parameters
1048+
:returns: Empty list or raises TypeError if the check fails.
1049+
"""
1050+
1051+
results = []
1052+
if terms is None:
1053+
return results
1054+
1055+
for check in terms:
1056+
count = self.count_terms(check, module_parameters["data"])
1057+
if count > 1:
1058+
results.append(check)
1059+
1060+
if results:
1061+
full_list = ["|".join(check) for check in results]
1062+
msg = "parameters are mutually exclusive: %s" % ", ".join(full_list)
1063+
raise TypeError(to_native(msg))
1064+
1065+
return results
1066+
9891067
def _check_required_if(self, spec, param=None):
9901068
""" ensure that parameters which conditionally required are present """
9911069
if spec is None:
@@ -1089,6 +1167,50 @@ def check_required_one_of(self, terms, module_parameters):
10891167

10901168
return results
10911169

1170+
def _check_required_together(self, spec, param=None):
1171+
if spec is None:
1172+
return
1173+
if param is None:
1174+
param = self.params
1175+
1176+
try:
1177+
self.check_required_together(spec, param)
1178+
except TypeError as e:
1179+
msg = to_native(e)
1180+
if self._options_context:
1181+
msg += " found in %s" % " -> ".join(self._options_context)
1182+
self.fail_json(msg=msg)
1183+
1184+
def check_required_together(self, terms, module_parameters):
1185+
"""Check each list of terms to ensure every parameter in each list exists
1186+
in the given module parameters
1187+
Accepts a list of lists or tuples
1188+
:arg terms: List of lists of terms to check. Each list should include
1189+
parameters that are all required when at least one is specified
1190+
in the module_parameters.
1191+
:arg module_parameters: Dictionary of module parameters
1192+
:returns: Empty list or raises TypeError if the check fails.
1193+
"""
1194+
1195+
results = []
1196+
if terms is None:
1197+
return results
1198+
1199+
for term in terms:
1200+
counts = [
1201+
self.count_terms(field, module_parameters["data"]) for field in term
1202+
]
1203+
non_zero = [c for c in counts if c > 0]
1204+
if len(non_zero) > 0:
1205+
if 0 in counts:
1206+
results.append(term)
1207+
if results:
1208+
for term in results:
1209+
msg = "parameters are required together: %s" % ", ".join(term)
1210+
raise TypeError(to_native(msg))
1211+
1212+
return results
1213+
10921214
def count_terms(self, terms, module_parameters):
10931215
"""Count the number of occurrences of a key in a given dictionary
10941216
:arg terms: String or iterable of values to check

0 commit comments

Comments
 (0)