NetBox 2.11 integration tests working and related 2.11 required changes (#511)

Author: FragmentedPacket

.github/workflows/main.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
@@ -46,8 +46,8 @@ jobs:
       matrix:
         include:
           - python-version: 3.6
-            VERSION: "v2.9"
-            INTEGRATION_TESTS: "v2.9"
+            VERSION: "v2.10"
+            INTEGRATION_TESTS: "v2.10"
           - python-version: 3.6
             VERSION: "latest"
             INTEGRATION_TESTS: "latest"
@@ -58,13 +58,14 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Clone & Start netbox-docker containers - latest
+      - name: Clone & Start netbox-docker containers - 2.10
         env:
           VERSION: ${{ matrix.VERSION }}
         run: |
           cd ..
           git clone https://github.com/netbox-community/netbox-docker.git
           cd netbox-docker
+          git checkout 1.1.0
           tee docker-compose.override.yml <<EOF
           version: '3.4'
           services:
@@ -75,26 +76,25 @@ jobs:
           docker-compose up -d --quiet-pull
           docker container ls
           cd ..
-        if: matrix.VERSION == 'latest'
-      - name: Clone & Start netbox-docker containers - 2.9
+        if: matrix.VERSION == 'v2.10'
+      - name: Clone & Start netbox-docker containers - latest
         env:
           VERSION: ${{ matrix.VERSION }}
         run: |
           cd ..
           git clone https://github.com/netbox-community/netbox-docker.git
           cd netbox-docker
-          git checkout 0.26.2
           tee docker-compose.override.yml <<EOF
           version: '3.4'
           services:
-            nginx:
+            netbox:
               ports:
                 - 32768:8080
           EOF
           docker-compose up -d --quiet-pull
           docker container ls
           cd ..
-        if: matrix.VERSION == 'v2.9'
+        if: matrix.VERSION == 'latest'
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1.1.1
         with:

README.md

@@ -12,7 +12,7 @@ To keep the code simple, we only officially support the two latest releases of N
 
 - NetBox 2.6+ or the two latest NetBox releases
 - Python 3.6+
-- Python modules: **pynetbox 5.0.4+**
+- Python modules: **pynetbox 5.0.4+**, **packages** if using Ansible < 2.10 as it's included in Ansible 2.10+
 - Ansible 2.9+
 - NetBox write-enabled token when using modules or read-only token for `nb_lookup/nb_inventory`
 

changelogs/fragments/2.11-changes.yml

@@ -0,0 +1,11 @@
+---
+minor_changes:
+  - **netbox_power_panel** - Add `location` option.
+  - **netbox_rack** - Add `location` option.
+  - **netbox_vlan_group** - Add `scope_type` option.
+  - **netbox_vlan_group** - Add `scope` option.
+  - **netbox_vlan_group** - Add `description` option.
+  - **netbox_vlan_group** - Add `custom_fields` option.
+
+major_changes:
+  - **packages** is now a required Python package and gets installed via Ansible 2.10+.

galaxy.yml

@@ -9,7 +9,7 @@ namespace: netbox
 name: netbox
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 3.0.0
+version: 3.1.0
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md
@@ -46,7 +46,7 @@ tags:
 # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
 # range specifiers can be set and are separated by ','
 # dependencies:
-  # Required for json_query used in lookup plugin integration tests
+# Required for json_query used in lookup plugin integration tests
 #  community.general: ">=1.0.0"
 
 # The URL of the originating SCM repository

meta/runtime.yml

@@ -3,4 +3,7 @@ requires_ansible: ">=2.9.10"
 plugin_routing:
   modules:
     netbox_interface:
+      tombstone:
+        removal_version: "0.1.0"
+        warning_text: "netbox_interface has been superseceded by netbox.netbox.netbox_device_interface"
       redirect: netbox.netbox.plugins.modules.netbox_device_interface

plugins/module_utils/netbox_utils.py

@@ -51,6 +51,7 @@
         "interfaces",
         "interface_templates",
         "inventory_items",
+        "locations",
         "manufacturers",
         "platforms",
         "power_feeds",
@@ -66,6 +67,7 @@
         "rear_port_templates",
         "regions",
         "sites",
+        "site_groups",
         "virtual_chassis",
     ],
     extras=["tags"],
@@ -101,6 +103,7 @@
     group="slug",
     installed_device="name",
     import_targets="name",
+    location="slug",
     manufacturer="slug",
     nat_inside="address",
     nat_outside="address",
@@ -125,6 +128,7 @@
     route_targets="name",
     slug="slug",
     site="slug",
+    site_group="slug",
     tenant="slug",
     tenant_group="slug",
     time_zone="timezone",
@@ -166,6 +170,7 @@
     "interface_template": "interface_templates",
     "ip_addresses": "ip_addresses",
     "ipaddresses": "ip_addresses",
+    "location": "locations",
     "lag": "interfaces",
     "manufacturer": "manufacturers",
     "master": "devices",
@@ -190,8 +195,11 @@
     "rear_port_template": "rear_port_templates",
     "rir": "rirs",
     "route_targets": "route_targets",
+    # Just a placeholder as scope can be several different types including sites.
+    "scope": "sites",
     "services": "services",
     "site": "sites",
+    "site_group": "site_groups",
     "tags": "tags",
     "tagged_vlans": "vlans",
     "tenant": "tenants",
@@ -232,6 +240,7 @@
     "interface_templates": "interface_template",
     "inventory_items": "inventory_item",
     "ip_addresses": "ip_address",
+    "locations": "location",
     "manufacturers": "manufacturer",
     "platforms": "platform",
     "power_feeds": "power_feed",
@@ -253,6 +262,7 @@
     "route_targets": "route_target",
     "services": "services",
     "sites": "site",
+    "site_groups": "site_group",
     "tags": "tags",
     "tenants": "tenant",
     "tenant_groups": "tenant_group",
@@ -300,6 +310,7 @@
     "ip_addresses": set(["address", "vrf", "device", "interface", "assigned_object"]),
     "ipaddresses": set(["address", "vrf", "device", "interface", "assigned_object"]),
     "lag": set(["name"]),
+    "location": set(["slug"]),
     "manufacturer": set(["slug"]),
     "master": set(["name"]),
     "nat_inside": set(["vrf", "address"]),
@@ -337,7 +348,7 @@
     "virtual_chassis": set(["name", "master"]),
     "virtual_machine": set(["name", "cluster"]),
     "vlan": set(["group", "name", "site", "tenant", "vid", "vlan_group"]),
-    "vlan_group": set(["slug", "site"]),
+    "vlan_group": set(["slug", "site", "scope"]),
     "vrf": set(["name", "tenant"]),
 }
 
@@ -351,6 +362,7 @@
         "rir",
         "vrf",
         "site",
+        "scope",
         "tenant",
         "type",
         "virtual_machine",
@@ -390,6 +402,7 @@
 # This is used to map non-clashing keys to Netbox API compliant keys to prevent bad logic in code for similar keys but different modules
 CONVERT_KEYS = {
     "assigned_object": "assigned_object_id",
+    "scope": "scope_id",
     "circuit_type": "type",
     "cluster_type": "type",
     "cluster_group": "group",
@@ -417,12 +430,14 @@
     "device_roles",
     "device_types",
     "ipam_roles",
+    "locations",
     "rack_groups",
     "rack_roles",
     "regions",
     "rirs",
     "roles",
     "sites",
+    "site_groups",
     "tags",
     "tenants",
     "tenant_groups",
@@ -432,6 +447,16 @@
     "vlan_groups",
 }
 
+SCOPE_TO_ENDPOINT = {
+    "dcim.location": "locations",
+    "dcim.rack": "racks",
+    "dcim.region": "regions",
+    "dcim.site": "sites",
+    "dcim.sitegroup": "site_groups",
+    "virtualization.cluster": "clusters",
+    "virtualization.clustergroup": "cluster_groups",
+}
+
 NETBOX_ARG_SPEC = dict(
     netbox_url=dict(type="str", required=True),
     netbox_token=dict(type="str", required=True, no_log=True),
@@ -609,8 +634,8 @@ def _convert_identical_keys(self, data):
             if self.endpoint == "power_panels" and key == "rack_group":
                 temp_dict[key] = data[key]
             elif key in CONVERT_KEYS:
-                # This will keep the original key for assigned_object, but also convert to assigned_object_id
-                if key == "assigned_object":
+                # This will keep the original key for keys in list, but also convert it.
+                if key in ("assigned_object", "scope"):
                     temp_dict[key] = data[key]
                 new_key = CONVERT_KEYS[key]
                 temp_dict[new_key] = data[key]
@@ -672,6 +697,8 @@ def _build_query_params(
             parent = module_data["termination_a_type"]
         elif parent == "termination_b" and module_data.get("termination_b_type"):
             parent = module_data["termination_b_type"]
+        elif parent == "scope":
+            parent = ENDPOINT_NAME_MAPPING[SCOPE_TO_ENDPOINT[module_data["scope_type"]]]
 
         query_dict = dict()
         if user_query_params:
@@ -862,6 +889,9 @@ def _find_ids(self, data, user_query_params):
                     endpoint = CONVERT_TO_ID[data.get("termination_b_type")]
                 elif k == "assigned_object":
                     endpoint = "interfaces"
+                elif k == "scope":
+                    # Determine endpoint name for scope ID resolution
+                    endpoint = SCOPE_TO_ENDPOINT[data["scope_type"]]
                 else:
                     endpoint = CONVERT_TO_ID[k]
                 search = v
@@ -911,6 +941,12 @@ def _find_ids(self, data, user_query_params):
                         query_params = self._build_query_params(
                             k, data, user_query_params
                         )
+                    elif k == "scope":
+                        query_params = {
+                            QUERY_TYPES.get(
+                                ENDPOINT_NAME_MAPPING[endpoint], "q"
+                            ): search
+                        }
                     else:
                         query_params = {QUERY_TYPES.get(k, "q"): search}
                     query_id = self._nb_endpoint_get(nb_endpoint, query_params, k)
@@ -960,7 +996,13 @@ def _normalize_data(self, data):
                         if sub_data_type == "slug":
                             data[k][subk] = self._to_slug(subv)
             else:
-                data_type = QUERY_TYPES.get(k, "q")
+                if k == "scope":
+                    data_type = QUERY_TYPES.get(
+                        ENDPOINT_NAME_MAPPING[SCOPE_TO_ENDPOINT[data["scope_type"]]],
+                        "q",
+                    )
+                else:
+                    data_type = QUERY_TYPES.get(k, "q")
                 if data_type == "slug":
                     data[k] = self._to_slug(v)
                 elif data_type == "timezone":