Inventory: Create groups based on rack groups, and rack roles (#204)

Author: DouglasHeriot

.travis.yml

@@ -119,6 +119,11 @@ before_script:
   # https://github.com/ansible/ansible/issues/68415
   - chmod +x tests/integration/targets/inventory/runme.sh
   - chmod +x tests/integration/targets/inventory/compare_inventory_json.py
+  - chmod +x tests/integration/render_config.sh
+
+  # Run render_config.sh to pass environment variables to integration tests
+  # https://www.ansible.com/blog/adding-integration-tests-to-ansible-content-collections
+  - tests/integration/render_config.sh tests/integration/targets/inventory/runme_config.template > tests/integration/targets/inventory/runme_config
 
 script:
   # Check python syntax

plugins/inventory/nb_inventory.py

@@ -112,6 +112,8 @@
                 - tenant
                 - racks
                 - rack
+                - rack_group
+                - rack_role
                 - tags
                 - tag
                 - device_roles
@@ -367,6 +369,8 @@ def group_extractors(self):
             self._pluralize_group_by("site"): self.extract_site,
             self._pluralize_group_by("tenant"): self.extract_tenant,
             self._pluralize_group_by("rack"): self.extract_rack,
+            "rack_group": self.extract_rack_group,
+            "rack_role": self.extract_rack_role,
             self._pluralize_group_by("tag"): self.extract_tags,
             self._pluralize_group_by("role"): self.extract_device_role,
             self._pluralize_group_by("platform"): self.extract_platform,
@@ -411,6 +415,28 @@ def _pluralize(self, extracted_value):
         else:
             return extracted_value
 
+    def _objects_array_following_parents(
+        self, initial_object_id, object_lookup, object_parent_lookup
+    ):
+        objects = []
+
+        object_id = initial_object_id
+
+        # Keep looping until the object has no parent
+        while object_id is not None:
+            object_slug = object_lookup[object_id]
+
+            if object_slug in objects:
+                # Won't ever happen - defensively guard against infinite loop
+                break
+
+            objects.append(object_slug)
+
+            # Get the parent of this object
+            object_id = object_parent_lookup[object_id]
+
+        return objects
+
     def extract_disk(self, host):
         return host.get("disk")
 
@@ -451,6 +477,35 @@ def extract_rack(self, host):
         except Exception:
             return
 
+    def extract_rack_group(self, host):
+        # A host may have a rack. A rack may have a rack_group. A rack_group may have a parent rack_group.
+        # Produce a list of rack_groups:
+        # - it will be empty if the device has no rack, or the rack has no rack_group
+        # - it will have 1 element if the rack's group has no parent
+        # - it will have multiple elements if the rack's group has a parent group
+
+        rack = host.get("rack", None)
+        if not isinstance(rack, dict):
+            # Device has no rack
+            return None
+
+        rack_id = rack.get("id", None)
+        if rack_id is None:
+            # Device has no rack
+            return None
+
+        return self._objects_array_following_parents(
+            initial_object_id=self.racks_group_lookup[rack_id],
+            object_lookup=self.rack_groups_lookup,
+            object_parent_lookup=self.rack_group_parent_lookup,
+        )
+
+    def extract_rack_role(self, host):
+        try:
+            return self.racks_role_lookup[host["rack"]["id"]]
+        except Exception:
+            return
+
     def extract_site(self, host):
         try:
             return self._pluralize(self.sites_lookup[host["site"]["id"]])
@@ -570,21 +625,11 @@ def extract_regions(self, host):
             # Device has no site
             return []
 
-        regions = []
-        region_id = self.sites_region_lookup[site_id]
-
-        # Keep looping until the region has no parent
-        while region_id is not None:
-            region_slug = self.regions_lookup[region_id]
-            if region_slug in regions:
-                # Won't ever happen - defensively guard against infinite loop
-                break
-            regions.append(region_slug)
-
-            # Get the parent of this region
-            region_id = self.regions_parent_lookup[region_id]
-
-        return regions
+        return self._objects_array_following_parents(
+            initial_object_id=self.sites_region_lookup[site_id],
+            object_lookup=self.regions_lookup,
+            object_parent_lookup=self.regions_parent_lookup,
+        )
 
     def extract_cluster(self, host):
         try:
@@ -628,9 +673,7 @@ def get_region_for_site(site):
                 return (site["id"], None)
 
         # Dictionary of site id to region id
-        self.sites_region_lookup = dict(
-            filter(lambda x: x is not None, map(get_region_for_site, sites))
-        )
+        self.sites_region_lookup = dict(map(get_region_for_site, sites))
 
     def refresh_regions_lookup(self):
         url = self.api_endpoint + "/api/dcim/regions/?limit=0"
@@ -659,6 +702,37 @@ def refresh_racks_lookup(self):
         racks = self.get_resource_list(api_url=url)
         self.racks_lookup = dict((rack["id"], rack["name"]) for rack in racks)
 
+        def get_group_for_rack(rack):
+            try:
+                return (rack["id"], rack["group"]["id"])
+            except Exception:
+                return (rack["id"], None)
+
+        def get_role_for_rack(rack):
+            try:
+                return (rack["id"], rack["role"]["slug"])
+            except Exception:
+                return (rack["id"], None)
+
+        self.racks_group_lookup = dict(map(get_group_for_rack, racks))
+        self.racks_role_lookup = dict(map(get_role_for_rack, racks))
+
+    def refresh_rack_groups_lookup(self):
+        url = self.api_endpoint + "/api/dcim/rack-groups/?limit=0"
+        rack_groups = self.get_resource_list(api_url=url)
+        self.rack_groups_lookup = dict(
+            (rack_group["id"], rack_group["slug"]) for rack_group in rack_groups
+        )
+
+        def get_rack_group_parent(rack_group):
+            try:
+                return (rack_group["id"], rack_group["parent"]["id"])
+            except Exception:
+                return (rack_group["id"], None)
+
+        # Dictionary of rack group id to parent rack group id
+        self.rack_group_parent_lookup = dict(map(get_rack_group_parent, rack_groups))
+
     def refresh_device_roles_lookup(self):
         url = self.api_endpoint + "/api/dcim/device-roles/?limit=0"
         device_roles = self.get_resource_list(api_url=url)
@@ -698,13 +772,8 @@ def get_cluster_group(cluster):
             except Exception:
                 return (cluster["id"], None)
 
-        self.clusters_type_lookup = dict(
-            filter(lambda x: x is not None, map(get_cluster_type, clusters))
-        )
-
-        self.clusters_group_lookup = dict(
-            filter(lambda x: x is not None, map(get_cluster_group, clusters))
-        )
+        self.clusters_type_lookup = dict(map(get_cluster_type, clusters))
+        self.clusters_group_lookup = dict(map(get_cluster_group, clusters))
 
     def refresh_services(self):
         url = self.api_endpoint + "/api/ipam/services/?limit=0"
@@ -855,6 +924,7 @@ def lookup_processes(self):
             self.refresh_regions_lookup,
             self.refresh_tenants_lookup,
             self.refresh_racks_lookup,
+            self.refresh_rack_groups_lookup,
             self.refresh_device_roles_lookup,
             self.refresh_platforms_lookup,
             self.refresh_device_types_lookup,
@@ -1184,6 +1254,9 @@ def _fill_host_variables(self, host, hostname):
             if attribute == "region":
                 attribute = "regions"
 
+            if attribute == "rack_group":
+                attribute = "rack_groups"
+
             # Flatten the dict into separate host vars, if enabled
             if isinstance(extracted_value, dict) and (
                 (attribute == "config_context" and self.flatten_config_context)

tests/integration/netbox-deploy.py

@@ -7,7 +7,11 @@
 import os
 import pynetbox
 
-# Set nb variable to connec to Netbox and use the veriable in future calls
+# NOTE: If anything depends on specific versions of NetBox, can check INTEGRATION_TESTS in env
+# os.environ["INTEGRATION_TESTS"]
+
+
+# Set nb variable to connect to Netbox and use the veriable in future calls
 nb = pynetbox.api("http://localhost:32768", "0123456789abcdef0123456789abcdef01234567")
 
 
@@ -159,23 +163,34 @@
 core_switch = nb.dcim.device_roles.get(slug="core-switch")
 
 
-## Create Racks
-racks = [{"name": "Test Rack", "slug": "test-rack", "site": test_site2.id}]
-created_racks = nb.dcim.racks.create(racks)
-test_rack = nb.dcim.racks.get(slug="test-rack")
-
-
 ## Create Rack Groups
 rack_groups = [
-    {"name": "Test Rack Group", "slug": "test-rack-group", "site": test_site.id}
+    {"name": "Test Rack Group", "slug": "test-rack-group", "site": test_site.id},
+    {"name": "Parent Rack Group", "slug": "parent-rack-group", "site": test_site.id},
 ]
 created_rack_groups = nb.dcim.rack_groups.create(rack_groups)
 
+### Create Rack Group Parent relationship
+created_rack_groups[0].parent = created_rack_groups[1]
+created_rack_groups[0].save()
 
 ## Create Rack Roles
 rack_roles = [{"name": "Test Rack Role", "slug": "test-rack-role", "color": "4287f5"}]
 created_rack_roles = nb.dcim.rack_roles.create(rack_roles)
 
+## Create Racks
+racks = [
+    {
+        "name": "Test Rack Site 2",
+        "site": test_site2.id,
+        "role": created_rack_roles[0].id,
+    },
+    {"name": "Test Rack", "site": test_site.id, "group": created_rack_groups[0].id},
+]
+created_racks = nb.dcim.racks.create(racks)
+test_rack = nb.dcim.racks.get(name="Test Rack")  # racks don't have slugs
+test_rack_site2 = nb.dcim.racks.get(name="Test Rack Site 2")
+
 
 ## Create Devices
 devices = [
@@ -191,13 +206,14 @@
         "device_type": cisco_test.id,
         "device_role": core_switch.id,
         "site": test_site.id,
+        "rack": test_rack.id,
     },
     {
         "name": "R1-Device",
         "device_type": cisco_test.id,
         "device_role": core_switch.id,
         "site": test_site2.id,
-        "rack": test_rack.id,
+        "rack": test_rack_site2.id,
     },
     {
         "name": "Test Nexus One",
@@ -316,7 +332,6 @@
 
 
 ## Create Services
-
 services = [
     {"device": test100.id, "name": "ssh", "port": 22, "protocol": "tcp"},
     {

tests/integration/render_config.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# The only way to pass variables to integration tests - write to a file inside the integration test directory.
+# Usage: render_config.sh runme_config.template > runme_config
+# Copied from:
+# https://www.ansible.com/blog/adding-integration-tests-to-ansible-content-collections
+# https://github.com/xlab-si/digital_ocean.digital_ocean/blob/master/tests/utils/render.sh
+
+set -o xtrace # Print commands as they're run
+set -o errexit # abort on nonzero exitstatus
+set -o nounset # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+
+
+function main()
+{
+    readonly template="$1"
+    readonly content="$(cat "${template}")"
+
+    eval "echo \"$content\""
+}
+
+main "$@"

tests/integration/targets/inventory/.gitignore

@@ -0,0 +1 @@
+runme_config