Merge pull request #353 from nautobot/release-4.2.4

Release 4.2.4

Author: scetron

.github/workflows/ci.yml

@@ -146,6 +146,9 @@ jobs:
           - python-version: "3.11"
             db-backend: "postgresql"
             nautobot-version: "2.2.3"
+          - python-version: "3.12"
+            db-backend: "mysql"
+            nautobot-version: "stable"
     runs-on: "ubuntu-22.04"
     env:
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}"

README.md

@@ -19,15 +19,15 @@ Regardless, the Onboarding App greatly simplifies the onboarding process by allo
 
 ### Support Matrix (Sync Devices From Network)
 
-|     Data Attribute      | Cisco IOS          | Cisco XE           | Cisco NXOS         | Cisco WLC          | Juniper Junos      | Arista EOS         | F5  | HP Comware | Palo Alto Panos |
-| ----------------------- | :----------------: |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  | :-: | :-: | :-: |
-| Hostname                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Platform                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Manufacturer            | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Serial Number           | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Device Type             | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Mgmt Interface          | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
-| Mgmt IP Address         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 |
+|     Data Attribute      | Cisco IOS          | Cisco XE           | Cisco NXOS         | Cisco WLC          | Juniper Junos      | Arista EOS         | F5  | HP Comware | Palo Alto Panos | Aruba AOSCX |
+| ----------------------- | :----------------: |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  | :-: | :-: | :-: | :-: |
+| Hostname                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Platform                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Manufacturer            | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Serial Number           | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Device Type             | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Mgmt Interface          | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
+| Mgmt IP Address         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 | 🧪 | 🧪 |
 
 ### Support Matrix (Sync Data From Network)
 

docs/admin/release_notes/version_4.2.md

@@ -10,6 +10,31 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 - [#200](https://github.com/nautobot/nautobot-app-device-onboarding/issues/200) - Added basic connectivity checker using Netutils tcp_ping method.
 - [#274](https://github.com/nautobot/nautobot-app-device-onboarding/issues/274) - Added TTP Parser support.
 - [#274](https://github.com/nautobot/nautobot-app-device-onboarding/issues/274) - Added Sync Device from Network support for Palo Alto Panos.
+- [#340](https://github.com/nautobot/nautobot-app-device-onboarding/issues/340) - Add Aruba AOSCX support for Sync Devices from Network Job.
+- [#278](https://github.com/nautobot/nautobot-app-device-onboarding/issues/278) - Optimized VLAN loading into diffsync from Nautbot
+- [#278](https://github.com/nautobot/nautobot-app-device-onboarding/issues/278) - Improved error handling when creating VLANs
+
+## [v4.2.4 (2025-04-08)](https://github.com/nautobot/nautobot-app-device-onboarding/releases/tag/v4.2.4)
+
+### Added
+
+- [#326](https://github.com/nautobot/nautobot-app-device-onboarding/issues/326) - Added testing for cable termination type when adding cables to diffsync store.
+- [#340](https://github.com/nautobot/nautobot-app-device-onboarding/issues/340) - Add Aruba AOSCX support for Sync Devices from Network Job.
+
+### Changed
+
+- [#278](https://github.com/nautobot/nautobot-app-device-onboarding/issues/278) - Optimized VLAN loading into diffsync from Nautbot
+- [#278](https://github.com/nautobot/nautobot-app-device-onboarding/issues/278) - Improved error handling when creating VLANs
+
+### Fixed
+
+- [#326](https://github.com/nautobot/nautobot-app-device-onboarding/issues/326) - Fixed incorrect call to cable termination type.
+- [#337](https://github.com/nautobot/nautobot-app-device-onboarding/issues/337) - Fixed logging call from an invalid path to the correct one.
+- [#346](https://github.com/nautobot/nautobot-app-device-onboarding/issues/346) - Fixed Juniper IP addresses not syncing secondary IP addresses.
+
+### Housekeeping
+
+- [#341](https://github.com/nautobot/nautobot-app-device-onboarding/issues/341) - Add MySQL testing back to the CI configuration.
 
 ## [v4.2.3 (2025-03-11)](https://github.com/nautobot/nautobot-app-device-onboarding/releases/tag/v4.2.3)
 

docs/dev/dev_environment.md

@@ -466,4 +466,4 @@ To run an individual test, you can run any or all of the following:
 ➜ invoke unittest
 ➜ invoke ruff
 ➜ invoke pylint
-```
+```

nautobot_device_onboarding/command_mappers/aruba_aoscx.yml

@@ -0,0 +1,34 @@
+---
+sync_devices:
+  hostname:
+    commands:
+      - command: "show system"
+        parser: "textfsm"
+        jpath: "[*].hostname"
+  serial:
+    commands:
+      - command: "show system"
+        parser: "textfsm"
+        jpath: "[*].serial[]"
+        post_processor: "{{ obj | unique | first }}"
+        iterable_type: "str"
+  device_type:
+    commands:
+      - command: "show system"
+        parser: "textfsm"
+        jpath: "[*].product[]"
+        post_processor: "{{ obj | unique | first }}"
+        iterable_type: "str"
+  mgmt_interface:
+    commands:
+      - command: "show interface"
+        parser: "textfsm"
+        jpath: "[?contains(ip_address, `{{ obj }}/`)].{name: interface, enabled: link_status}"
+        post_processor: "{{ (obj | selectattr('enabled', 'eq', 'up') | list | first ).name }}"
+  mask_length:
+    commands:
+      - command: "show interface"
+        parser: "textfsm"
+        jpath: "[?contains(ip_address, `{{ obj }}/`)].ip_address"
+        post_processor: "{{ obj[0].split('/')[1] }}"
+        iterable_type: "int"

nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py

@@ -214,7 +214,6 @@ def execute_command_getter(self):
                     raise Exception(  # pylint: disable=broad-exception-raised
                         "Platform.network_driver missing"
                     )
-
         result = sync_devices_command_getter(
             self.job.job_result,
             self.job.logger.getEffectiveLevel(),

nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py

@@ -5,6 +5,7 @@
 import diffsync
 from diffsync.enum import DiffSyncModelFlags
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from nautobot.dcim.models import Interface
 from nautobot.ipam.models import VLAN, VRF, IPAddress
@@ -129,9 +130,11 @@ def load_vlans(self):
         """
         Load Vlans into the Diffsync store.
 
-        Only Vlans that were returned by the CommandGetter job should be synced.
+        Only Vlans that share locations with devices included in the sync should be loaded.
         """
-        for vlan in VLAN.objects.all():
+        # TODO: update this to support multiple locations per VLAN after the setting for this feature has been added.
+        location_ids = list(self.job.devices_to_load.values_list("location__id", flat=True))
+        for vlan in VLAN.objects.filter(location__in=location_ids):
             network_vlan = self.vlan(
                 adapter=self,
                 name=vlan.name,
@@ -249,9 +252,13 @@ def load_cables(self):
 
         Only cables returned by the CommandGetter job should be synced.
         """
+        dcim_interface_content_type = ContentType.objects.get_for_model(Interface)
         for device in self.job.devices_to_load:
             for cable in device.get_cables():
-                if cable.termination_a.type != "dcim.interface" or cable.termination_b.type != "dcim.interface":
+                if (
+                    cable.termination_a_type != dcim_interface_content_type
+                    or cable.termination_b_type != dcim_interface_content_type
+                ):
                     self.job.logger.warning(
                         f"Skipping Cable: {cable}. Only cables with interface terminations are supported."
                     )
@@ -590,7 +597,6 @@ def load_vlans(self):
         ) in self.job.command_getter_result.items():  # pylint: disable=too-many-nested-blocks
             if self.job.debug:
                 self.job.logger.debug(f"Loading Vlans from {hostname}")
-            # for interface in device_data["interfaces"]:
             for _, interface_data in device_data["interfaces"].items():
                 # add tagged vlans
                 for tagged_vlan in interface_data["tagged_vlans"]:

nautobot_device_onboarding/diffsync/models/sync_network_data_models.py

@@ -210,18 +210,18 @@ def create(cls, adapter, ids, attrs):
         location = None
         try:
             location = Location.objects.get(name=ids["location__name"])
-        except ObjectDoesNotExist:
-            adapter.job.logger.warning(
+        except ObjectDoesNotExist as err:
+            adapter.job.logger.error(
                 f"While creating VLAN {ids['vid']} - {ids['name']}, "
-                f"unable to find a Location with name: {ids['location__name']}. "
-                "This VLAN will be created without a Location"
+                f"unable to find a Location with name: {ids['location__name']}."
             )
-        except MultipleObjectsReturned:
-            adapter.job.logger.warning(
+            raise diffsync_exceptions.ObjectNotCreated(err)
+        except MultipleObjectsReturned as err:
+            adapter.job.logger.error(
                 f"While creating VLAN {ids['vid']} - {ids['name']}, "
-                f"Multiple Locations were found with name: {ids['location__name']}. "
-                "This VLAN will be created without a Location"
+                f"Multiple Locations were found with name: {ids['location__name']}."
             )
+            raise diffsync_exceptions.ObjectNotCreated(err)
         try:
             vlan = VLAN(
                 name=ids["name"],
@@ -232,6 +232,7 @@ def create(cls, adapter, ids, attrs):
             vlan.validated_save()
         except ValidationError as err:
             adapter.job.logger.error(f"VLAN {vlan} failed to create, {err}")
+            raise diffsync_exceptions.ObjectNotCreated(err)
 
         return super().create(adapter, ids, attrs)
 
@@ -576,7 +577,7 @@ def _get_and_assign_vrf(cls, adapter, attrs, interface, diff_method_type):
             vrf.validated_save()
             interface.vrf = vrf
         except Exception as err:
-            adapter.logger.error(f"Failed to assign device: [{interface.device}] to vrf: [{vrf}], {err}")
+            adapter.job.logger.error(f"Failed to assign device: [{interface.device}] to vrf: [{vrf}], {err}")
             if diff_method_type == "create":
                 raise diffsync_exceptions.ObjectNotCreated(err)
             if diff_method_type == "update":

nautobot_device_onboarding/jinja_filters.py

@@ -4,6 +4,7 @@
 from itertools import chain
 
 from django_jinja import library
+from netutils.ip import is_ip
 from netutils.vlan import vlanconfig_to_list
 
 from nautobot_device_onboarding.constants import INTERFACE_TYPE_MAP_STATIC
@@ -174,11 +175,27 @@ def parse_junos_ip_address(item):
     """
     if isinstance(item, list) and len(item) > 0:
         if item[0]["prefix_length"] and item[0]["ip_address"]:
-            return [
-                {"prefix_length": item[0]["prefix_length"][0].split("/")[-1], "ip_address": item[0]["ip_address"][0]}
-            ]
+            result = []
+            for i in range(len(item[0]["ip_address"])):
+                prefix = item[0]["prefix_length"][i].split("/")[-1]
+                result.append(
+                    {
+                        "prefix_length": prefix,
+                        "ip_address": item[0]["ip_address"][i],
+                    }
+                )
+            return result
         if not item[0]["prefix_length"] and item[0]["ip_address"]:
-            return [{"prefix_length": 32, "ip_address": item[0]["ip_address"][0]}]
+            result = []
+            for i in range(len(item[0]["ip_address"])):
+                if is_ip(item[0]["ip_address"][i]):
+                    result.append(
+                        {
+                            "prefix_length": 32,
+                            "ip_address": item[0]["ip_address"][i],
+                        }
+                    )
+            return result
     return []
 
 

nautobot_device_onboarding/tests/mock/aruba_aoscx/command_getter_result_1.json

@@ -0,0 +1,103 @@
+{
+    "show system": [
+        {
+            "hostname": "SW-CORE",
+            "contact": "network",
+            "location": "Main",
+            "vendor": "Aruba",
+            "product": "JL664A 6300M 24G 4SFP56 Swch         ",
+            "serial": "SG9BKN123V",
+            "base_mac": "883a30-a73b80",
+            "version": "FL.10.10.1080",
+            "time_zone": "America/Edmonton",
+            "uptime_weeks": "43",
+            "uptime_days": "4",
+            "uptime_hours": "23",
+            "uptime_minutes": "3"
+        }
+    ],
+    "show interface": [
+        {
+            "interface": "1/1/1",
+            "link_status": "up",
+            "link_admin": "up",
+            "link_state_info": "",
+            "link_transitions": "5",
+            "interface_desc": "Server",
+            "hw_type": "Ethernet",
+            "mac_address": "00:3a:30:a7:3b:a7",
+            "mtu": "1500",
+            "if_type": "1GbT",
+            "ip_address": "",
+            "duplex": "Full",
+            "qos_trust": "dscp",
+            "speed": "100 Mb/s",
+            "auto_neg": "on",
+            "flow_control": "off",
+            "error_control": "off",
+            "vlan_mode": "access",
+            "vlan_access": "32",
+            "vlan_native": "",
+            "vlan_trunk": [],
+            "aggregated_interfaces": [],
+            "rx_total_packets": "237186061",
+            "rx_total_bytes": "59967394350",
+            "rx_unicast_packets": "237185944",
+            "rx_mcast_packets": "0",
+            "rx_broadcast_packets": "117",
+            "rx_errors": "0",
+            "rx_dropped": "0",
+            "rx_crc_fcs": "0",
+            "tx_total_packets": "233679330",
+            "tx_total_bytes": "63892999335",
+            "tx_unicast_packets": "218950785",
+            "tx_mcast_packets": "14101983",
+            "tx_broadcast_packets": "626562",
+            "tx_errors": "0",
+            "tx_dropped": "0",
+            "tx_crc_fcs": "n/a",
+            "tx_collision": "0"
+        },
+        {
+            "auto_neg": "",
+            "duplex": "",
+            "error_control": "",
+            "flow_control": "",
+            "hw_type": "Ethernet",
+            "if_type": "",
+            "interface": "vlan100",
+            "interface_desc": "mgmt",
+            "link_admin": "up",
+            "link_state_info": "",
+            "link_status": "up",
+            "link_transitions": "",
+            "mac_address": "bc:d7:a5:be:a5:00",
+            "mtu": "",
+            "qos_trust": "",
+            "rx_broadcast_packets": "",
+            "rx_crc_fcs": "",
+            "rx_dropped": "",
+            "rx_errors": "",
+            "rx_mcast_packets": "",
+            "rx_total_bytes": "",
+            "rx_total_packets": "",
+            "rx_unicast_packets": "",
+            "speed": "",
+            "tx_broadcast_packets": "",
+            "tx_collision": "",
+            "tx_crc_fcs": "",
+            "tx_dropped": "",
+            "tx_errors": "",
+            "tx_mcast_packets": "",
+            "tx_total_bytes": "",
+            "tx_total_packets": "",
+            "tx_unicast_packets": "",
+            "vlan_access": "",
+            "vlan_mode": "",
+            "ip_address": "198.51.100.1/24",
+            "vlan_trunk": [],
+            "vlan_native": "",
+            "aggregated_interfaces": []
+        }
+    ]
+}

nautobot_device_onboarding/tests/mock/aruba_aoscx/sync_devices/expected_result_1.json

@@ -0,0 +1,7 @@
+{
+    "device_type": "JL664A 6300M 24G 4SFP56 Swch         ",
+    "hostname": "SW-CORE",
+    "mask_length": 24,
+    "mgmt_interface": "vlan100",
+    "serial": "SG9BKN123V"
+}