From b58cec97bbc560d6b2eb50b4ed17e1eb59eb9c15 Mon Sep 17 00:00:00 2001 From: Susan Hooks Date: Thu, 3 Oct 2024 20:30:58 +0000 Subject: [PATCH 1/7] update to fix mtu int error --- .../adapters/sync_network_data_adapters.py | 244 +++++++++++++----- 1 file changed, 184 insertions(+), 60 deletions(-) diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py index 04511c27..294fdb2c 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py @@ -13,7 +13,9 @@ from netutils.interface import canonical_interface_name from nautobot_device_onboarding.diffsync.models import sync_network_data_models -from nautobot_device_onboarding.nornir_plays.command_getter import sync_network_data_command_getter +from nautobot_device_onboarding.nornir_plays.command_getter import ( + sync_network_data_command_getter, +) from nautobot_device_onboarding.utils import diffsync_utils app_settings = settings.PLUGINS_CONFIG["nautobot_device_onboarding"] @@ -29,7 +31,9 @@ class FilteredNautobotAdapter(NautobotAdapter): def _load_objects(self, diffsync_model): # pylint: disable=protected-access """Given a diffsync model class, load a list of models from the database and return them.""" parameter_names = self._get_parameter_names(diffsync_model) - for database_object in diffsync_model._get_queryset(adapter=self): # pylint: disable=protected-access + for database_object in diffsync_model._get_queryset( + adapter=self + ): # pylint: disable=protected-access self._load_single_object(database_object, diffsync_model, parameter_names) @@ -39,11 +43,17 @@ class SyncNetworkDataNautobotAdapter(FilteredNautobotAdapter): device = sync_network_data_models.SyncNetworkDataDevice interface = sync_network_data_models.SyncNetworkDataInterface ip_address = sync_network_data_models.SyncNetworkDataIPAddress - ipaddress_to_interface = sync_network_data_models.SyncNetworkDataIPAddressToInterface + ipaddress_to_interface = ( + sync_network_data_models.SyncNetworkDataIPAddressToInterface + ) vlan = sync_network_data_models.SyncNetworkDataVLAN vrf = sync_network_data_models.SyncNetworkDataVRF - tagged_vlans_to_interface = sync_network_data_models.SyncNetworkDataTaggedVlansToInterface - untagged_vlan_to_interface = sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface + tagged_vlans_to_interface = ( + sync_network_data_models.SyncNetworkDataTaggedVlansToInterface + ) + untagged_vlan_to_interface = ( + sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface + ) lag_to_interface = sync_network_data_models.SyncNetworkDataLagToInterface vrf_to_interface = sync_network_data_models.SyncNetworkDataVrfToInterface cable = sync_network_data_models.SyncNetworkDataCable @@ -80,6 +90,12 @@ def load_param_mac_address(self, parameter_name, database_object): return str(database_object.mac_address) return "" + def load_param_mtu(self, parameter_name, database_object): + """Convert interface mtu to string.""" + if database_object.mtu: + return str(database_object.mtu) + return "" + def load_ip_addresses(self): """Load IP addresses into the DiffSync store. @@ -93,7 +109,9 @@ def load_ip_addresses(self): if ip_address: ip_address_hosts.add(ip_address["ip_address"]) if "" in ip_address_hosts: - ip_address_hosts.remove("") # do not attempt to filter ip addresses with empty strings + ip_address_hosts.remove( + "" + ) # do not attempt to filter ip addresses with empty strings for ip_address in IPAddress.objects.filter( host__in=ip_address_hosts, parent__namespace__name=self.job.namespace.name, @@ -154,7 +172,9 @@ def load_tagged_vlans_to_interface(self): name=interface.name, tagged_vlans=tagged_vlans, ) - network_tagged_vlans_to_interface.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST + network_tagged_vlans_to_interface.model_flags = ( + DiffSyncModelFlags.SKIP_UNMATCHED_DST + ) self.add(network_tagged_vlans_to_interface) def load_untagged_vlan_to_interface(self): @@ -175,7 +195,9 @@ def load_untagged_vlan_to_interface(self): name=interface.name, untagged_vlan=untagged_vlan, ) - network_untagged_vlan_to_interface.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST + network_untagged_vlan_to_interface.model_flags = ( + DiffSyncModelFlags.SKIP_UNMATCHED_DST + ) self.add(network_untagged_vlan_to_interface) def load_lag_to_interface(self): @@ -240,7 +262,10 @@ def load_cables(self): """ for device in self.job.devices_to_load: for cable in device.get_cables(): - if cable.termination_b.device.name == "" or cable.termination_a.device.name == "": + if ( + cable.termination_b.device.name == "" + or cable.termination_a.device.name == "" + ): self.job.logger.warning( f"Device attached to a cable is missing a name. Devices must have a name to utilize cable onboarding. " f"Skipping Cable: {cable}" @@ -322,15 +347,21 @@ def sync_complete(self, source, diff, *args, **kwargs): This method only runs if data was changed. """ if self.job.debug: - self.job.logger.debug("Sync Complete method called, checking for missing primary ip addresses...") - for device in self.job.devices_to_load.all(): # refresh queryset after sync is complete + self.job.logger.debug( + "Sync Complete method called, checking for missing primary ip addresses..." + ) + for ( + device + ) in self.job.devices_to_load.all(): # refresh queryset after sync is complete if not device.primary_ip: ip_address = "" try: ip_address = IPAddress.objects.get(id=self.primary_ips[device.id]) device.primary_ip4 = ip_address device.validated_save() - self.job.logger.info(f"Assigning {ip_address} as primary IP Address for Device: {device.name}") + self.job.logger.info( + f"Assigning {ip_address} as primary IP Address for Device: {device.name}" + ) except Exception as err: # pylint: disable=broad-exception-caught self.job.logger.error( f"Unable to set Primary IP for {device.name}, {err.args}. " @@ -338,7 +369,9 @@ def sync_complete(self, source, diff, *args, **kwargs): ) if ip_address: try: - interface = Interface.objects.get(device=device, ip_addresses__in=[ip_address]) + interface = Interface.objects.get( + device=device, ip_addresses__in=[ip_address] + ) interface.mgmt_only = True interface.validated_save() self.job.logger.info( @@ -374,11 +407,17 @@ def __init__(self, *args, job, sync=None, **kwargs): device = sync_network_data_models.SyncNetworkDataDevice interface = sync_network_data_models.SyncNetworkDataInterface ip_address = sync_network_data_models.SyncNetworkDataIPAddress - ipaddress_to_interface = sync_network_data_models.SyncNetworkDataIPAddressToInterface + ipaddress_to_interface = ( + sync_network_data_models.SyncNetworkDataIPAddressToInterface + ) vlan = sync_network_data_models.SyncNetworkDataVLAN vrf = sync_network_data_models.SyncNetworkDataVRF - tagged_vlans_to_interface = sync_network_data_models.SyncNetworkDataTaggedVlansToInterface - untagged_vlan_to_interface = sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface + tagged_vlans_to_interface = ( + sync_network_data_models.SyncNetworkDataTaggedVlansToInterface + ) + untagged_vlan_to_interface = ( + sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface + ) lag_to_interface = sync_network_data_models.SyncNetworkDataLagToInterface vrf_to_interface = sync_network_data_models.SyncNetworkDataVrfToInterface cable = sync_network_data_models.SyncNetworkDataCable @@ -416,8 +455,10 @@ def _handle_failed_devices(self, device_data): for hostname in failed_devices: del device_data[hostname] - device_queryset, devices_with_errors = diffsync_utils.generate_device_queryset_from_command_getter_result( - job=self.job, command_getter_result=device_data + device_queryset, devices_with_errors = ( + diffsync_utils.generate_device_queryset_from_command_getter_result( + job=self.job, command_getter_result=device_data + ) ) # remove devices that have errors found while creating the queryset from the command getter results @@ -433,19 +474,25 @@ def _handle_failed_devices(self, device_data): def _handle_general_load_exception(self, error, hostname, data, model_type): """If a diffsync model fails to load, log the error.""" - self.job.logger.error(f"Failed to load {model_type} model for {hostname}. {error}, {error.args}") + self.job.logger.error( + f"Failed to load {model_type} model for {hostname}. {error}, {error.args}" + ) if self.job.debug: self.job.logger.debug(f"HOSTNAME: {hostname}, DATA: {data}") def execute_command_getter(self): """Query devices for data.""" result = sync_network_data_command_getter( - self.job.job_result, self.job.logger.getEffectiveLevel(), self.job.job_result.task_kwargs + self.job.job_result, + self.job.logger.getEffectiveLevel(), + self.job.job_result.task_kwargs, ) # verify data returned is a dict data_type_check = diffsync_utils.check_data_type(result) if self.job.debug: - self.job.logger.debug(f"CommandGetter data type check result: {data_type_check}") + self.job.logger.debug( + f"CommandGetter data type check result: {data_type_check}" + ) if data_type_check: self._handle_failed_devices(device_data=result) else: @@ -472,11 +519,15 @@ def load_devices(self): ) self.add(network_device) except Exception as err: # pylint: disable=broad-exception-caught - self._handle_general_load_exception(error=err, hostname=hostname, data=device_data, model_type="device") + self._handle_general_load_exception( + error=err, hostname=hostname, data=device_data, model_type="device" + ) continue # for interface in device_data["interfaces"]: for interface_name, interface_data in device_data["interfaces"].items(): - network_interface = self.load_interface(hostname, interface_name, interface_data) + network_interface = self.load_interface( + hostname, interface_name, interface_data + ) network_device.add_child(network_interface) # def _get_vlan_name(self, interface_data): @@ -494,8 +545,10 @@ def load_interface(self, hostname, interface_name, interface_data): device__name=hostname, status__name=self.job.interface_status.name, type=interface_data["type"], - mac_address=self._process_mac_address(mac_address=interface_data["mac_address"]), - mtu=interface_data["mtu"] if interface_data["mtu"] else 1500, + mac_address=self._process_mac_address( + mac_address=interface_data["mac_address"] + ), + mtu=interface_data["mtu"] if interface_data["mtu"] else "1500", description=interface_data["description"], enabled=interface_data["link_status"], mode=interface_data["802.1Q_mode"], @@ -508,16 +561,25 @@ def load_interface(self, hostname, interface_name, interface_data): def load_ip_addresses(self): """Load IP addresses into the DiffSync store.""" - for hostname, device_data in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks + for ( + hostname, + device_data, + ) in ( + self.job.command_getter_result.items() + ): # pylint: disable=too-many-nested-blocks if self.job.debug: self.job.logger.debug(f"Loading IP Addresses from {hostname}") # for interface in device_data["interfaces"]: for interface_name, interface_data in device_data["interfaces"].items(): if interface_data["ip_addresses"]: for ip_address in interface_data["ip_addresses"]: - if ip_address["ip_address"]: # the ip_address and mask_length may be empty, skip these + if ip_address[ + "ip_address" + ]: # the ip_address and mask_length may be empty, skip these if self.job.debug: - self.job.logger.debug(f"Loading {ip_address} from {interface_name} on {hostname}") + self.job.logger.debug( + f"Loading {ip_address} from {interface_name} on {hostname}" + ) try: network_ip_address = self.ip_address( adapter=self, @@ -534,9 +596,14 @@ def load_ip_addresses(self): "DiffSync store. This is a duplicate IP Address." ) continue - except Exception as err: # pylint: disable=broad-exception-caught + except ( + Exception + ) as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="ip_address" + error=err, + hostname=hostname, + data=device_data, + model_type="ip_address", ) continue @@ -546,7 +613,12 @@ def load_vlans(self): for device in self.job.devices_to_load: location_names[device.name] = device.location.name - for hostname, device_data in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks + for ( + hostname, + device_data, + ) in ( + self.job.command_getter_result.items() + ): # pylint: disable=too-many-nested-blocks if self.job.debug: self.job.logger.debug(f"Loading Vlans from {hostname}") # for interface in device_data["interfaces"]: @@ -565,7 +637,10 @@ def load_vlans(self): continue except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="vlan" + error=err, + hostname=hostname, + data=device_data, + model_type="vlan", ) continue # check for untagged vlan and add if necessary @@ -582,13 +657,21 @@ def load_vlans(self): continue except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="vlan" + error=err, + hostname=hostname, + data=device_data, + model_type="vlan", ) continue def load_vrfs(self): """Load vrfs into the Diffsync store.""" - for hostname, device_data in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks + for ( + hostname, + device_data, + ) in ( + self.job.command_getter_result.items() + ): # pylint: disable=too-many-nested-blocks if self.job.debug: self.job.logger.debug(f"Loading Vrfs from {hostname}") # for interface in device_data["interfaces"]: @@ -605,30 +688,49 @@ def load_vrfs(self): continue except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="vrf" + error=err, + hostname=hostname, + data=device_data, + model_type="vrf", ) continue def load_ip_address_to_interfaces(self): """Load ip address interface assignments into the Diffsync store.""" - for hostname, device_data in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks + for ( + hostname, + device_data, + ) in ( + self.job.command_getter_result.items() + ): # pylint: disable=too-many-nested-blocks for interface_name, interface_data in device_data["interfaces"].items(): for ip_address in interface_data["ip_addresses"]: - if ip_address["ip_address"]: # the ip_address and mask_length may be empty, skip these + if ip_address[ + "ip_address" + ]: # the ip_address and mask_length may be empty, skip these try: - network_ip_address_to_interface = self.ipaddress_to_interface( - adapter=self, - interface__device__name=hostname, - interface__name=interface_name, - ip_address__host=ip_address["ip_address"], - ip_address__mask_length=( - int(ip_address["prefix_length"]) if ip_address["prefix_length"] else None - ), + network_ip_address_to_interface = ( + self.ipaddress_to_interface( + adapter=self, + interface__device__name=hostname, + interface__name=interface_name, + ip_address__host=ip_address["ip_address"], + ip_address__mask_length=( + int(ip_address["prefix_length"]) + if ip_address["prefix_length"] + else None + ), + ) ) self.add(network_ip_address_to_interface) - except Exception as err: # pylint: disable=broad-exception-caught + except ( + Exception + ) as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="ip_address to interface" + error=err, + hostname=hostname, + data=device_data, + model_type="ip_address to interface", ) continue @@ -647,7 +749,10 @@ def load_tagged_vlans_to_interface(self): self.add(network_tagged_vlans_to_interface) except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="tagged vlan to interface" + error=err, + hostname=hostname, + data=device_data, + model_type="tagged vlan to interface", ) continue @@ -657,16 +762,21 @@ def load_untagged_vlan_to_interface(self): # for interface in device_data["interfaces"]: for interface_name, interface_data in device_data["interfaces"].items(): try: - network_untagged_vlan_to_interface = self.untagged_vlan_to_interface( - adapter=self, - device__name=hostname, - name=interface_name, - untagged_vlan=interface_data["untagged_vlan"], + network_untagged_vlan_to_interface = ( + self.untagged_vlan_to_interface( + adapter=self, + device__name=hostname, + name=interface_name, + untagged_vlan=interface_data["untagged_vlan"], + ) ) self.add(network_untagged_vlan_to_interface) except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="untagged vlan to interface" + error=err, + hostname=hostname, + data=device_data, + model_type="untagged vlan to interface", ) continue @@ -680,12 +790,17 @@ def load_lag_to_interface(self): adapter=self, device__name=hostname, name=interface_name, - lag__interface__name=interface_data["lag"] if interface_data["lag"] else "", + lag__interface__name=( + interface_data["lag"] if interface_data["lag"] else "" + ), ) self.add(network_lag_to_interface) except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="lag to interface" + error=err, + hostname=hostname, + data=device_data, + model_type="lag to interface", ) continue @@ -704,7 +819,10 @@ def load_vrf_to_interface(self): self.add(network_vrf_to_interface) except Exception as err: # pylint: disable=broad-exception-caught self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="vrf to interface" + error=err, + hostname=hostname, + data=device_data, + model_type="vrf to interface", ) continue @@ -712,7 +830,9 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements """Load cables into the Diffsync store.""" for hostname, device_data in self.job.command_getter_result.items(): if "cables" not in device_data: - self.job.logger.warning(f"No cable data found for {hostname}. Skipping cable load.") + self.job.logger.warning( + f"No cable data found for {hostname}. Skipping cable load." + ) return if self.job.debug: self.job.logger.debug(f"Loading Cables from {hostname}") @@ -723,7 +843,9 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements try: for neighbor_data in device_data["cables"]: try: - local_interface = canonical_interface_name(neighbor_data["local_interface"]) + local_interface = canonical_interface_name( + neighbor_data["local_interface"] + ) except KeyError as error: error.args = error.args + "Local interface is missing a name." self._handle_general_load_exception( @@ -735,7 +857,9 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements continue try: - remote_interface = canonical_interface_name(neighbor_data["remote_interface"]) + remote_interface = canonical_interface_name( + neighbor_data["remote_interface"] + ) except KeyError as error: error.args = error.args + "Remote interface is missing a name." self._handle_general_load_exception( From 23a6c53f419ee3243e147c8aafc4856d950ded6e Mon Sep 17 00:00:00 2001 From: Susan Hooks Date: Thu, 3 Oct 2024 21:50:00 +0000 Subject: [PATCH 2/7] update changelog --- changes/238.fixed | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/238.fixed diff --git a/changes/238.fixed b/changes/238.fixed new file mode 100644 index 00000000..0666a4d9 --- /dev/null +++ b/changes/238.fixed @@ -0,0 +1 @@ +Added a fix for mtu as an integer rather than a string. \ No newline at end of file From 89399e16e89effe91ba3132663ed9d8f7414b0cf Mon Sep 17 00:00:00 2001 From: Susan Hooks Date: Fri, 4 Oct 2024 18:03:09 +0000 Subject: [PATCH 3/7] pylint --- .../adapters/sync_network_data_adapters.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py index 294fdb2c..d05e83e4 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py @@ -31,9 +31,11 @@ class FilteredNautobotAdapter(NautobotAdapter): def _load_objects(self, diffsync_model): # pylint: disable=protected-access """Given a diffsync model class, load a list of models from the database and return them.""" parameter_names = self._get_parameter_names(diffsync_model) - for database_object in diffsync_model._get_queryset( + for ( + database_object + ) in diffsync_model._get_queryset( # pylint: disable=protected-access adapter=self - ): # pylint: disable=protected-access + ): self._load_single_object(database_object, diffsync_model, parameter_names) @@ -561,12 +563,10 @@ def load_interface(self, hostname, interface_name, interface_data): def load_ip_addresses(self): """Load IP addresses into the DiffSync store.""" - for ( + for ( # pylint: disable=too-many-nested-blocks hostname, device_data, - ) in ( - self.job.command_getter_result.items() - ): # pylint: disable=too-many-nested-blocks + ) in self.job.command_getter_result.items(): if self.job.debug: self.job.logger.debug(f"Loading IP Addresses from {hostname}") # for interface in device_data["interfaces"]: @@ -597,8 +597,8 @@ def load_ip_addresses(self): ) continue except ( - Exception - ) as err: # pylint: disable=broad-exception-caught + Exception # pylint: disable=broad-exception-caught + ) as err: self._handle_general_load_exception( error=err, hostname=hostname, @@ -724,8 +724,8 @@ def load_ip_address_to_interfaces(self): ) self.add(network_ip_address_to_interface) except ( - Exception - ) as err: # pylint: disable=broad-exception-caught + Exception # pylint: disable=broad-exception-caught + ) as err: self._handle_general_load_exception( error=err, hostname=hostname, From f9c4cb0553cc5d2d9f2ac80765348a54e078443e Mon Sep 17 00:00:00 2001 From: Susan Hooks Date: Fri, 4 Oct 2024 22:12:59 +0000 Subject: [PATCH 4/7] added fqdn support, formatting --- changes/241.added | 1 + .../adapters/sync_devices_adapters.py | 100 ++++++-- nautobot_device_onboarding/jobs.py | 227 ++++++++++++++---- 3 files changed, 252 insertions(+), 76 deletions(-) create mode 100644 changes/241.added diff --git a/changes/241.added b/changes/241.added new file mode 100644 index 00000000..924b2c30 --- /dev/null +++ b/changes/241.added @@ -0,0 +1 @@ +Added FQDN support to the sync network device job. \ No newline at end of file diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py index 16d3fb4a..ec0bab21 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py @@ -3,6 +3,7 @@ from collections import defaultdict from typing import DefaultDict, Dict, FrozenSet, Hashable, Tuple, Type +import socket import diffsync import netaddr from django.contrib.contenttypes.models import ContentType @@ -11,7 +12,9 @@ from nautobot.dcim.models import Device, DeviceType, Manufacturer, Platform from nautobot_device_onboarding.diffsync.models import sync_devices_models -from nautobot_device_onboarding.nornir_plays.command_getter import sync_devices_command_getter +from nautobot_device_onboarding.nornir_plays.command_getter import ( + sync_devices_command_getter, +) from nautobot_device_onboarding.utils import diffsync_utils ParameterSet = FrozenSet[Tuple[str, Hashable]] @@ -54,7 +57,9 @@ def get_from_orm_cache(self, parameters: Dict, model_class: Type[Model]): return cached_object # As we are using `get` here, this will error if there is not exactly one object that corresponds to the # parameter set. We intentionally pass these errors through. - self._cache[model_cache_key][parameter_set] = model_class.objects.get(**dict(parameter_set)) + self._cache[model_cache_key][parameter_set] = model_class.objects.get( + **dict(parameter_set) + ) return self._cache[model_cache_key][parameter_set] def load_manufacturers(self): @@ -62,7 +67,9 @@ def load_manufacturers(self): for manufacturer in Manufacturer.objects.all(): if self.job.debug: self.job.logger.debug("Loading Manufacturer data from Nautobot...") - onboarding_manufacturer = self.manufacturer(adapter=self, pk=manufacturer.pk, name=manufacturer.name) + onboarding_manufacturer = self.manufacturer( + adapter=self, pk=manufacturer.pk, name=manufacturer.name + ) self.add(onboarding_manufacturer) if self.job.debug: self.job.logger.debug(f"Manufacturer: {manufacturer.name} loaded.") @@ -76,8 +83,12 @@ def load_platforms(self): adapter=self, pk=platform.pk, name=platform.name, - network_driver=platform.network_driver if platform.network_driver else "", - manufacturer__name=platform.manufacturer.name if platform.manufacturer else None, + network_driver=( + platform.network_driver if platform.network_driver else "" + ), + manufacturer__name=( + platform.manufacturer.name if platform.manufacturer else None + ), ) self.add(onboarding_platform) if self.job.debug: @@ -104,7 +115,9 @@ def load_devices(self): if self.job.debug: self.job.logger.debug("Loading Device data from Nautobot...") - for device in Device.objects.filter(primary_ip4__host__in=self.job.ip_addresses): + for device in Device.objects.filter( + primary_ip4__host__in=self.job.ip_addresses + ): interface_list = [] # Only interfaces with the device's primary ip should be considered for diff calculations # Ultimately, only the first matching interface is used but this list could support multiple @@ -125,12 +138,18 @@ def load_devices(self): name=device.name, platform__name=device.platform.name if device.platform else "", primary_ip4__host=device.primary_ip4.host if device.primary_ip4 else "", - primary_ip4__status__name=device.primary_ip4.status.name if device.primary_ip4 else "", + primary_ip4__status__name=( + device.primary_ip4.status.name if device.primary_ip4 else "" + ), role__name=device.role.name, status__name=device.status.name, - secrets_group__name=device.secrets_group.name if device.secrets_group else "", + secrets_group__name=( + device.secrets_group.name if device.secrets_group else "" + ), interfaces=interfaces, - mask_length=device.primary_ip4.mask_length if device.primary_ip4 else None, + mask_length=( + device.primary_ip4.mask_length if device.primary_ip4 else None + ), serial=device.serial, ) self.add(onboarding_device) @@ -167,12 +186,19 @@ def _validate_ip_addresses(self, ip_addresses): """Validate the format of each IP Address in a list of IP Addresses.""" # Validate IP Addresses validation_successful = True - for ip_address in ip_addresses: + for i, ip_address in enumerate(ip_addresses): try: netaddr.IPAddress(ip_address) except netaddr.AddrFormatError: - self.job.logger.error(f"[{ip_address}] is not a valid IP Address ") - validation_successful = False + try: + resolved_ip = socket.gethostbyname(ip_address) + self.job.logger.info(f"[{ip_address}] resolved to [{resolved_ip}]") + ip_addresses[i] = resolved_ip + except socket.gaierror: + self.job.logger.error( + f"[{ip_address}] is not a valid IP Address or name." + ) + validation_successful = False if validation_successful: return True raise netaddr.AddrConversionError @@ -188,7 +214,9 @@ def _handle_failed_devices(self, device_data): self.failed_ip_addresses = [] for ip_address in device_data: if not device_data[ip_address]: - self.job.logger.error(f"{ip_address}: Connection or data error, this device will not be synced.") + self.job.logger.error( + f"{ip_address}: Connection or data error, this device will not be synced." + ) self.failed_ip_addresses.append(ip_address) for ip_address in self.failed_ip_addresses: del device_data[ip_address] @@ -203,16 +231,22 @@ def execute_command_getter(self): f"The selected platform, {self.job.platform} " "does not have a network driver, please update the Platform." ) - raise Exception("Platform.network_driver missing") # pylint: disable=broad-exception-raised + raise Exception( # pylint: disable=broad-exception-raised + "Platform.network_driver missing" + ) result = sync_devices_command_getter( - self.job.job_result, self.job.logger.getEffectiveLevel(), self.job.job_result.task_kwargs + self.job.job_result, + self.job.logger.getEffectiveLevel(), + self.job.job_result.task_kwargs, ) if self.job.debug: self.job.logger.debug(f"Command Getter Result: {result}") data_type_check = diffsync_utils.check_data_type(result) if self.job.debug: - self.job.logger.debug(f"CommandGetter data type check resut: {data_type_check}") + self.job.logger.debug( + f"CommandGetter data type check resut: {data_type_check}" + ) if data_type_check: self._handle_failed_devices(device_data=result) else: @@ -297,8 +331,16 @@ def load_device_types(self): def _fields_missing_data(self, device_data, ip_address, platform): """Verify that all of the fields returned from a device actually contain data.""" fields_missing_data = [] - required_fields_from_device = ["device_type", "hostname", "mgmt_interface", "mask_length", "serial"] - if platform: # platform is only returned with device data if not provided on the job form/csv + required_fields_from_device = [ + "device_type", + "hostname", + "mgmt_interface", + "mask_length", + "serial", + ] + if ( + platform + ): # platform is only returned with device data if not provided on the job form/csv required_fields_from_device.append("platform") for field in required_fields_from_device: data = device_data[ip_address] @@ -311,7 +353,9 @@ def load_devices(self): for ip_address in self.device_data: if self.job.debug: self.job.logger.debug(f"loading device data for {ip_address}") - platform = None # If an excption is caught below, the platform must still be set. + platform = ( + None # If an excption is caught below, the platform must still be set. + ) onboarding_device = None try: location = diffsync_utils.retrieve_submitted_value( @@ -321,7 +365,9 @@ def load_devices(self): job=self.job, ip_address=ip_address, query_string="platform" ) primary_ip4__status = diffsync_utils.retrieve_submitted_value( - job=self.job, ip_address=ip_address, query_string="ip_address_status" + job=self.job, + ip_address=ip_address, + query_string="ip_address_status", ) device_role = diffsync_utils.retrieve_submitted_value( job=self.job, ip_address=ip_address, query_string="device_role" @@ -338,7 +384,11 @@ def load_devices(self): device_type__model=self.device_data[ip_address]["device_type"], location__name=location.name, name=self.device_data[ip_address]["hostname"], - platform__name=platform.name if platform else self.device_data[ip_address]["platform"], + platform__name=( + platform.name + if platform + else self.device_data[ip_address]["platform"] + ), primary_ip4__host=ip_address, primary_ip4__status__name=primary_ip4__status.name, role__name=device_role.name, @@ -372,7 +422,9 @@ def load_devices(self): try: self.add(onboarding_device) if self.job.debug: - self.job.logger.debug(f"Device: {self.device_data[ip_address]['hostname']} loaded.") + self.job.logger.debug( + f"Device: {self.device_data[ip_address]['hostname']} loaded." + ) except diffsync.ObjectAlreadyExists: self.job.logger.error( f"Device: {self.device_data[ip_address]['hostname']} has already been loaded! " @@ -383,7 +435,9 @@ def load_devices(self): else: self._add_ip_address_to_failed_list(ip_address=ip_address) if self.job.debug: - self.job.logger.debug(f"{ip_address} was added to the failed ip_address list") + self.job.logger.debug( + f"{ip_address} was added to the failed ip_address list" + ) def load(self): """Load network data.""" diff --git a/nautobot_device_onboarding/jobs.py b/nautobot_device_onboarding/jobs.py index 6ee5f279..5c589664 100755 --- a/nautobot_device_onboarding/jobs.py +++ b/nautobot_device_onboarding/jobs.py @@ -10,11 +10,30 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist, ValidationError -from nautobot.apps.jobs import BooleanVar, ChoiceVar, FileVar, IntegerVar, Job, MultiObjectVar, ObjectVar, StringVar +from nautobot.apps.jobs import ( + BooleanVar, + ChoiceVar, + FileVar, + IntegerVar, + Job, + MultiObjectVar, + ObjectVar, + StringVar, +) from nautobot.core.celery import register_jobs from nautobot.dcim.models import Device, DeviceType, Location, Platform -from nautobot.extras.choices import CustomFieldTypeChoices, SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices -from nautobot.extras.models import CustomField, Role, SecretsGroup, SecretsGroupAssociation, Status +from nautobot.extras.choices import ( + CustomFieldTypeChoices, + SecretsGroupAccessTypeChoices, + SecretsGroupSecretTypeChoices, +) +from nautobot.extras.models import ( + CustomField, + Role, + SecretsGroup, + SecretsGroupAssociation, + Status, +) from nautobot.ipam.models import Namespace from nautobot_plugin_nornir.constants import NORNIR_SETTINGS from nautobot_ssot.jobs.base import DataSource @@ -32,7 +51,10 @@ ) from nautobot_device_onboarding.exceptions import OnboardException from nautobot_device_onboarding.netdev_keeper import NetdevKeeper -from nautobot_device_onboarding.nornir_plays.command_getter import _parse_credentials, netmiko_send_commands +from nautobot_device_onboarding.nornir_plays.command_getter import ( + _parse_credentials, + netmiko_send_commands, +) from nautobot_device_onboarding.nornir_plays.empty_inventory import EmptyInventory from nautobot_device_onboarding.nornir_plays.inventory_creator import _set_inventory from nautobot_device_onboarding.nornir_plays.logger import NornirLogger @@ -62,7 +84,9 @@ class OnboardingTask(Job): # pylint: disable=too-many-instance-attributes port = IntegerVar(default=22) timeout = IntegerVar(default=30) credentials = ObjectVar( - model=SecretsGroup, required=False, description="SecretsGroup for Device connection credentials." + model=SecretsGroup, + required=False, + description="SecretsGroup for Device connection credentials.", ) platform = ObjectVar( model=Platform, @@ -128,7 +152,9 @@ def run(self, *args, **data): self._onboard(address=address) except OnboardException as err: self.logger.exception( - "The following exception occurred when attempting to onboard %s: %s", address, str(err) + "The following exception occurred when attempting to onboard %s: %s", + address, + str(err), ) if not data["continue_on_failure"]: raise OnboardException( @@ -146,9 +172,15 @@ def _onboard(self, address): username=self.username, password=self.password, secret=self.secret, - napalm_driver=self.platform.napalm_driver if self.platform and self.platform.napalm_driver else None, + napalm_driver=( + self.platform.napalm_driver + if self.platform and self.platform.napalm_driver + else None + ), optional_args=( - self.platform.napalm_args if self.platform and self.platform.napalm_args else settings.NAPALM_ARGS + self.platform.napalm_args + if self.platform and self.platform.napalm_args + else settings.NAPALM_ARGS ), ) netdev.get_onboarding_facts() @@ -159,10 +191,14 @@ def _onboard(self, address): "netdev_mgmt_ip_address": address, "netdev_nb_location_name": self.location.name, "netdev_nb_device_type_name": self.device_type, - "netdev_nb_role_name": self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"], + "netdev_nb_role_name": ( + self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"] + ), "netdev_nb_role_color": PLUGIN_SETTINGS["default_device_role_color"], "netdev_nb_platform_name": self.platform.name if self.platform else None, - "netdev_nb_credentials": self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None, + "netdev_nb_credentials": ( + self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None + ), # Kwargs discovered on the Onboarded Device: "netdev_hostname": netdev_dict["netdev_hostname"], "netdev_vendor": netdev_dict["netdev_vendor"], @@ -175,16 +211,24 @@ def _onboard(self, address): "driver_addon_result": netdev_dict["driver_addon_result"], } onboarding_cls = netdev_dict["onboarding_class"]() - onboarding_cls.credentials = {"username": self.username, "password": self.password, "secret": self.secret} + onboarding_cls.credentials = { + "username": self.username, + "password": self.password, + "secret": self.secret, + } onboarding_cls.run(onboarding_kwargs=onboarding_kwargs) self.logger.info( - "Successfully onboarded %s with a management IP of %s", netdev_dict["netdev_hostname"], address + "Successfully onboarded %s with a management IP of %s", + netdev_dict["netdev_hostname"], + address, ) def _parse_credentials(self, credentials): """Parse and return dictionary of credentials.""" if credentials: - self.logger.info("Attempting to parse credentials from selected SecretGroup") + self.logger.info( + "Attempting to parse credentials from selected SecretGroup" + ) try: self.username = credentials.get_secret_value( access_type=SecretsGroupAccessTypeChoices.TYPE_GENERIC, @@ -205,10 +249,14 @@ def _parse_credentials(self, credentials): self.logger.exception( "Unable to use SecretsGroup selected, ensure Access Type is set to Generic & at minimum Username & Password types are set." ) - raise OnboardException("fail-credentials - Unable to parse selected credentials.") from err + raise OnboardException( + "fail-credentials - Unable to parse selected credentials." + ) from err else: - self.logger.info("Using napalm credentials configured in nautobot_config.py") + self.logger.info( + "Using napalm credentials configured in nautobot_config.py" + ) self.username = settings.NAPALM_USERNAME self.password = settings.NAPALM_PASSWORD self.secret = settings.NAPALM_ARGS.get("secret", None) @@ -236,7 +284,9 @@ class Meta: description="Enable for more verbose logging.", ) csv_file = FileVar( - label="CSV File", required=False, description="If a file is provided all the options below will be ignored." + label="CSV File", + required=False, + description="If a file is provided all the options below will be ignored.", ) location = ObjectVar( model=Location, @@ -244,10 +294,12 @@ class Meta: query_params={"content_type": "dcim.device"}, description="Assigned Location for all synced device(s)", ) - namespace = ObjectVar(model=Namespace, required=False, description="Namespace ip addresses belong to.") + namespace = ObjectVar( + model=Namespace, required=False, description="Namespace ip addresses belong to." + ) ip_addresses = StringVar( required=False, - description="IP address of the device to sync, specify in a comma separated list for multiple devices.", + description="IP address or FQDN of the device to sync, specify in a comma separated list for multiple devices.", label="IPv4 addresses", ) port = IntegerVar(required=False, default=22) @@ -288,7 +340,9 @@ class Meta: description="Status to be applied to all new synced IP addresses. This value does not update with additional syncs.", ) secrets_group = ObjectVar( - model=SecretsGroup, required=False, description="SecretsGroup for device connection credentials." + model=SecretsGroup, + required=False, + description="SecretsGroup for device connection credentials.", ) platform = ObjectVar( model=Platform, @@ -333,11 +387,14 @@ def _process_csv_data(self, csv_file): query = f"location_name: {row.get('location_name')}, location_parent_name: {row.get('location_parent_name')}" if row.get("location_parent_name"): location = Location.objects.get( - name=row["location_name"].strip(), parent__name=row["location_parent_name"].strip() + name=row["location_name"].strip(), + parent__name=row["location_parent_name"].strip(), ) else: query = query = f"location_name: {row.get('location_name')}" - location = Location.objects.get(name=row["location_name"].strip(), parent=None) + location = Location.objects.get( + name=row["location_name"].strip(), parent=None + ) query = f"device_role: {row.get('device_role_name')}" device_role = Role.objects.get( name=row["device_role_name"].strip(), @@ -380,27 +437,47 @@ def _process_csv_data(self, csv_file): processed_csv_data[row["ip_address_host"]] = {} processed_csv_data[row["ip_address_host"]]["location"] = location processed_csv_data[row["ip_address_host"]]["namespace"] = namespace - processed_csv_data[row["ip_address_host"]]["port"] = int(row["port"].strip()) - processed_csv_data[row["ip_address_host"]]["timeout"] = int(row["timeout"].strip()) - processed_csv_data[row["ip_address_host"]]["set_mgmt_only"] = set_mgmgt_only - processed_csv_data[row["ip_address_host"]]["update_devices_without_primary_ip"] = ( - update_devices_without_primary_ip + processed_csv_data[row["ip_address_host"]]["port"] = int( + row["port"].strip() + ) + processed_csv_data[row["ip_address_host"]]["timeout"] = int( + row["timeout"].strip() ) + processed_csv_data[row["ip_address_host"]][ + "set_mgmt_only" + ] = set_mgmgt_only + processed_csv_data[row["ip_address_host"]][ + "update_devices_without_primary_ip" + ] = update_devices_without_primary_ip processed_csv_data[row["ip_address_host"]]["device_role"] = device_role - processed_csv_data[row["ip_address_host"]]["device_status"] = device_status - processed_csv_data[row["ip_address_host"]]["interface_status"] = interface_status - processed_csv_data[row["ip_address_host"]]["ip_address_status"] = ip_address_status - processed_csv_data[row["ip_address_host"]]["secrets_group"] = secrets_group + processed_csv_data[row["ip_address_host"]][ + "device_status" + ] = device_status + processed_csv_data[row["ip_address_host"]][ + "interface_status" + ] = interface_status + processed_csv_data[row["ip_address_host"]][ + "ip_address_status" + ] = ip_address_status + processed_csv_data[row["ip_address_host"]][ + "secrets_group" + ] = secrets_group processed_csv_data[row["ip_address_host"]]["platform"] = platform # Prepare ids to send to the job in celery self.task_kwargs_csv_data[row["ip_address_host"]] = {} - self.task_kwargs_csv_data[row["ip_address_host"]]["port"] = int(row["port"].strip()) - self.task_kwargs_csv_data[row["ip_address_host"]]["timeout"] = int(row["timeout"].strip()) + self.task_kwargs_csv_data[row["ip_address_host"]]["port"] = int( + row["port"].strip() + ) + self.task_kwargs_csv_data[row["ip_address_host"]]["timeout"] = int( + row["timeout"].strip() + ) self.task_kwargs_csv_data[row["ip_address_host"]]["secrets_group"] = ( secrets_group.id if secrets_group else "" ) - self.task_kwargs_csv_data[row["ip_address_host"]]["platform"] = platform.id if platform else "" + self.task_kwargs_csv_data[row["ip_address_host"]]["platform"] = ( + platform.id if platform else "" + ) row_count += 1 except ObjectDoesNotExist as err: self.logger.error(f"(row {sum([row_count, 1])}), {err} {query}") @@ -452,9 +529,14 @@ def run( for ip_address in self.processed_csv_data: self.ip_addresses.append(ip_address) # prepare the task_kwargs needed by the CommandGetterDO job - self.job_result.task_kwargs = {"debug": debug, "csv_file": self.task_kwargs_csv_data} + self.job_result.task_kwargs = { + "debug": debug, + "csv_file": self.task_kwargs_csv_data, + } else: - raise ValidationError(message="CSV check failed. No devices will be synced.") + raise ValidationError( + message="CSV check failed. No devices will be synced." + ) else: # Verify that all requried form inputs have been provided @@ -472,13 +554,19 @@ def run( } missing_required_inputs = [ - form_field for form_field, input_value in required_inputs.items() if not input_value + form_field + for form_field, input_value in required_inputs.items() + if not input_value ] if not missing_required_inputs: pass else: - self.logger.error(f"Missing requried inputs from job form: {missing_required_inputs}") - raise ValidationError(message=f"Missing required inputs {missing_required_inputs}") + self.logger.error( + f"Missing requried inputs from job form: {missing_required_inputs}" + ) + raise ValidationError( + message=f"Missing required inputs {missing_required_inputs}" + ) self.location = location self.namespace = namespace @@ -532,11 +620,19 @@ class Meta: description = "Synchronize extended device attribute information into Nautobot from one or more network devices. Information includes Interfaces, IP Addresses, Prefixes, VLANs and VRFs." debug = BooleanVar(description="Enable for more verbose logging.") - sync_vlans = BooleanVar(default=False, description="Sync VLANs and interface VLAN assignments.") - sync_vrfs = BooleanVar(default=False, description="Sync VRFs and interface VRF assignments.") - sync_cables = BooleanVar(default=False, description="Sync cables between interfaces via a LLDP or CDP.") + sync_vlans = BooleanVar( + default=False, description="Sync VLANs and interface VLAN assignments." + ) + sync_vrfs = BooleanVar( + default=False, description="Sync VRFs and interface VRF assignments." + ) + sync_cables = BooleanVar( + default=False, description="Sync cables between interfaces via a LLDP or CDP." + ) namespace = ObjectVar( - model=Namespace, required=True, description="The namespace for all IP addresses created or updated in the sync." + model=Namespace, + required=True, + description="The namespace for all IP addresses created or updated in the sync.", ) interface_status = ObjectVar( model=Status, @@ -585,7 +681,9 @@ def load_source_adapter(self): """Load network data adapter.""" # do not load source data if the job form does not filter which devices to sync if self.filtered_devices: - self.source_adapter = SyncNetworkDataNetworkAdapter(job=self, sync=self.sync) + self.source_adapter = SyncNetworkDataNetworkAdapter( + job=self, sync=self.sync + ) self.source_adapter.load() def load_target_adapter(self): @@ -632,7 +730,9 @@ def run( if self.debug: self.logger.debug("Checking for last_network_data_sync custom field") try: - cf = CustomField.objects.get(key="last_network_data_sync") # pylint:disable=invalid-name + cf = CustomField.objects.get( # pylint:disable=invalid-name + key="last_network_data_sync" + ) except ObjectDoesNotExist: cf, _ = CustomField.objects.get_or_create( # pylint:disable=invalid-name label="Last Network Data Sync", @@ -646,7 +746,9 @@ def run( if self.debug: self.logger.debug("Custom field found or created") except Exception as err: # pylint: disable=broad-exception-caught - self.logger.error(f"Failed to get or create last_network_data_sync custom field, {err}") + self.logger.error( + f"Failed to get or create last_network_data_sync custom field, {err}" + ) return # Filter devices based on form input @@ -662,7 +764,9 @@ def run( if device_filter: # prevent all devices from being returned by an empty filter self.filtered_devices = Device.objects.filter(**device_filter) else: - self.logger.error("No device filter options were provided, no devices will be synced.") + self.logger.error( + "No device filter options were provided, no devices will be synced." + ) # Stop the job if no devices are returned after filtering if not self.filtered_devices: @@ -670,7 +774,9 @@ def run( return # Log the devices that will be synced - filtered_devices_names = list(self.filtered_devices.values_list("name", flat=True)) + filtered_devices_names = list( + self.filtered_devices.values_list("name", flat=True) + ) self.logger.info(f"{len(filtered_devices_names)} devices will be synced") if len(filtered_devices_names) <= 300: self.logger.info(f"Devices: {filtered_devices_names}") @@ -716,7 +822,9 @@ def run(self, *args, **kwargs): # pragma: no cover ip_addresses = kwargs["ip_addresses"].replace(" ", "").split(",") port = kwargs["port"] platform = kwargs["platform"] - username, password, secret = _parse_credentials(kwargs["secrets_group"]) # pylint:disable=unused-variable + username, password, secret = ( # pylint:disable=unused-variable + _parse_credentials(kwargs["secrets_group"]) + ) # Initiate Nornir instance with empty inventory compiled_results = {} @@ -734,21 +842,27 @@ def run(self, *args, **kwargs): # pragma: no cover entered_ip, platform, port, username, password ) nornir_obj.inventory.hosts.update(single_host_inventory_constructed) - nr_with_processors = nornir_obj.with_processors([TroubleshootingProcessor(compiled_results)]) + nr_with_processors = nornir_obj.with_processors( + [TroubleshootingProcessor(compiled_results)] + ) if kwargs["ssot_job_type"] == "both": kwargs.update({"sync_vrfs": True}) kwargs.update({"sync_vlans": True}) kwargs.update({"sync_cables": True}) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], + command_getter_yaml_data=nornir_obj.inventory.defaults.data[ + "platform_parsing_info" + ], command_getter_job="sync_devices", logger=logger, **kwargs, ) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], + command_getter_yaml_data=nornir_obj.inventory.defaults.data[ + "platform_parsing_info" + ], command_getter_job="sync_network_data", logger=logger, **kwargs, @@ -760,7 +874,9 @@ def run(self, *args, **kwargs): # pragma: no cover kwargs.update({"sync_cables": True}) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], + command_getter_yaml_data=nornir_obj.inventory.defaults.data[ + "platform_parsing_info" + ], command_getter_job=kwargs["ssot_job_type"], logger=logger, **kwargs, @@ -771,5 +887,10 @@ def run(self, *args, **kwargs): # pragma: no cover return f"Successfully ran the following commands: {', '.join(list(compiled_results.keys()))}" -jobs = [OnboardingTask, SSOTSyncDevices, SSOTSyncNetworkData, DeviceOnboardingTroubleshootingJob] +jobs = [ + OnboardingTask, + SSOTSyncDevices, + SSOTSyncNetworkData, + DeviceOnboardingTroubleshootingJob, +] register_jobs(*jobs) From 9d3e485fc7f2269f7a39076b40858579a289f950 Mon Sep 17 00:00:00 2001 From: Susan Hooks Date: Fri, 11 Oct 2024 16:08:32 +0000 Subject: [PATCH 5/7] ruff formatting --- .../adapters/sync_devices_adapters.py | 66 +++++-------------- 1 file changed, 16 insertions(+), 50 deletions(-) diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py index ec0bab21..2c793adb 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py @@ -57,9 +57,7 @@ def get_from_orm_cache(self, parameters: Dict, model_class: Type[Model]): return cached_object # As we are using `get` here, this will error if there is not exactly one object that corresponds to the # parameter set. We intentionally pass these errors through. - self._cache[model_cache_key][parameter_set] = model_class.objects.get( - **dict(parameter_set) - ) + self._cache[model_cache_key][parameter_set] = model_class.objects.get(**dict(parameter_set)) return self._cache[model_cache_key][parameter_set] def load_manufacturers(self): @@ -67,9 +65,7 @@ def load_manufacturers(self): for manufacturer in Manufacturer.objects.all(): if self.job.debug: self.job.logger.debug("Loading Manufacturer data from Nautobot...") - onboarding_manufacturer = self.manufacturer( - adapter=self, pk=manufacturer.pk, name=manufacturer.name - ) + onboarding_manufacturer = self.manufacturer(adapter=self, pk=manufacturer.pk, name=manufacturer.name) self.add(onboarding_manufacturer) if self.job.debug: self.job.logger.debug(f"Manufacturer: {manufacturer.name} loaded.") @@ -83,12 +79,8 @@ def load_platforms(self): adapter=self, pk=platform.pk, name=platform.name, - network_driver=( - platform.network_driver if platform.network_driver else "" - ), - manufacturer__name=( - platform.manufacturer.name if platform.manufacturer else None - ), + network_driver=(platform.network_driver if platform.network_driver else ""), + manufacturer__name=(platform.manufacturer.name if platform.manufacturer else None), ) self.add(onboarding_platform) if self.job.debug: @@ -115,9 +107,7 @@ def load_devices(self): if self.job.debug: self.job.logger.debug("Loading Device data from Nautobot...") - for device in Device.objects.filter( - primary_ip4__host__in=self.job.ip_addresses - ): + for device in Device.objects.filter(primary_ip4__host__in=self.job.ip_addresses): interface_list = [] # Only interfaces with the device's primary ip should be considered for diff calculations # Ultimately, only the first matching interface is used but this list could support multiple @@ -138,18 +128,12 @@ def load_devices(self): name=device.name, platform__name=device.platform.name if device.platform else "", primary_ip4__host=device.primary_ip4.host if device.primary_ip4 else "", - primary_ip4__status__name=( - device.primary_ip4.status.name if device.primary_ip4 else "" - ), + primary_ip4__status__name=(device.primary_ip4.status.name if device.primary_ip4 else ""), role__name=device.role.name, status__name=device.status.name, - secrets_group__name=( - device.secrets_group.name if device.secrets_group else "" - ), + secrets_group__name=(device.secrets_group.name if device.secrets_group else ""), interfaces=interfaces, - mask_length=( - device.primary_ip4.mask_length if device.primary_ip4 else None - ), + mask_length=(device.primary_ip4.mask_length if device.primary_ip4 else None), serial=device.serial, ) self.add(onboarding_device) @@ -195,9 +179,7 @@ def _validate_ip_addresses(self, ip_addresses): self.job.logger.info(f"[{ip_address}] resolved to [{resolved_ip}]") ip_addresses[i] = resolved_ip except socket.gaierror: - self.job.logger.error( - f"[{ip_address}] is not a valid IP Address or name." - ) + self.job.logger.error(f"[{ip_address}] is not a valid IP Address or name.") validation_successful = False if validation_successful: return True @@ -214,9 +196,7 @@ def _handle_failed_devices(self, device_data): self.failed_ip_addresses = [] for ip_address in device_data: if not device_data[ip_address]: - self.job.logger.error( - f"{ip_address}: Connection or data error, this device will not be synced." - ) + self.job.logger.error(f"{ip_address}: Connection or data error, this device will not be synced.") self.failed_ip_addresses.append(ip_address) for ip_address in self.failed_ip_addresses: del device_data[ip_address] @@ -244,9 +224,7 @@ def execute_command_getter(self): self.job.logger.debug(f"Command Getter Result: {result}") data_type_check = diffsync_utils.check_data_type(result) if self.job.debug: - self.job.logger.debug( - f"CommandGetter data type check resut: {data_type_check}" - ) + self.job.logger.debug(f"CommandGetter data type check resut: {data_type_check}") if data_type_check: self._handle_failed_devices(device_data=result) else: @@ -338,9 +316,7 @@ def _fields_missing_data(self, device_data, ip_address, platform): "mask_length", "serial", ] - if ( - platform - ): # platform is only returned with device data if not provided on the job form/csv + if platform: # platform is only returned with device data if not provided on the job form/csv required_fields_from_device.append("platform") for field in required_fields_from_device: data = device_data[ip_address] @@ -353,9 +329,7 @@ def load_devices(self): for ip_address in self.device_data: if self.job.debug: self.job.logger.debug(f"loading device data for {ip_address}") - platform = ( - None # If an excption is caught below, the platform must still be set. - ) + platform = None # If an excption is caught below, the platform must still be set. onboarding_device = None try: location = diffsync_utils.retrieve_submitted_value( @@ -384,11 +358,7 @@ def load_devices(self): device_type__model=self.device_data[ip_address]["device_type"], location__name=location.name, name=self.device_data[ip_address]["hostname"], - platform__name=( - platform.name - if platform - else self.device_data[ip_address]["platform"] - ), + platform__name=(platform.name if platform else self.device_data[ip_address]["platform"]), primary_ip4__host=ip_address, primary_ip4__status__name=primary_ip4__status.name, role__name=device_role.name, @@ -422,9 +392,7 @@ def load_devices(self): try: self.add(onboarding_device) if self.job.debug: - self.job.logger.debug( - f"Device: {self.device_data[ip_address]['hostname']} loaded." - ) + self.job.logger.debug(f"Device: {self.device_data[ip_address]['hostname']} loaded.") except diffsync.ObjectAlreadyExists: self.job.logger.error( f"Device: {self.device_data[ip_address]['hostname']} has already been loaded! " @@ -435,9 +403,7 @@ def load_devices(self): else: self._add_ip_address_to_failed_list(ip_address=ip_address) if self.job.debug: - self.job.logger.debug( - f"{ip_address} was added to the failed ip_address list" - ) + self.job.logger.debug(f"{ip_address} was added to the failed ip_address list") def load(self): """Load network data.""" From bdbee35d94b607c996c8a2352c1bfd7cadc84ad8 Mon Sep 17 00:00:00 2001 From: scetron Date: Fri, 18 Oct 2024 15:35:56 -0400 Subject: [PATCH 6/7] fixed merge error, re-run linting --- .../adapters/sync_network_data_adapters.py | 163 ++++++------------ nautobot_device_onboarding/jinja_filters.py | 6 +- nautobot_device_onboarding/jobs.py | 142 ++++----------- 3 files changed, 87 insertions(+), 224 deletions(-) diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py index 294b1887..914afdf1 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_network_data_adapters.py @@ -31,10 +31,10 @@ class FilteredNautobotAdapter(NautobotAdapter): def _load_objects(self, diffsync_model): # pylint: disable=protected-access """Given a diffsync model class, load a list of models from the database and return them.""" parameter_names = self._get_parameter_names(diffsync_model) - for ( - database_object - ) in diffsync_model._get_queryset( # pylint: disable=protected-access - + for database_object in diffsync_model._get_queryset( # pylint: disable=protected-access + adapter=self + ): + self._load_single_object(database_object, diffsync_model, parameter_names) class SyncNetworkDataNautobotAdapter(FilteredNautobotAdapter): @@ -43,17 +43,11 @@ class SyncNetworkDataNautobotAdapter(FilteredNautobotAdapter): device = sync_network_data_models.SyncNetworkDataDevice interface = sync_network_data_models.SyncNetworkDataInterface ip_address = sync_network_data_models.SyncNetworkDataIPAddress - ipaddress_to_interface = ( - sync_network_data_models.SyncNetworkDataIPAddressToInterface - ) + ipaddress_to_interface = sync_network_data_models.SyncNetworkDataIPAddressToInterface vlan = sync_network_data_models.SyncNetworkDataVLAN vrf = sync_network_data_models.SyncNetworkDataVRF - tagged_vlans_to_interface = ( - sync_network_data_models.SyncNetworkDataTaggedVlansToInterface - ) - untagged_vlan_to_interface = ( - sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface - ) + tagged_vlans_to_interface = sync_network_data_models.SyncNetworkDataTaggedVlansToInterface + untagged_vlan_to_interface = sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface lag_to_interface = sync_network_data_models.SyncNetworkDataLagToInterface vrf_to_interface = sync_network_data_models.SyncNetworkDataVrfToInterface cable = sync_network_data_models.SyncNetworkDataCable @@ -109,9 +103,7 @@ def load_ip_addresses(self): if ip_address: ip_address_hosts.add(ip_address["ip_address"]) if "" in ip_address_hosts: - ip_address_hosts.remove( - "" - ) # do not attempt to filter ip addresses with empty strings + ip_address_hosts.remove("") # do not attempt to filter ip addresses with empty strings for ip_address in IPAddress.objects.filter( host__in=ip_address_hosts, parent__namespace__name=self.job.namespace.name, @@ -172,9 +164,7 @@ def load_tagged_vlans_to_interface(self): name=interface.name, tagged_vlans=tagged_vlans, ) - network_tagged_vlans_to_interface.model_flags = ( - DiffSyncModelFlags.SKIP_UNMATCHED_DST - ) + network_tagged_vlans_to_interface.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST self.add(network_tagged_vlans_to_interface) def load_untagged_vlan_to_interface(self): @@ -195,9 +185,7 @@ def load_untagged_vlan_to_interface(self): name=interface.name, untagged_vlan=untagged_vlan, ) - network_untagged_vlan_to_interface.model_flags = ( - DiffSyncModelFlags.SKIP_UNMATCHED_DST - ) + network_untagged_vlan_to_interface.model_flags = DiffSyncModelFlags.SKIP_UNMATCHED_DST self.add(network_untagged_vlan_to_interface) def load_lag_to_interface(self): @@ -262,10 +250,7 @@ def load_cables(self): """ for device in self.job.devices_to_load: for cable in device.get_cables(): - if ( - cable.termination_b.device.name == "" - or cable.termination_a.device.name == "" - ): + if cable.termination_b.device.name == "" or cable.termination_a.device.name == "": self.job.logger.warning( f"Device attached to a cable is missing a name. Devices must have a name to utilize cable onboarding. " f"Skipping Cable: {cable}" @@ -347,21 +332,15 @@ def sync_complete(self, source, diff, *args, **kwargs): This method only runs if data was changed. """ if self.job.debug: - self.job.logger.debug( - "Sync Complete method called, checking for missing primary ip addresses..." - ) - for ( - device - ) in self.job.devices_to_load.all(): # refresh queryset after sync is complete + self.job.logger.debug("Sync Complete method called, checking for missing primary ip addresses...") + for device in self.job.devices_to_load.all(): # refresh queryset after sync is complete if not device.primary_ip: ip_address = "" try: ip_address = IPAddress.objects.get(id=self.primary_ips[device.id]) device.primary_ip4 = ip_address device.validated_save() - self.job.logger.info( - f"Assigning {ip_address} as primary IP Address for Device: {device.name}" - ) + self.job.logger.info(f"Assigning {ip_address} as primary IP Address for Device: {device.name}") except Exception as err: # pylint: disable=broad-exception-caught self.job.logger.error( f"Unable to set Primary IP for {device.name}, {err.args}. " @@ -369,9 +348,7 @@ def sync_complete(self, source, diff, *args, **kwargs): ) if ip_address: try: - interface = Interface.objects.get( - device=device, ip_addresses__in=[ip_address] - ) + interface = Interface.objects.get(device=device, ip_addresses__in=[ip_address]) interface.mgmt_only = True interface.validated_save() self.job.logger.info( @@ -407,17 +384,11 @@ def __init__(self, *args, job, sync=None, **kwargs): device = sync_network_data_models.SyncNetworkDataDevice interface = sync_network_data_models.SyncNetworkDataInterface ip_address = sync_network_data_models.SyncNetworkDataIPAddress - ipaddress_to_interface = ( - sync_network_data_models.SyncNetworkDataIPAddressToInterface - ) + ipaddress_to_interface = sync_network_data_models.SyncNetworkDataIPAddressToInterface vlan = sync_network_data_models.SyncNetworkDataVLAN vrf = sync_network_data_models.SyncNetworkDataVRF - tagged_vlans_to_interface = ( - sync_network_data_models.SyncNetworkDataTaggedVlansToInterface - ) - untagged_vlan_to_interface = ( - sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface - ) + tagged_vlans_to_interface = sync_network_data_models.SyncNetworkDataTaggedVlansToInterface + untagged_vlan_to_interface = sync_network_data_models.SyncNetworkDataUnTaggedVlanToInterface lag_to_interface = sync_network_data_models.SyncNetworkDataLagToInterface vrf_to_interface = sync_network_data_models.SyncNetworkDataVrfToInterface cable = sync_network_data_models.SyncNetworkDataCable @@ -455,10 +426,8 @@ def _handle_failed_devices(self, device_data): for hostname in failed_devices: del device_data[hostname] - device_queryset, devices_with_errors = ( - diffsync_utils.generate_device_queryset_from_command_getter_result( - job=self.job, command_getter_result=device_data - ) + device_queryset, devices_with_errors = diffsync_utils.generate_device_queryset_from_command_getter_result( + job=self.job, command_getter_result=device_data ) # remove devices that have errors found while creating the queryset from the command getter results @@ -474,9 +443,7 @@ def _handle_failed_devices(self, device_data): def _handle_general_load_exception(self, error, hostname, data, model_type): """If a diffsync model fails to load, log the error.""" - self.job.logger.error( - f"Failed to load {model_type} model for {hostname}. {error}, {error.args}" - ) + self.job.logger.error(f"Failed to load {model_type} model for {hostname}. {error}, {error.args}") if self.job.debug: self.job.logger.debug(f"HOSTNAME: {hostname}, DATA: {data}") @@ -490,9 +457,7 @@ def execute_command_getter(self): # verify data returned is a dict data_type_check = diffsync_utils.check_data_type(result) if self.job.debug: - self.job.logger.debug( - f"CommandGetter data type check result: {data_type_check}" - ) + self.job.logger.debug(f"CommandGetter data type check result: {data_type_check}") if data_type_check: self._handle_failed_devices(device_data=result) else: @@ -519,15 +484,11 @@ def load_devices(self): ) self.add(network_device) except Exception as err: # pylint: disable=broad-exception-caught - self._handle_general_load_exception( - error=err, hostname=hostname, data=device_data, model_type="device" - ) + self._handle_general_load_exception(error=err, hostname=hostname, data=device_data, model_type="device") continue # for interface in device_data["interfaces"]: for interface_name, interface_data in device_data["interfaces"].items(): - network_interface = self.load_interface( - hostname, interface_name, interface_data - ) + network_interface = self.load_interface(hostname, interface_name, interface_data) network_device.add_child(network_interface) # def _get_vlan_name(self, interface_data): @@ -545,9 +506,7 @@ def load_interface(self, hostname, interface_name, interface_data): device__name=hostname, status__name=self.job.interface_status.name, type=interface_data["type"], - mac_address=self._process_mac_address( - mac_address=interface_data["mac_address"] - ), + mac_address=self._process_mac_address(mac_address=interface_data["mac_address"]), mtu=interface_data["mtu"] if interface_data["mtu"] else "1500", description=interface_data["description"], enabled=interface_data["link_status"], @@ -571,13 +530,9 @@ def load_ip_addresses(self): for interface_name, interface_data in device_data["interfaces"].items(): if interface_data["ip_addresses"]: for ip_address in interface_data["ip_addresses"]: - if ip_address[ - "ip_address" - ]: # the ip_address and mask_length may be empty, skip these + if ip_address["ip_address"]: # the ip_address and mask_length may be empty, skip these if self.job.debug: - self.job.logger.debug( - f"Loading {ip_address} from {interface_name} on {hostname}" - ) + self.job.logger.debug(f"Loading {ip_address} from {interface_name} on {hostname}") try: network_ip_address = self.ip_address( adapter=self, @@ -614,9 +569,7 @@ def load_vlans(self): for ( hostname, device_data, - ) in ( - self.job.command_getter_result.items() - ): # pylint: disable=too-many-nested-blocks + ) in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks if self.job.debug: self.job.logger.debug(f"Loading Vlans from {hostname}") # for interface in device_data["interfaces"]: @@ -667,9 +620,7 @@ def load_vrfs(self): for ( hostname, device_data, - ) in ( - self.job.command_getter_result.items() - ): # pylint: disable=too-many-nested-blocks + ) in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks if self.job.debug: self.job.logger.debug(f"Loading Vrfs from {hostname}") # for interface in device_data["interfaces"]: @@ -698,27 +649,19 @@ def load_ip_address_to_interfaces(self): for ( hostname, device_data, - ) in ( - self.job.command_getter_result.items() - ): # pylint: disable=too-many-nested-blocks + ) in self.job.command_getter_result.items(): # pylint: disable=too-many-nested-blocks for interface_name, interface_data in device_data["interfaces"].items(): for ip_address in interface_data["ip_addresses"]: - if ip_address[ - "ip_address" - ]: # the ip_address and mask_length may be empty, skip these + if ip_address["ip_address"]: # the ip_address and mask_length may be empty, skip these try: - network_ip_address_to_interface = ( - self.ipaddress_to_interface( - adapter=self, - interface__device__name=hostname, - interface__name=interface_name, - ip_address__host=ip_address["ip_address"], - ip_address__mask_length=( - int(ip_address["prefix_length"]) - if ip_address["prefix_length"] - else None - ), - ) + network_ip_address_to_interface = self.ipaddress_to_interface( + adapter=self, + interface__device__name=hostname, + interface__name=interface_name, + ip_address__host=ip_address["ip_address"], + ip_address__mask_length=( + int(ip_address["prefix_length"]) if ip_address["prefix_length"] else None + ), ) self.add(network_ip_address_to_interface) except ( @@ -760,13 +703,11 @@ def load_untagged_vlan_to_interface(self): # for interface in device_data["interfaces"]: for interface_name, interface_data in device_data["interfaces"].items(): try: - network_untagged_vlan_to_interface = ( - self.untagged_vlan_to_interface( - adapter=self, - device__name=hostname, - name=interface_name, - untagged_vlan=interface_data["untagged_vlan"], - ) + network_untagged_vlan_to_interface = self.untagged_vlan_to_interface( + adapter=self, + device__name=hostname, + name=interface_name, + untagged_vlan=interface_data["untagged_vlan"], ) self.add(network_untagged_vlan_to_interface) except Exception as err: # pylint: disable=broad-exception-caught @@ -788,9 +729,7 @@ def load_lag_to_interface(self): adapter=self, device__name=hostname, name=interface_name, - lag__interface__name=( - interface_data["lag"] if interface_data["lag"] else "" - ), + lag__interface__name=(interface_data["lag"] if interface_data["lag"] else ""), ) self.add(network_lag_to_interface) except Exception as err: # pylint: disable=broad-exception-caught @@ -828,9 +767,7 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements """Load cables into the Diffsync store.""" for hostname, device_data in self.job.command_getter_result.items(): if "cables" not in device_data: - self.job.logger.warning( - f"No cable data found for {hostname}. Skipping cable load." - ) + self.job.logger.warning(f"No cable data found for {hostname}. Skipping cable load.") return if self.job.debug: self.job.logger.debug(f"Loading Cables from {hostname}") @@ -841,9 +778,7 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements try: for neighbor_data in device_data["cables"]: try: - local_interface = canonical_interface_name( - neighbor_data["local_interface"] - ) + local_interface = canonical_interface_name(neighbor_data["local_interface"]) except KeyError as error: error.args = error.args + "Local interface is missing a name." self._handle_general_load_exception( @@ -855,9 +790,7 @@ def load_cables(self): # pylint: disable=inconsistent-return-statements continue try: - remote_interface = canonical_interface_name( - neighbor_data["remote_interface"] - ) + remote_interface = canonical_interface_name(neighbor_data["remote_interface"]) except KeyError as error: error.args = error.args + "Remote interface is missing a name." self._handle_general_load_exception( diff --git a/nautobot_device_onboarding/jinja_filters.py b/nautobot_device_onboarding/jinja_filters.py index 17792c8e..aadd8129 100755 --- a/nautobot_device_onboarding/jinja_filters.py +++ b/nautobot_device_onboarding/jinja_filters.py @@ -193,8 +193,8 @@ def junos_get_valid_interfaces(interfaces): """Get valid interfaces from Junos.""" result = {} for interface in interfaces: - result[interface['name']] = {} - if interface['units']: - for unit in interface['units']: + result[interface["name"]] = {} + if interface["units"]: + for unit in interface["units"]: result[f"{interface['name']}.{unit}"] = {} return result diff --git a/nautobot_device_onboarding/jobs.py b/nautobot_device_onboarding/jobs.py index 5c589664..400bf9af 100755 --- a/nautobot_device_onboarding/jobs.py +++ b/nautobot_device_onboarding/jobs.py @@ -172,15 +172,9 @@ def _onboard(self, address): username=self.username, password=self.password, secret=self.secret, - napalm_driver=( - self.platform.napalm_driver - if self.platform and self.platform.napalm_driver - else None - ), + napalm_driver=(self.platform.napalm_driver if self.platform and self.platform.napalm_driver else None), optional_args=( - self.platform.napalm_args - if self.platform and self.platform.napalm_args - else settings.NAPALM_ARGS + self.platform.napalm_args if self.platform and self.platform.napalm_args else settings.NAPALM_ARGS ), ) netdev.get_onboarding_facts() @@ -191,14 +185,10 @@ def _onboard(self, address): "netdev_mgmt_ip_address": address, "netdev_nb_location_name": self.location.name, "netdev_nb_device_type_name": self.device_type, - "netdev_nb_role_name": ( - self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"] - ), + "netdev_nb_role_name": (self.role.name if self.role else PLUGIN_SETTINGS["default_device_role"]), "netdev_nb_role_color": PLUGIN_SETTINGS["default_device_role_color"], "netdev_nb_platform_name": self.platform.name if self.platform else None, - "netdev_nb_credentials": ( - self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None - ), + "netdev_nb_credentials": (self.credentials if PLUGIN_SETTINGS["assign_secrets_group"] else None), # Kwargs discovered on the Onboarded Device: "netdev_hostname": netdev_dict["netdev_hostname"], "netdev_vendor": netdev_dict["netdev_vendor"], @@ -226,9 +216,7 @@ def _onboard(self, address): def _parse_credentials(self, credentials): """Parse and return dictionary of credentials.""" if credentials: - self.logger.info( - "Attempting to parse credentials from selected SecretGroup" - ) + self.logger.info("Attempting to parse credentials from selected SecretGroup") try: self.username = credentials.get_secret_value( access_type=SecretsGroupAccessTypeChoices.TYPE_GENERIC, @@ -249,14 +237,10 @@ def _parse_credentials(self, credentials): self.logger.exception( "Unable to use SecretsGroup selected, ensure Access Type is set to Generic & at minimum Username & Password types are set." ) - raise OnboardException( - "fail-credentials - Unable to parse selected credentials." - ) from err + raise OnboardException("fail-credentials - Unable to parse selected credentials.") from err else: - self.logger.info( - "Using napalm credentials configured in nautobot_config.py" - ) + self.logger.info("Using napalm credentials configured in nautobot_config.py") self.username = settings.NAPALM_USERNAME self.password = settings.NAPALM_PASSWORD self.secret = settings.NAPALM_ARGS.get("secret", None) @@ -294,9 +278,7 @@ class Meta: query_params={"content_type": "dcim.device"}, description="Assigned Location for all synced device(s)", ) - namespace = ObjectVar( - model=Namespace, required=False, description="Namespace ip addresses belong to." - ) + namespace = ObjectVar(model=Namespace, required=False, description="Namespace ip addresses belong to.") ip_addresses = StringVar( required=False, description="IP address or FQDN of the device to sync, specify in a comma separated list for multiple devices.", @@ -392,9 +374,7 @@ def _process_csv_data(self, csv_file): ) else: query = query = f"location_name: {row.get('location_name')}" - location = Location.objects.get( - name=row["location_name"].strip(), parent=None - ) + location = Location.objects.get(name=row["location_name"].strip(), parent=None) query = f"device_role: {row.get('device_role_name')}" device_role = Role.objects.get( name=row["device_role_name"].strip(), @@ -437,47 +417,27 @@ def _process_csv_data(self, csv_file): processed_csv_data[row["ip_address_host"]] = {} processed_csv_data[row["ip_address_host"]]["location"] = location processed_csv_data[row["ip_address_host"]]["namespace"] = namespace - processed_csv_data[row["ip_address_host"]]["port"] = int( - row["port"].strip() + processed_csv_data[row["ip_address_host"]]["port"] = int(row["port"].strip()) + processed_csv_data[row["ip_address_host"]]["timeout"] = int(row["timeout"].strip()) + processed_csv_data[row["ip_address_host"]]["set_mgmt_only"] = set_mgmgt_only + processed_csv_data[row["ip_address_host"]]["update_devices_without_primary_ip"] = ( + update_devices_without_primary_ip ) - processed_csv_data[row["ip_address_host"]]["timeout"] = int( - row["timeout"].strip() - ) - processed_csv_data[row["ip_address_host"]][ - "set_mgmt_only" - ] = set_mgmgt_only - processed_csv_data[row["ip_address_host"]][ - "update_devices_without_primary_ip" - ] = update_devices_without_primary_ip processed_csv_data[row["ip_address_host"]]["device_role"] = device_role - processed_csv_data[row["ip_address_host"]][ - "device_status" - ] = device_status - processed_csv_data[row["ip_address_host"]][ - "interface_status" - ] = interface_status - processed_csv_data[row["ip_address_host"]][ - "ip_address_status" - ] = ip_address_status - processed_csv_data[row["ip_address_host"]][ - "secrets_group" - ] = secrets_group + processed_csv_data[row["ip_address_host"]]["device_status"] = device_status + processed_csv_data[row["ip_address_host"]]["interface_status"] = interface_status + processed_csv_data[row["ip_address_host"]]["ip_address_status"] = ip_address_status + processed_csv_data[row["ip_address_host"]]["secrets_group"] = secrets_group processed_csv_data[row["ip_address_host"]]["platform"] = platform # Prepare ids to send to the job in celery self.task_kwargs_csv_data[row["ip_address_host"]] = {} - self.task_kwargs_csv_data[row["ip_address_host"]]["port"] = int( - row["port"].strip() - ) - self.task_kwargs_csv_data[row["ip_address_host"]]["timeout"] = int( - row["timeout"].strip() - ) + self.task_kwargs_csv_data[row["ip_address_host"]]["port"] = int(row["port"].strip()) + self.task_kwargs_csv_data[row["ip_address_host"]]["timeout"] = int(row["timeout"].strip()) self.task_kwargs_csv_data[row["ip_address_host"]]["secrets_group"] = ( secrets_group.id if secrets_group else "" ) - self.task_kwargs_csv_data[row["ip_address_host"]]["platform"] = ( - platform.id if platform else "" - ) + self.task_kwargs_csv_data[row["ip_address_host"]]["platform"] = platform.id if platform else "" row_count += 1 except ObjectDoesNotExist as err: self.logger.error(f"(row {sum([row_count, 1])}), {err} {query}") @@ -534,9 +494,7 @@ def run( "csv_file": self.task_kwargs_csv_data, } else: - raise ValidationError( - message="CSV check failed. No devices will be synced." - ) + raise ValidationError(message="CSV check failed. No devices will be synced.") else: # Verify that all requried form inputs have been provided @@ -554,19 +512,13 @@ def run( } missing_required_inputs = [ - form_field - for form_field, input_value in required_inputs.items() - if not input_value + form_field for form_field, input_value in required_inputs.items() if not input_value ] if not missing_required_inputs: pass else: - self.logger.error( - f"Missing requried inputs from job form: {missing_required_inputs}" - ) - raise ValidationError( - message=f"Missing required inputs {missing_required_inputs}" - ) + self.logger.error(f"Missing requried inputs from job form: {missing_required_inputs}") + raise ValidationError(message=f"Missing required inputs {missing_required_inputs}") self.location = location self.namespace = namespace @@ -620,15 +572,9 @@ class Meta: description = "Synchronize extended device attribute information into Nautobot from one or more network devices. Information includes Interfaces, IP Addresses, Prefixes, VLANs and VRFs." debug = BooleanVar(description="Enable for more verbose logging.") - sync_vlans = BooleanVar( - default=False, description="Sync VLANs and interface VLAN assignments." - ) - sync_vrfs = BooleanVar( - default=False, description="Sync VRFs and interface VRF assignments." - ) - sync_cables = BooleanVar( - default=False, description="Sync cables between interfaces via a LLDP or CDP." - ) + sync_vlans = BooleanVar(default=False, description="Sync VLANs and interface VLAN assignments.") + sync_vrfs = BooleanVar(default=False, description="Sync VRFs and interface VRF assignments.") + sync_cables = BooleanVar(default=False, description="Sync cables between interfaces via a LLDP or CDP.") namespace = ObjectVar( model=Namespace, required=True, @@ -681,9 +627,7 @@ def load_source_adapter(self): """Load network data adapter.""" # do not load source data if the job form does not filter which devices to sync if self.filtered_devices: - self.source_adapter = SyncNetworkDataNetworkAdapter( - job=self, sync=self.sync - ) + self.source_adapter = SyncNetworkDataNetworkAdapter(job=self, sync=self.sync) self.source_adapter.load() def load_target_adapter(self): @@ -746,9 +690,7 @@ def run( if self.debug: self.logger.debug("Custom field found or created") except Exception as err: # pylint: disable=broad-exception-caught - self.logger.error( - f"Failed to get or create last_network_data_sync custom field, {err}" - ) + self.logger.error(f"Failed to get or create last_network_data_sync custom field, {err}") return # Filter devices based on form input @@ -764,9 +706,7 @@ def run( if device_filter: # prevent all devices from being returned by an empty filter self.filtered_devices = Device.objects.filter(**device_filter) else: - self.logger.error( - "No device filter options were provided, no devices will be synced." - ) + self.logger.error("No device filter options were provided, no devices will be synced.") # Stop the job if no devices are returned after filtering if not self.filtered_devices: @@ -774,9 +714,7 @@ def run( return # Log the devices that will be synced - filtered_devices_names = list( - self.filtered_devices.values_list("name", flat=True) - ) + filtered_devices_names = list(self.filtered_devices.values_list("name", flat=True)) self.logger.info(f"{len(filtered_devices_names)} devices will be synced") if len(filtered_devices_names) <= 300: self.logger.info(f"Devices: {filtered_devices_names}") @@ -842,27 +780,21 @@ def run(self, *args, **kwargs): # pragma: no cover entered_ip, platform, port, username, password ) nornir_obj.inventory.hosts.update(single_host_inventory_constructed) - nr_with_processors = nornir_obj.with_processors( - [TroubleshootingProcessor(compiled_results)] - ) + nr_with_processors = nornir_obj.with_processors([TroubleshootingProcessor(compiled_results)]) if kwargs["ssot_job_type"] == "both": kwargs.update({"sync_vrfs": True}) kwargs.update({"sync_vlans": True}) kwargs.update({"sync_cables": True}) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data[ - "platform_parsing_info" - ], + command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], command_getter_job="sync_devices", logger=logger, **kwargs, ) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data[ - "platform_parsing_info" - ], + command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], command_getter_job="sync_network_data", logger=logger, **kwargs, @@ -874,9 +806,7 @@ def run(self, *args, **kwargs): # pragma: no cover kwargs.update({"sync_cables": True}) nr_with_processors.run( task=netmiko_send_commands, - command_getter_yaml_data=nornir_obj.inventory.defaults.data[ - "platform_parsing_info" - ], + command_getter_yaml_data=nornir_obj.inventory.defaults.data["platform_parsing_info"], command_getter_job=kwargs["ssot_job_type"], logger=logger, **kwargs, From bbbfc40b24b2a5cfa4fd90f47b9dcb638835d1de Mon Sep 17 00:00:00 2001 From: scetron Date: Fri, 18 Oct 2024 15:50:44 -0400 Subject: [PATCH 7/7] ruff again --- .../diffsync/adapters/sync_devices_adapters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py index 2c793adb..7dcc638e 100644 --- a/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py +++ b/nautobot_device_onboarding/diffsync/adapters/sync_devices_adapters.py @@ -1,9 +1,9 @@ """DiffSync adapters.""" +import socket from collections import defaultdict from typing import DefaultDict, Dict, FrozenSet, Hashable, Tuple, Type -import socket import diffsync import netaddr from django.contrib.contenttypes.models import ContentType