multi-RE junos fix

.github/workflows/ci.yml

@@ -134,9 +134,9 @@ jobs:
           - python-version: "3.11"
             db-backend: "postgresql"
             nautobot-version: "2.2.3"
-          # - python-version: "3.11"
-          #   db-backend: "mysql"
-          #   nautobot-version: "stable"
+            # - python-version: "3.11"
+            #   db-backend: "mysql"
+            #   nautobot-version: "stable"
     runs-on: "ubuntu-22.04"
     env:
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}"

docs/user/faq.md

@@ -74,3 +74,10 @@ Optional arguments are often used to define a `secret` for Cisco devices and oth
 ## Why don't I see a webhook generated when a new device is onboarded successfully?
 
 It's expected that any changes done asynchronously in Nautobot currently (within a worker) will not generate a webhook.
+
+## Why should I limit as much as possible the initial job?
+
+- No access to "all" data, such as config context.
+- Several assumptions made for Nornir inventory that would be different in all other Nornir inventory jobs.
+- An inventory created for each device.
+    - Causes additional SQL connections which may benefit from the use of `serial` runner.

nautobot_device_onboarding/command_mappers/juniper_junos.yml

@@ -2,19 +2,22 @@
 sync_devices:
   hostname:
     commands:
-      - command: "show version | display json"
+      - command: "show system information | display json"
         parser: "none"
-        jpath: '"software-information"[]."host-name"[].data'  # yamllint disable-line rule:quoted-strings
+        jpath: '"system-information"[]."host-name"[].data'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | unique | first }}"
   serial:
     commands:
-      - command: "show chassis hardware | display json"
+      - command: "show system information | display json"
         parser: "none"
-        jpath: '"chassis-inventory"[]."chassis"[]."serial-number"[].data'  # yamllint disable-line rule:quoted-strings
+        jpath: '"system-information"[]."serial-number"[].data'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | unique | first }}"
   device_type:
     commands:
-      - command: "show chassis hardware | display json"
+      - command: "show system information | display json"
         parser: "none"
-        jpath: '"chassis-inventory"[]."chassis"[]."description"[].data'  # yamllint disable-line rule:quoted-strings
+        jpath: '"system-information"[]."hardware-model"[].data'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | unique | first | upper }}"
   mgmt_interface:
     commands:
       - command: "show interfaces terse | display json"
@@ -23,25 +26,26 @@ sync_devices:
         post_processor: "{% for entry in obj %}{% if entry['ip'] %}{% for ipaddr in entry['ip'] %}{% if original_host in ipaddr %}{{ entry['name'] | first }}{% endif %}{% endfor %}{% endif %}{% endfor %}"
   mask_length:
     commands:
-      - command: "show route protocol direct | display json"
+      - command: "show configuration interfaces | display json"
         parser: "none"
-        jpath: '"route-information"[]."route-table"[]."rt"[]."rt-destination"[].data'  # yamllint disable-line rule:quoted-strings
-        post_processor: "{% set mask = [] %}{% for ip_route in obj %}{% if ip_route | is_network %}{% if ip_route | ipaddress_network('version') == 4 %}{% if original_host | is_ip_within(ip_route) %}{% set _=mask.append(ip_route.split('/')[1]) %}{% endif %}{% endif %}{% endif %}{% endfor %}{{ mask | unique | first}}"
+        jpath: "configuration.interfaces.interface[].unit[?family.inet.address[?contains(name, `{{ obj }}`)]][].family.*.address[][].name"
+        post_processor: "{{ obj[0].split('/')[1] }}"
         iterable_type: "int"
 sync_network_data:
   serial:
     commands:
-      - command: "show chassis hardware | display json"
+      - command: "show system information | display json"
         parser: "none"
-        jpath: '"chassis-inventory"[]."chassis"[]."serial-number"[].data'  # yamllint disable-line rule:quoted-strings
+        jpath: '"system-information"[]."serial-number"[].data'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | unique | first }}"
   interfaces:
     root_key: true
     commands:
-      - command: "show interfaces | display json"
+      - command: "show configuration interfaces | display json"
         parser: "none"
         # when root_key=true this extracted value is what becomes interable in keys using __ under `current_key`.
-        jpath: '"interface-information"[]."physical-interface"[].[name[].data, "logical-interface"[].name[].data][][]'   # yamllint disable-line rule:quoted-strings
-        post_processor: "{% set result={} %}{% for interface in obj %}{{ result.update({interface: {}}) or '' }}{% endfor %}{{ result | tojson }}"
+        jpath: 'configuration.interfaces.interface[].{name: name, units:unit[].name}'   # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | junos_get_valid_interfaces | tojson }}"
   interfaces__type:
     commands:
       - command: "show interfaces | display json"

nautobot_device_onboarding/jinja_filters.py

@@ -186,3 +186,15 @@ def parse_junos_ip_address(item):
 def remove_fqdn(hostname):
     """Remove the FQDN from the hostname."""
     return hostname.split(".")[0]
+
+
+@library.filter
+def junos_get_valid_interfaces(interfaces):
+    """Get valid interfaces from Junos."""
+    result = {}
+    for interface in interfaces:
+        result[interface['name']] = {}
+        if interface['units']:
+            for unit in interface['units']:
+                result[f"{interface['name']}.{unit}"] = {}
+    return result

nautobot_device_onboarding/nornir_plays/formatter.py

@@ -175,7 +175,7 @@ def perform_data_extraction(host, command_info_dict, command_outputs_dict, job_d
                 if len(field_nesting) > 1:
                     # Means there is "anticipated" data nesting `interfaces__mtu` means final data would be
                     # {"Ethernet1/1": {"mtu": <value>}}
-                    for current_key in root_key_pre:
+                    for current_key in root_key_post:
                         # current_key is a single iteration from the root_key extracted value. Typically we want this to be
                         # a list of data that we want to become our nested key. E.g. current_key "Ethernet1/1"
                         # These get passed into the render context for the template render to allow nested jpaths to use