fix vlans, fix ci and app schemas

fix vlans across all platforms and more unittest

fix ci test to match new 2.2.3 lower constraint

merge tasks.py from develop

.github/workflows/ci.yml

@@ -106,7 +106,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.11"]
-        nautobot-version: ["2.2.2"]
+        nautobot-version: ["2.2.3"]
     env:
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -152,7 +152,7 @@ jobs:
         include:
           - python-version: "3.11"
             db-backend: "postgresql"
-            nautobot-version: "2.2.2"
+            nautobot-version: "2.2.3"
           # - python-version: "3.11"
           #   db-backend: "mysql"
           #   nautobot-version: "stable"

README.md

@@ -21,7 +21,7 @@ Regardless, the Onboarding App greatly simplifies the onboarding process by allo
 
 Device Onboarding is a Job that allows you to provide a few required pieces of information and onboard the device.
 
-![job input](https://raw.githubusercontent.com/nautobot/nautobot-app-device-onboarding/develop/docs/images/do_job_inputs.png)
+![job input](https://raw.githubusercontent.com/nautobot/nautobot-app-device-onboarding/develop/docs/images/sync_devices_inputs.png)
 
 ## Try it out!
 

app-config-schema.json

@@ -0,0 +1 @@
+true

changes/181.added

@@ -0,0 +1,5 @@
+Sync Devices from Network job was added which utilizes the SSoT framework to onboard devices.
+Sync Data from Network job was added which utilizes the SSoT framework to onboard devices.
+Git Datasource object to be able to use a Git Repo to overload new SSoT job YAML file definitions.
+Create a Nornir inventory `EmptyInventory` to support ondemand inventory population for `Sync Devices` job.
+Add `nautobot-app-nornir` dependency to reuse `NautobotORMInventory` to support inventory creation for `Sync Data` job.

changes/181.changed

@@ -0,0 +1 @@
+The `OnboardingTask` job is changed to `hidden` by default.

development/app_config_schema.py

@@ -0,0 +1,65 @@
+"""App Config Schema Generator and Validator."""
+
+import json
+from importlib import import_module
+from os import getenv
+from pathlib import Path
+from urllib.parse import urlparse
+
+import jsonschema
+import toml
+from django.conf import settings
+from to_json_schema.to_json_schema import SchemaBuilder
+
+
+def _enrich_object_schema(schema, defaults, required):
+    schema["additionalProperties"] = False
+    for key, value in schema["properties"].items():
+        if required and key in required:
+            value["required"] = True
+        default_value = defaults and defaults.get(key, None)
+        if value["type"] == "object" and "properties" in value:
+            _enrich_object_schema(value, default_value, None)
+        elif default_value is not None:
+            value["default"] = default_value
+
+
+def _main():
+    pyproject = toml.loads(Path("pyproject.toml").read_text())
+    url = urlparse(pyproject["tool"]["poetry"]["repository"])
+    _, owner, repository = url.path.split("/")
+    package_name = pyproject["tool"]["poetry"]["packages"][0]["include"]
+    app_config = settings.PLUGINS_CONFIG[package_name]  # type: ignore
+    schema_path = Path(package_name) / "app-config-schema.json"
+    command = getenv("APP_CONFIG_SCHEMA_COMMAND", "")
+    if command == "generate":
+        schema = {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json",
+            "$comment": "TBD: Update $id, replace `develop` with the future release tag",
+            **SchemaBuilder().to_json_schema(app_config),  # type: ignore
+        }
+        app_config = import_module(package_name).config
+        _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings)
+        schema_path.write_text(json.dumps(schema, indent=4) + "\n")
+        print(f"\n==================\nGenerated schema:\n\n{schema_path}\n")
+        print(
+            "WARNING: Review and edit the generated file before committing.\n"
+            "\n"
+            "Its content is inferred from:\n"
+            "\n"
+            "- The current configuration in `PLUGINS_CONFIG`\n"
+            "- `NautobotAppConfig.default_settings`\n"
+            "- `NautobotAppConfig.required_settings`"
+        )
+    elif command == "validate":
+        schema = json.loads(schema_path.read_text())
+        jsonschema.validate(app_config, schema)
+        print(
+            f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid."
+        )
+    else:
+        raise RuntimeError(f"Unknown command: {command}")
+
+
+_main()

development/towncrier_template.j2

@@ -1,6 +1,6 @@
 
 {% if render_title %}
-## v{{ versiondata.version }} ({{ versiondata.date }})
+## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/nautobot/nautobot-app-device-onboarding/releases/tag/v{{ versiondata.version}})
 
 {% endif %}
 {% for section, _ in sections.items() %}

docs/admin/release_notes/version_4.0.md

@@ -2,17 +2,3 @@
 
 !!! warning
     Nautobot Device Onboarding v4.0.0 completely revamps the applications design and framework. The original `OnboardingTask` job is still packaged with the app to provide a backwards compatible way for users that have used its extensions framework in the past to solve complex problems. However, that job is now hidden by default to avoid confusion with the two new SSoT based onboarding jobs that v4.0.0 exposes.
-
-## v4.0.0 TBD
-
-### Added
-
-- [#TBD](TBD) - Sync Devices from Network job was added which utilizes the SSoT framework to onboard devices.
-- [#TBD](TBD) - Sync Data from Network job was added which utilizes the SSoT framework to onboard devices.
-- [#TBD](TBD) - Git Datasource object to be able to use a Git Repo to overload new SSoT job YAML file definitions.
-- [#TBD](TBD) - Create a Nornir inventory `EmptyInventory` to support ondemand inventory population for `Sync Devices` job.
-- [#TBD](TBD) - Add `nautobot-app-nornir` dependency to reuse `NautobotORMInventory` to support inventory creation for `Sync Data` job.
-
-### Changed
-
-- [#TBD](tbd) - The `OnboardingTask` job is changed to `hidden` by default.

docs/user/app_detailed_design.md

@@ -52,7 +52,8 @@ To learn how to extend the app, or update its default YAML definitions visit [Ex
 
 ## Detailed Design Diagram
 
-Here are two diagrams detailing the SSoT based jobs in deeper detail.
+Here are three diagrams detailing the SSoT based jobs in deeper detail.
 
-![Sync Devices]().
-![Sync Network Data]().
+![C4 Onboarding Overview](../images/device-onboarding-4.0-Overview.png).
+![Sync Devices](../images/device-onboarding-4.0-Sync%20Device%20Job.png).
+![Sync Network Data](../images/device-onboarding-4.0-Sync%20Network%20Data%20Job.png).

docs/user/app_getting_started.md

@@ -38,6 +38,6 @@ Navigate to the SSoT dashboard and run `Sync Devices` to get basic device and in
 
 You can check out the [Use Cases](app_use_cases.md) section for more examples or try out the job inputs with at least the required fields.
 
-![job input](../images/do_job_inputs.png)
+![job input](../images/sync_devices_inputs.png)
 
 The Nautobot job will pass the job execution to the worker which will initiate an onboarding and will reach out to the device and attempt to onboard it.

docs/user/app_overview.md

@@ -9,39 +9,7 @@ This [Nautobot](https://github.com/nautobot/nautobot) App allows to easily onboa
 
 ## Description/Overview
 
-### Original Implementation
-
-!!! info
-    The original job and extensions pattern will remain a part of this App for the near future, this will allow custom extensions to continue working without causes issues to users that have taken the time and understand the original framework.  The newer SSoT implementation will be discussed in the next section.
-
-The `nautobot-device-onboarding` app uses the [netmiko](https://github.com/ktbyers/netmiko) and [NAPALM](https://napalm.readthedocs.io/en/latest/) libraries to simplify the onboarding process of a new device into Nautobot down to, in many cases, an *IP Address* and a *Location*. In some cases, the user may also have to specify a specific *Device Platform* and *Device Port*.
-
-Regardless, the Onboarding App greatly simplifies the onboarding process by allowing the user to specify a small amount of info and having the app populate a much larger amount of device data in Nautobot.
-
-In most cases, the user would specify:
-
-- Device Name
-- Location
-- Platform *
-- Transport Port *
-
-!!! note
-    * `Platform` and `Transport Port` are necessary for onboarding NXOS API, Arista EOS, or any other platform not using SSH as a transport.
-
-And the Onboarding App would populate the following:
-
-- Device Type (Model) - Creates it if it does not exist.
-- Role - via a Role of content-type "dcim:device". Defaults to `network`.
-- Platform - Creates Cisco IOS, Cisco NXOS (ssh), and Junos Platforms if they do not exist.
-- Manufacturer - Creates Cisco/Juniper/Arista if it does not exist.
-- Management Interface.
-- Management Interface IP.
-- Serial Number (when available).
-
-The goal of this app is not to import everything about a device into Nautobot. Rather, the goal is to quickly build an inventory of basic device data in Nautobot that provides basic info on how to access the devices.
-For example, getting the Management IP and Platform data into Nautobot allows a follow-on tool that uses the basic info to access each device, retrieve data, and then populate Nautobot with that data.
-
-One example of a solution that can retrieve that additional device data and import it into Nautobot is the [Network Importer](https://github.com/networktocode/network-importer). Other options would include an Ansible playbook or a Python script.
+This section descibes the new implementaiton (SSoT Implementation) and the Original implementation.
 
 ### New SSoT Implementation
 
@@ -90,6 +58,40 @@ Additional References:
 - To understand the lower level details of how the Network-SSoT framework is designed see [Network-SSoT Design](./app_detailed_design.md)
 - To learn how to add additonal platform/OS support visit [Extending](./external_interactions.md).
 
+### Original Implementation
+
+!!! info
+    The original job and extensions pattern will remain a part of this App for the near future, this will allow custom extensions to continue working without causes issues to users that have taken the time and understand the original framework.  The newer SSoT implementation will be discussed in the next section.
+
+The `nautobot-device-onboarding` app uses the [netmiko](https://github.com/ktbyers/netmiko) and [NAPALM](https://napalm.readthedocs.io/en/latest/) libraries to simplify the onboarding process of a new device into Nautobot down to, in many cases, an *IP Address* and a *Location*. In some cases, the user may also have to specify a specific *Device Platform* and *Device Port*.
+
+Regardless, the Onboarding App greatly simplifies the onboarding process by allowing the user to specify a small amount of info and having the app populate a much larger amount of device data in Nautobot.
+
+In most cases, the user would specify:
+
+- Device Name
+- Location
+- Platform *
+- Transport Port *
+
+!!! note
+    * `Platform` and `Transport Port` are necessary for onboarding NXOS API, Arista EOS, or any other platform not using SSH as a transport.
+
+And the Onboarding App would populate the following:
+
+- Device Type (Model) - Creates it if it does not exist.
+- Role - via a Role of content-type "dcim:device". Defaults to `network`.
+- Platform - Creates Cisco IOS, Cisco NXOS (ssh), and Junos Platforms if they do not exist.
+- Manufacturer - Creates Cisco/Juniper/Arista if it does not exist.
+- Management Interface.
+- Management Interface IP.
+- Serial Number (when available).
+
+The goal of this app is not to import everything about a device into Nautobot. Rather, the goal is to quickly build an inventory of basic device data in Nautobot that provides basic info on how to access the devices.
+For example, getting the Management IP and Platform data into Nautobot allows a follow-on tool that uses the basic info to access each device, retrieve data, and then populate Nautobot with that data.
+
+One example of a solution that can retrieve that additional device data and import it into Nautobot is the [Network Importer](https://github.com/networktocode/network-importer). Other options would include an Ansible playbook or a Python script.
+
 ## Audience (User Personas) - Who should use this App?
 
 The Onboarding App is meant for new Nautobot users who want to start importing their devices directly rather than from another, existing, source. Even with other sources for device information, they may not include everything that is necessary.

docs/user/app_use_cases.md

@@ -60,14 +60,14 @@ The Onboarding App will automatically create Platforms for vendor operating syst
 
 ## Onboarding a Device
 
-### Onboard a New Device
+### Onboard a New Device Using Sync Devices From Network Job
 
 A new device can be onboarded via :
 
-- A SSoT job execution.
+- A SSoT job execution using the `Sync Devices from Network` job.
     - Via Jobs menu
     - Via SSoT Dashboard
-- API, via a `POST` to `/api/extras/jobs/TODO/run` or `/api/extras/jobs/{id}/run` 
+- API, via a `POST` to `/api/extras/jobs/SSOTSyncDevices/run` or `/api/extras/jobs/{id}/run` 
 
 !!! note
     The SSoT Job's ID (UUID) will be different per Nautobot instance. 
@@ -85,10 +85,10 @@ The status of onboarding jobs can be viewed via the UI (Jobs > Job Results) or r
 To run the SSoT Sync Devices Job via the api:
 
 
-Post to `/api/extras/jobs/TODO/run/` with the relevent onboarding data: 
+Post to `/api/extras/jobs/SSOTSyncDevices/run/` with the relevent onboarding data: 
 
 ```bash
-curl -X "POST" <nautobot URL>/api/extras/jobs/TODO/run/ -H "Content-Type: application/json" -H "Authorization: Token $NAUTOBOT_TOKEN" -d '{"data": {"location": "<valid location UUID>", "ip_address": "<reachable IP to onboard>", "port": 22, "timeout": 30}}
+curl -X "POST" <nautobot URL>/api/extras/jobs/SSOTSyncDevices/run/ -H "Content-Type: application/json" -H "Authorization: Token $NAUTOBOT_TOKEN" -d '{"data": {"location": "<valid location UUID>", "ip_address": "<reachable IP to onboard>", "port": 22, "timeout": 30}}
 ```
 
 Required Fields:
@@ -113,7 +113,7 @@ A existing devices data can be expanded to include additonal objects by:
 - A SSoT job execution.
     - Via Jobs menu
     - Via SSoT Dashboard
-- API, via a `POST` to `/api/extras/jobs/TODO/run` or `/api/extras/jobs/{id}/run` 
+- API, via a `POST` to `/api/extras/jobs/SSOTSyncNetworkData/run` or `/api/extras/jobs/{id}/run` 
 
 !!! note
     The SSoT Job's ID (UUID) will be different per Nautobot instance. 
@@ -126,13 +126,13 @@ The status of onboarding jobs can be viewed via the UI (Jobs > Job Results) or r
 
 ### API
 
-To run the SSoT Sync Devices Job via the api:
+To run the SSoT Sync Network Data Job via the api:
 
 
-Post to `/api/extras/jobs/TODO/run/` with the relevent onboarding data: 
+Post to `/api/extras/jobs/SSOTSyncNetworkData/run/` with the relevent onboarding data: 
 
 ```bash
-curl -X "POST" <nautobot URL>/api/extras/jobs/TODO/run/ -H "Content-Type: application/json" -H "Authorization: Token $NAUTOBOT_TOKEN" -d '{"data": {"devices": "<valid devices UUID>"}
+curl -X "POST" <nautobot URL>/api/extras/jobs/SSOTSyncNetworkData/run/ -H "Content-Type: application/json" -H "Authorization: Token $NAUTOBOT_TOKEN" -d '{"data": {"devices": "<valid devices UUID>"}
 ```
 
 Required Fields:

docs/user/app_yaml_overrides.md

@@ -12,6 +12,7 @@ There are only a few components to the file and they're described below:
 - `parser` - Whether to use a parser (textfsm, pyats, ttp, etc) alternatively `no` can be used if the platform supports some other method to return structured data. E.g. (`| display json`) or an equivalent.
 - `jpath` - The jmespath (specifically jdiffs implementation) to extract the data from the parsed json returned from parser.
 - `post_processor` - Jinja2 capable code to further transform the returned data post jpath extraction.
+- `iterable_type` - A optional value to force a parsed result to a specific data type.
 
 As an example:
 

mkdocs.yml

@@ -113,6 +113,7 @@ nav:
       - Compatibility Matrix: "admin/compatibility_matrix.md"
       - Release Notes:
           - "admin/release_notes/index.md"
+          - v4.0: "admin/release_notes/version_4.0.md"
           - v3.0: "admin/release_notes/version_3.0.md"
           - v2.0: "admin/release_notes/version_2.0.md"
           - v1.2: "admin/release_notes/version_1.2.md"

nautobot_device_onboarding/command_mappers/arista_eos.yml

@@ -56,7 +56,6 @@ sync_network_data:
     commands:
       - command: "show interfaces | json"
         parser: "none"
-        # jpath: 'interfaces."{{ current_key }}".interfaceAddress[].[{ip_address: primaryIp.address, prefix_length: primaryIp.maskLen},{ip_address: secondaryIps.address, prefix_length: secondaryIps.maskLen}][]'
         jpath: 'interfaces."{{ current_key }}".interfaceAddress[].[{ip_address: primaryIp.address, prefix_length: primaryIp.maskLen }][]' # yamllint disable-line rule:quoted-strings
         post_processor: "{{ obj | tojson }}"
   interfaces__mtu:
@@ -85,7 +84,8 @@ sync_network_data:
     commands:
       - command: "show interfaces switchport | json"
         parser: "none"
-        jpath: 'switchports."{{ current_key }}".switchportInfo.mode'  # yamllint disable-line rule:quoted-strings
+        jpath: '{admin_mode: switchports."{{ current_key }}".switchportInfo.mode, mode: switchports."{{ current_key }}".switchportInfo.mode, trunking_vlans: switchports."{{ current_key }}".switchportInfo.trunkAllowedVlans}'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | interface_mode_logic }}"
         iterable_type: "str"
   interfaces__lag:
     commands:
@@ -104,12 +104,12 @@ sync_network_data:
     commands:
       - command: "show interfaces switchport | json"
         parser: "none"
-        jpath: '{admin_mode: switchports."{{ current_key }}".switchportInfo.mode, mode: switchports."{{ current_key }}".switchportInfo.mode, access_vlan: switchports."{{ current_key }}".switchportInfo.accessVlanId, trunking_vlans: switchports."{{ current_key }}".switchportInfo.trunkAllowedVlans}'  # yamllint disable-line rule:quoted-strings
-        post_processor: "{{ obj | get_vlan_data(vlan_map) | tojson }}"
+        jpath: '{admin_mode: switchports."{{ current_key }}".switchportInfo.mode, mode: switchports."{{ current_key }}".switchportInfo.mode, access_vlan: switchports."{{ current_key }}".switchportInfo.accessVlanId, trunking_vlans: switchports."{{ current_key }}".switchportInfo.trunkAllowedVlans, native_vlan: switchports."{{ current_key }}".switchportInfo.trunkingNativeVlanId}'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | get_vlan_data(vlan_map, 'tagged') | tojson }}"
   interfaces__untagged_vlan:
     commands:
       - command: "show interfaces switchport | json"
         parser: "none"
-        jpath: '{id: switchports."{{ current_key }}".switchportInfo.trunkingNativeVlanId, name: switchports."{{ current_key }}".switchportInfo.trunkingNativeVlanName}'  # yamllint disable-line rule:quoted-strings
-        post_processor: "{{ obj | key_exist_or_default('id') | tojson }}"
+        jpath: '{admin_mode: switchports."{{ current_key }}".switchportInfo.mode, mode: switchports."{{ current_key }}".switchportInfo.mode, access_vlan: switchports."{{ current_key }}".switchportInfo.accessVlanId, trunking_vlans: switchports."{{ current_key }}".switchportInfo.trunkAllowedVlans, native_vlan: switchports."{{ current_key }}".switchportInfo.trunkingNativeVlanId}'  # yamllint disable-line rule:quoted-strings
+        post_processor: "{{ obj | get_vlan_data(vlan_map, 'untagged') | tojson }}"
         iterable_type: "dict"