Prep 4.0.0

.github/workflows/ci.yml

@@ -106,7 +106,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.11"]
-        nautobot-version: ["2.2.2"]
+        nautobot-version: ["2.2.3"]
     env:
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_DEVICE_ONBOARDING_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -152,7 +152,7 @@ jobs:
         include:
           - python-version: "3.11"
             db-backend: "postgresql"
-            nautobot-version: "2.2.2"
+            nautobot-version: "2.2.3"
           # - python-version: "3.11"
           #   db-backend: "mysql"
           #   nautobot-version: "stable"

README.md

@@ -3,7 +3,7 @@
 <p align="center">
   <img src="https://raw.githubusercontent.com/nautobot/nautobot-app-device-onboarding/develop/docs/images/icon-DeviceOnboarding.png" class="logo" height="200px">
   <br>
-  <a href="https://github.com/nautobot/nautobot-app-device-onboarding/actions"><img src="https://github.com/nautobot/nautobot-app-device-onboarding/actions/workflows/ci.yml/badge.svg?branch=main"></a>
+  <a href="https://github.com/nautobot/nautobot-app-device-onboarding/actions"><img src="https://github.com/nautobot/nautobot-app-device-onboarding/actions/workflows/ci.yml/badge.svg?branch=develop"></a>
   <a href="https://docs.nautobot.com/projects/device-onboarding/en/latest/"><img src="https://readthedocs.org/projects/nautobot-plugin-device-onboarding/badge/"></a>
   <a href="https://pypi.org/project/nautobot-device-onboarding/"><img src="https://img.shields.io/pypi/v/nautobot-device-onboarding"></a>
   <a href="https://pypi.org/project/nautobot-device-onboarding/"><img src="https://img.shields.io/pypi/dm/nautobot-device-onboarding"></a>
@@ -21,7 +21,7 @@ Regardless, the Onboarding App greatly simplifies the onboarding process by allo
 
 Device Onboarding is a Job that allows you to provide a few required pieces of information and onboard the device.
 
-![job input](https://raw.githubusercontent.com/nautobot/nautobot-app-device-onboarding/develop/docs/images/do_job_inputs.png)
+![job input](https://raw.githubusercontent.com/nautobot/nautobot-app-device-onboarding/develop/docs/images/sync_devices_inputs.png)
 
 ## Try it out!
 

changes/181.added

@@ -0,0 +1,5 @@
+Sync Devices from Network job was added which utilizes the SSoT framework to onboard devices.
+Sync Data from Network job was added which utilizes the SSoT framework to onboard devices.
+Git Datasource object to be able to use a Git Repo to overload new SSoT job YAML file definitions.
+Create a Nornir inventory `EmptyInventory` to support ondemand inventory population for `Sync Devices` job.
+Add `nautobot-app-nornir` dependency to reuse `NautobotORMInventory` to support inventory creation for `Sync Data` job.

changes/181.changed

@@ -0,0 +1 @@
+The `OnboardingTask` job is changed to `hidden` by default.

development/Dockerfile

@@ -6,8 +6,8 @@
 # -------------------------------------------------------------------------------------
 # !!! USE CAUTION WHEN MODIFYING LINES BELOW
 
-# Accepts a desired Nautobot version as build argument, default to 2.0.3
-ARG NAUTOBOT_VER="2.0.3"
+# Accepts a desired Nautobot version as build argument, default to 2.2.3
+ARG NAUTOBOT_VER="2.2.3"
 
 # Accepts a desired Python version as build argument, default to 3.11
 ARG PYTHON_VER="3.11"
@@ -28,7 +28,7 @@ ENV INVOKE_NAUTOBOT_DEVICE_ONBOARDING_LOCAL=true
 # Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary
 # We also don't need virtual environments in container
 RUN which poetry || curl -sSL https://install.python-poetry.org | python3 - && \
-    poetry config virtualenvs.create false
+  poetry config virtualenvs.create false
 
 # !!! USE CAUTION WHEN MODIFYING LINES ABOVE
 # -------------------------------------------------------------------------------------

development/development.env

@@ -36,3 +36,5 @@ POSTGRES_DB=${NAUTOBOT_DB_NAME}
 MYSQL_USER=${NAUTOBOT_DB_USER}
 MYSQL_DATABASE=${NAUTOBOT_DB_NAME}
 MYSQL_ROOT_HOST=%
+
+NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS="True"

development/docker-compose.base.yml

@@ -13,7 +13,6 @@ x-nautobot-base: &nautobot-base
     - "creds.env"
   tty: true
 
-version: "3.8"
 services:
   nautobot:
     depends_on:

development/docker-compose.dev.yml

@@ -3,7 +3,6 @@
 # any override will need to include these volumes to use them.
 # see:  https://github.com/docker/compose/issues/3729
 ---
-version: "3.8"
 services:
   nautobot:
     command: "nautobot-server runserver 0.0.0.0:8080"

development/docker-compose.mysql.yml

@@ -1,6 +1,4 @@
 ---
-version: "3.8"
-
 services:
   nautobot:
     environment:

development/docker-compose.postgres.yml

@@ -1,6 +1,4 @@
 ---
-version: "3.8"
-
 services:
   nautobot:
     environment:

development/docker-compose.redis.yml

@@ -1,5 +1,4 @@
 ---
-version: "3.8"
 services:
   redis:
     image: "redis:6-alpine"

development/nautobot_config.py

@@ -93,11 +93,11 @@
         "disable_existing_loggers": False,
         "formatters": {
             "normal": {
-                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s : %(message)s",
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n  %(message)s",
                 "datefmt": "%H:%M:%S",
             },
             "verbose": {
-                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() : %(message)s",
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n  %(message)s",
                 "datefmt": "%H:%M:%S",
             },
         },
@@ -127,13 +127,24 @@
 #
 
 # Enable installed Apps. Add the name of each App to the list.
-PLUGINS = ["nautobot_device_onboarding"]
+PLUGINS = ["nautobot_device_onboarding", "nautobot_ssot", "nautobot_plugin_nornir"]
 
 # Apps configuration settings. These settings are used by various Apps that the user may have installed.
 # Each key in the dictionary is the name of an installed App and its value is a dictionary of settings.
-# PLUGINS_CONFIG = {
-#     'nautobot_device_onboarding': {
-#         'foo': 'bar',
-#         'buzz': 'bazz'
-#     }
-# }
+PLUGINS_CONFIG = {
+    "nautobot_device_onboarding": {},
+    "nautobot_ssot": {
+        "hide_example_jobs": is_truthy(os.getenv("NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS")),
+    },
+    "nautobot_plugin_nornir": {
+        "nornir_settings": {
+            "credentials": "nautobot_plugin_nornir.plugins.credentials.nautobot_secrets.CredentialsNautobotSecrets",
+            "runner": {
+                "plugin": "threaded",
+                "options": {
+                    "num_workers": 20,
+                },
+            },
+        },
+    },
+}

development/towncrier_template.j2

@@ -1,6 +1,6 @@
 
 {% if render_title %}
-## [v{{ versiondata.version }} ({{ versiondata.date }})]({{ cookiecutter.repo_url }}/releases/tag/v{{ versiondata.version}})
+## [v{{ versiondata.version }} ({{ versiondata.date }})](https://github.com/nautobot/nautobot-app-device-onboarding/releases/tag/v{{ versiondata.version}})
 
 {% endif %}
 {% for section, _ in sections.items() %}

docs/admin/compatibility_matrix.md

@@ -7,3 +7,4 @@
 | 1.2.0                     | 1.4.0                          | 1.99                          |
 | 2.0.0                     | 2.0.0                          | 2.99                          |
 | 3.0.0                     | 2.0.3                          | 2.99                          |
+| 4.0.0                     | 2.2.3                          | 2.99                          |

docs/admin/install.md

@@ -12,23 +12,13 @@ Here you will find detailed instructions on how to **install** and **configure**
 
 ### Access Requirements
 
-#### NAPALM Credentials
-
-The Onboarding App uses NAPALM. You can configure a default NAPALM username and password in `nautobot_config.py`.
-
-When `NAPALM_USERNAME`, `NAPALM_PASSWORD` and `NAPALM_ARGS` are configured in `nautobot_config.py`, the user does not have to use the Credentials/SecretGroup fields in the Device Onboarding job, unless they wish to override the values in `nautobot_config.py`:
-
-```python
-# Credentials that Nautobot will use to authenticate to devices when connecting via NAPALM.
-NAPALM_USERNAME = "<napalm username>"
-NAPALM_PASSWORD = "<napalm pwd>"
-NAPALM_ARGS = {"secret": "<enable secret pwd>"}
-```
+- The original OnboardingTask Job used NAPALM Credentials.
+- The new SSoT based jobs use Nautobot Secrets in a similar method to other apps. This is a more flexible and supportable way for the future.
 
 ## Install Guide
 
 !!! note
-    Apps can be installed from the [Python Package Index](https://pypi.org/) or locally. See the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/installation/app-install/) for more details. The pip package name for this app is [`nautobot-device-onboarding`](https://pypi.org/project/nautobot-device-onboarding/).
+    Apps can be installed manually or using Python's `pip`. See the [nautobot documentation](https://nautobot.readthedocs.io/en/latest/plugins/#install-the-package) for more details. The pip package name for this app is [`nautobot-device-onboarding`](https://pypi.org/project/nautobot-device-onboarding/).
 
 The app is available as a Python package via PyPI and can be installed with `pip`:
 
@@ -74,7 +64,7 @@ Then restart (if necessary) the Nautobot services which may include:
 sudo systemctl restart nautobot nautobot-worker nautobot-scheduler
 ```
 
-## App Configuration
+## App Configuration (Original)
 
 Although the app can run without providing any settings, the app behavior can be controlled with the following list of settings defined in `nautobot_config.py`:
 

docs/admin/release_notes/version_4.0.md

@@ -0,0 +1,4 @@
+# v4.0 Release Notes
+
+!!! warning
+    Nautobot Device Onboarding v4.0.0 completely revamps the applications design and framework. The original `OnboardingTask` job is still packaged with the app to provide a backwards compatible way for users that have used its extensions framework in the past to solve complex problems. However, that job is now hidden by default to avoid confusion with the two new SSoT based onboarding jobs that v4.0.0 exposes.

docs/dev/arch_decision.md

@@ -0,0 +1,3 @@
+# Architecture Decision Records
+
+The intention is to document deviations from a standard Model View Controller (MVC) design. And other general Application decisions.

docs/dev/contributing.md

@@ -1,5 +1,7 @@
 # Contributing to the App
 
+Pull requests are welcome and automatically built and tested against multiple version of Python and multiple version of Nautobot through GitHub Actions.
+
 The project is packaged with a light [development environment](dev_environment.md) based on `docker-compose` to help with the local development of the project and to run tests.
 
 The project is following Network to Code software development guidelines and is leveraging the following:
@@ -41,32 +43,8 @@ All pull requests to `next` or `develop` must include a changelog fragment file
 
 ## Branching Policy
 
-The branching policy includes the following tenets:
-
-- The `develop` branch is the branch of the next major and minor paired version planned.
-- PRs intended to add new features should be sourced from the `develop` branch.
-- PRs intended to fix issues in the Nautobot LTM compatible release should be sourced from the latest `ltm-<major.minor>` branch instead of `develop`.
-
-Device Onboarding will observe semantic versioning, as of 1.0. This may result in a quick turnaround in minor versions to keep pace with an ever growing feature set.
+Please fork the release and add a new branch to your fork. Make changes to your branch in your fork and submit PRs from there.
 
 ## Release Policy
 
-Device Onboarding has currently no intended scheduled release schedule, and will release new features in minor versions.
-
-When a new release, from `develop` to `main`, is created the following should happen.
-
-- A release PR is created from `develop` with:
-  - Update the release notes in `docs/admin/release_notes/version_<major>.<minor>.md` file to reflect the changes.
-  - Change the version from `<major>.<minor>.<patch>-beta` to `<major>.<minor>.<patch>` in `pyproject.toml`.
-  - Set the PR to the `main` branch.
-- Ensure the tests for the PR pass.
-- Merge the PR.
-- Create a new tag:
-  - The tag should be in the form of `v<major>.<minor>.<patch>`.
-  - The title should be in the form of `v<major>.<minor>.<patch>`.
-  - The description should be the changes that were added to the `version_<major>.<minor>.md` document.
-- If merged into `main`, then push from `main` to `develop`, in order to retain the merge commit created when the PR was merged
-- A post release PR is created with:
-  - Change the version from `<major>.<minor>.<patch>` to `<major>.<minor>.<patch + 1>-beta` in both `pyproject.toml` and `nautobot.__init__.__version__`.
-  - Set the PR to the proper branch, `develop`.
-  - Once tests pass, merge.
+New versions are released as bug fixes and features are introduced. We will make sure to release new versions to support the latest Nautobot versions as necessary.

docs/dev/dev_environment.md

@@ -74,7 +74,7 @@ nautobot-server migrate
 !!! note
     If you want to develop on the latest develop branch of Nautobot, run the following command: `poetry add --optional git+https://github.com/nautobot/nautobot@develop`. After the `@` symbol must match either a branch or a tag.
 
-You can now run `nautobot-server` commands as you would from the [Nautobot documentation](https://docs.nautobot.com/projects/core/en/stable/user-guide/administration/tools/nautobot-server/) for example to start the development server:
+You can now run `nautobot-server` commands as you would from the [Nautobot documentation](https://nautobot.readthedocs.io/en/latest/) for example to start the development server:
 
 ```shell
 nautobot-server runserver 0.0.0.0:8080 --insecure
@@ -473,21 +473,3 @@ To run an individual test, you can run any or all of the following:
 ➜ invoke ruff
 ➜ invoke pylint
 ```
-
-### App Configuration Schema
-
-In the package source, there is the `nautobot_device_onboarding/app-config-schema.json` file, conforming to the [JSON Schema](https://json-schema.org/) format. This file is used to validate the configuration of the app in CI pipelines.
-
-If you make changes to `PLUGINS_CONFIG` or the configuration schema, you can run the following command to validate the schema:
-
-```bash
-invoke validate-app-config
-```
-
-To generate the `app-config-schema.json` file based on the current `PLUGINS_CONFIG` configuration, run the following command:
-
-```bash
-invoke generate-app-config-schema
-```
-
-This command can only guess the schema, so it's up to the developer to manually update the schema as needed.

docs/dev/extending.md

@@ -2,8 +2,21 @@
 
 Extending the application is welcome, however it is best to open an issue first, to ensure that a PR would be accepted and makes sense in terms of features and design.
 
-## Customizing Onboarding Behavior With Extensions
+## Customizing Onboarding Behavior With Extensions (Original)
 
 This plugin provides methods to customize onboarding behavior. By creating onboarding extensions, it is possible to onboard switch stacks, HA pairs and perform other customizations.
 
 Please see the dedicated FAQ for [device onboarding extensions](onboarding_extensions.md).
+
+!!! warn
+    This is the original onboarding extensions.  The next section covers how to extend the new framework.
+
+## Extending SSoT jobs (Sync Devices From Network, and Sync Network Data From Network)
+
+Extending the platform support for the SSoT specific jobs should be accomplished with adding a yaml file that defines commands, jdiff jmespaths, and post_processors. A PR into this library is welcomed, but this app exposes the Nautobot core datasource capabilities to be able to load in overrides from a Git repository.
+
+### Adding Platform/OS Support
+
+New platform support should be simplified in this framework, by providing a YAML file.
+
+The format of these YAML files are and how to extend this application is covered in [App YAML Overrides](../user/app_yaml_overrides.md).

docs/dev/onboarding_extensions.md

@@ -1,4 +1,7 @@
-# Onboarding Extensions
+# Onboarding Extensions (Original)
+
+!!! warn
+    This is the original onboarding extensions.
 
 ## What are onboarding extensions?