From 70f1be81948d45992704ee0a4e20e885e2a4b4ae Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Mon, 12 Jun 2023 16:55:02 +0000 Subject: [PATCH 01/11] chore: Add ServiceNow files --- development/servicenow/Dockerfile | 79 +++ development/servicenow/creds.example.env | 16 + development/servicenow/dev.env | 16 + .../servicenow/docker-compose.base.yml | 40 ++ development/servicenow/docker-compose.dev.yml | 20 + .../servicenow/docker-compose.docs.yml | 11 + .../docker-compose.requirements.yml | 25 + development/servicenow/nautobot_config.py | 78 +++ .../integrations/servicenow/__init__.py | 41 ++ .../servicenow/data/mappings.yaml | 85 +++ .../servicenow/diffsync/__init__.py | 0 .../servicenow/diffsync/adapter_nautobot.py | 232 +++++++ .../servicenow/diffsync/adapter_servicenow.py | 299 +++++++++ .../servicenow/diffsync/models.py | 281 ++++++++ .../integrations/servicenow/forms.py | 28 + nautobot_ssot/integrations/servicenow/jobs.py | 140 ++++ .../servicenow/migrations/0001_initial.py | 38 ++ .../servicenow/migrations/__init__.py | 0 .../integrations/servicenow/models.py | 42 ++ .../integrations/servicenow/servicenow.py | 55 ++ .../integrations/servicenow/signals.py | 43 ++ .../servicenow/third_party/pysnow/__init__.py | 21 + .../third_party/pysnow/attachment.py | 93 +++ .../servicenow/third_party/pysnow/client.py | 214 ++++++ .../third_party/pysnow/criterion.py | 506 ++++++++++++++ .../servicenow/third_party/pysnow/enums.py | 112 ++++ .../third_party/pysnow/exceptions.py | 77 +++ .../third_party/pysnow/legacy_exceptions.py | 84 +++ .../third_party/pysnow/legacy_request.py | 451 +++++++++++++ .../third_party/pysnow/oauth_client.py | 171 +++++ .../third_party/pysnow/params_builder.py | 194 ++++++ .../third_party/pysnow/query_builder.py | 318 +++++++++ .../servicenow/third_party/pysnow/request.py | 169 +++++ .../servicenow/third_party/pysnow/resource.py | 164 +++++ .../servicenow/third_party/pysnow/response.py | 280 ++++++++ .../third_party/pysnow/url_builder.py | 68 ++ nautobot_ssot/integrations/servicenow/urls.py | 10 + .../integrations/servicenow/utils.py | 41 ++ .../integrations/servicenow/views.py | 32 + .../ServiceNow_logo.svg | 14 + .../nautobot_ssot_servicenow/config.html | 12 + nautobot_ssot/tests/servicenow/__init__.py | 1 + .../tests/servicenow/test_adapter_nautobot.py | 78 +++ .../servicenow/test_adapter_servicenow.py | 618 ++++++++++++++++++ nautobot_ssot/tests/servicenow/test_basic.py | 16 + nautobot_ssot/tests/servicenow/test_jobs.py | 149 +++++ 46 files changed, 5462 insertions(+) create mode 100644 development/servicenow/Dockerfile create mode 100644 development/servicenow/creds.example.env create mode 100644 development/servicenow/dev.env create mode 100644 development/servicenow/docker-compose.base.yml create mode 100644 development/servicenow/docker-compose.dev.yml create mode 100644 development/servicenow/docker-compose.docs.yml create mode 100644 development/servicenow/docker-compose.requirements.yml create mode 100644 development/servicenow/nautobot_config.py create mode 100644 nautobot_ssot/integrations/servicenow/__init__.py create mode 100644 nautobot_ssot/integrations/servicenow/data/mappings.yaml create mode 100644 nautobot_ssot/integrations/servicenow/diffsync/__init__.py create mode 100644 nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py create mode 100644 nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py create mode 100644 nautobot_ssot/integrations/servicenow/diffsync/models.py create mode 100644 nautobot_ssot/integrations/servicenow/forms.py create mode 100644 nautobot_ssot/integrations/servicenow/jobs.py create mode 100644 nautobot_ssot/integrations/servicenow/migrations/0001_initial.py create mode 100644 nautobot_ssot/integrations/servicenow/migrations/__init__.py create mode 100644 nautobot_ssot/integrations/servicenow/models.py create mode 100644 nautobot_ssot/integrations/servicenow/servicenow.py create mode 100644 nautobot_ssot/integrations/servicenow/signals.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/__init__.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/attachment.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/criterion.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/enums.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/exceptions.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_exceptions.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_request.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/oauth_client.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/params_builder.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/query_builder.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/request.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/resource.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/response.py create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/url_builder.py create mode 100644 nautobot_ssot/integrations/servicenow/urls.py create mode 100644 nautobot_ssot/integrations/servicenow/utils.py create mode 100644 nautobot_ssot/integrations/servicenow/views.py create mode 100644 nautobot_ssot/static/nautobot_ssot_servicenow/ServiceNow_logo.svg create mode 100644 nautobot_ssot/templates/nautobot_ssot_servicenow/config.html create mode 100644 nautobot_ssot/tests/servicenow/__init__.py create mode 100644 nautobot_ssot/tests/servicenow/test_adapter_nautobot.py create mode 100644 nautobot_ssot/tests/servicenow/test_adapter_servicenow.py create mode 100644 nautobot_ssot/tests/servicenow/test_basic.py create mode 100644 nautobot_ssot/tests/servicenow/test_jobs.py diff --git a/development/servicenow/Dockerfile b/development/servicenow/Dockerfile new file mode 100644 index 00000000..cb501301 --- /dev/null +++ b/development/servicenow/Dockerfile @@ -0,0 +1,79 @@ +# ------------------------------------------------------------------------------------- +# Nautobot App Developement Dockerfile Template +# Version: 1.0.0 +# +# Apps that need to add additional steps or packages can do in the section below. +# ------------------------------------------------------------------------------------- +# !!! USE CAUTION WHEN MODIFYING LINES BELOW + +# Accepts a desired Nautobot version as build argument, default to 1.4.0 +ARG NAUTOBOT_VER="1.4" + +# Accepts a desired Python version as build argument, default to 3.8 +ARG PYTHON_VER="3.8" + +# Retreive published development image of Nautobot base which should include most CI dependencies +FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} + +# Runtime argument and environment setup +ARG NAUTOBOT_ROOT=/opt/nautobot + +ENV prometheus_multiproc_dir=/prom_cache +ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT} + +# Install Poetry manually via its installer script; +# We might be using an older version of Nautobot that includes an older version of Poetry +# and CI and local development may have a newer version of Poetry +# Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary +# We also don't need virtual environments in container +RUN curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \ + python /tmp/install-poetry.py && \ + rm -f /tmp/install-poetry.py && \ + poetry config virtualenvs.create false + +# !!! USE CAUTION WHEN MODIFYING LINES ABOVE +# ------------------------------------------------------------------------------------- +# App-specifc system build/test dependencies. +# +# Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. +# ------------------------------------------------------------------------------------- +# --> Start safe to modify section + +# Uncomment the line below if you are apt-installing any package. +# RUN apt update +# RUN apt install libldap2-dev + +# --> Stop safe to modify section +# ------------------------------------------------------------------------------------- +# Install Nautobot App +# ------------------------------------------------------------------------------------- +# !!! USE CAUTION WHEN MODIFYING LINES BELOW + +# Copy in the source code +WORKDIR /source +COPY . /source + +# Get container's installed Nautobot version as a forced constraint +# NAUTOBOT_VER may be a branch name and not a published release therefor we need to get the installed version +# so pip can use it to recognize local constraints. +RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > constraints.txt + +# Use Poetry to grab dev dependencies from the lock file +# Can be improved in Poetry 1.2 which allows `poetry install --only dev` +# +# We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies, +# especially those that are only direct to Nautobot but the container included versions slightly mismatch +RUN poetry export -f requirements.txt --without-hashes --output poetry_freeze_base.txt +RUN poetry export -f requirements.txt --dev --without-hashes --output poetry_freeze_all.txt +RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt + +# Install all local project as editable, constrained on Nautobot version, to get any additional +# direct dependencies of the app +RUN pip install -c constraints.txt -e . + +# Install any dev dependencies frozen from Poetry +# Can be improved in Poetry 1.2 which allows `poetry install --only dev` +RUN pip install -c constraints.txt -r poetry_freeze_dev.txt + +COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py +# !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/servicenow/creds.example.env b/development/servicenow/creds.example.env new file mode 100644 index 00000000..8583b0ae --- /dev/null +++ b/development/servicenow/creds.example.env @@ -0,0 +1,16 @@ +NAUTOBOT_DB_PASSWORD=notverysecurepwd +NAUTOBOT_REDIS_PASSWORD=notverysecurepwd +NAUTOBOT_SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj +NAUTOBOT_CREATE_SUPERUSER=true +NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 +NAUTOBOT_SUPERUSER_PASSWORD=admin +# Needed for Postgres, must match the values above +PGPASSWORD=notverysecurepwd +POSTGRES_DB=nautobot +POSTGRES_PASSWORD=notverysecurepwd +POSTGRES_USER=nautobot +# Needed for Redis, must match the values above +REDIS_PASSWORD=notverysecurepwd +# NAUTOBOT_DB_HOST=localhost +# NAUTOBOT_REDIS_HOST=localhost +# NAUTOBOT_ROOT=./development diff --git a/development/servicenow/dev.env b/development/servicenow/dev.env new file mode 100644 index 00000000..a557b394 --- /dev/null +++ b/development/servicenow/dev.env @@ -0,0 +1,16 @@ +NAUTOBOT_ALLOWED_HOSTS=* +NAUTOBOT_DEBUG=True +NAUTOBOT_METRICS_ENABLED=True +NAUTOBOT_ROOT=/opt/nautobot +NAUTOBOT_DB_NAME=nautobot +NAUTOBOT_DB_HOST=postgres +NAUTOBOT_DB_USER=nautobot +NAUTOBOT_REDIS_HOST=redis +NAUTOBOT_REDIS_PORT=6379 +# NAUTOBOT_REDIS_SSL=True +# Uncomment NAUTOBOT_REDIS_SSL if using SSL +SUPERUSER_EMAIL=admin@example.com +SUPERUSER_NAME=admin + +NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=1500 +NAUTOBOT_CELERY_TASK_TIME_LIMIT=1800 diff --git a/development/servicenow/docker-compose.base.yml b/development/servicenow/docker-compose.base.yml new file mode 100644 index 00000000..17b0ee9c --- /dev/null +++ b/development/servicenow/docker-compose.base.yml @@ -0,0 +1,40 @@ +--- +x-nautobot-build: &nautobot-build + build: + args: + NAUTOBOT_VER: "${NAUTOBOT_VER}" + PYTHON_VER: "${PYTHON_VER}" + context: "../" + dockerfile: "development/Dockerfile" +x-nautobot-base: &nautobot-base + image: "nautobot-ssot-servicenow/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" + env_file: + - "dev.env" + - "creds.env" + tty: true + +version: "3.4" +services: + nautobot: + ports: + - "8080:8080" + depends_on: + - "postgres" + - "redis" + <<: *nautobot-build + <<: *nautobot-base + worker: + entrypoint: "nautobot-server rqworker" + depends_on: + - "nautobot" + healthcheck: + disable: true + <<: *nautobot-base + celery_worker: + entrypoint: "nautobot-server celery worker -l INFO" + depends_on: + - "nautobot" + - "redis" + healthcheck: + disable: true + <<: *nautobot-base diff --git a/development/servicenow/docker-compose.dev.yml b/development/servicenow/docker-compose.dev.yml new file mode 100644 index 00000000..610c0782 --- /dev/null +++ b/development/servicenow/docker-compose.dev.yml @@ -0,0 +1,20 @@ +# We can't remove volumes in a compose override, for the test configuration using the final containers +# we don't want the volumes so this is the default override file to add the volumes in the dev case +# any override will need to include these volumes to use them. +# see: https://github.com/docker/compose/issues/3729 +--- +version: "3.4" +services: + nautobot: + command: "nautobot-server runserver 0.0.0.0:8080 --insecure" + volumes: + - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" + - "../:/source" + worker: + volumes: + - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" + - "../:/source" + celery_worker: + volumes: + - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" + - "../:/source" diff --git a/development/servicenow/docker-compose.docs.yml b/development/servicenow/docker-compose.docs.yml new file mode 100644 index 00000000..a09bafa1 --- /dev/null +++ b/development/servicenow/docker-compose.docs.yml @@ -0,0 +1,11 @@ +--- +version: "3.4" +services: + docs: + image: "nautobot-ssot-servicenow/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" + entrypoint: "mkdocs serve -v -a 0.0.0.0:8080" + volumes: + - "../docs:/source/docs:ro" + - "../mkdocs.yml:/source/mkdocs.yml:ro" + ports: + - "8001:8080" diff --git a/development/servicenow/docker-compose.requirements.yml b/development/servicenow/docker-compose.requirements.yml new file mode 100644 index 00000000..175cd297 --- /dev/null +++ b/development/servicenow/docker-compose.requirements.yml @@ -0,0 +1,25 @@ +--- +version: "3.4" +services: + postgres: + image: "postgres:13-alpine" + env_file: + - "dev.env" + - "creds.env" + volumes: + - "postgres_data:/var/lib/postgresql/data" + ports: + - "5432:5432" + redis: + image: "redis:6-alpine" + command: + - "sh" + - "-c" # this is to evaluate the $REDIS_PASSWORD from the env + - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD" + env_file: + - "dev.env" + - "creds.env" + ports: + - "6379:6379" +volumes: + postgres_data: {} diff --git a/development/servicenow/nautobot_config.py b/development/servicenow/nautobot_config.py new file mode 100644 index 00000000..ed1a4242 --- /dev/null +++ b/development/servicenow/nautobot_config.py @@ -0,0 +1,78 @@ +######################### +# # +# Required settings # +# # +######################### + +import os +import sys + +from nautobot.core.settings import * # noqa: F401,F403 +from nautobot.core.settings_funcs import parse_redis_connection + +TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" + +# This is a list of valid fully-qualified domain names (FQDNs) for the Nautobot server. Nautobot will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['nautobot.example.com', 'nautobot.internal.local'] +ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS").split(" ") + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASES = { + "default": { + "NAME": os.getenv("NAUTOBOT_DB_NAME", "nautobot"), # Database name + "USER": os.getenv("NAUTOBOT_DB_USER", ""), # Database username + "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""), # Datbase password + "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"), # Database server + "PORT": os.getenv("NAUTOBOT_DB_PORT", ""), # Database port (leave blank for default) + "CONN_MAX_AGE": os.getenv("NAUTOBOT_DB_TIMEOUT", 300), # Database timeout + "ENGINE": "django.db.backends.postgresql", # Database driver (Postgres only supported!) + } +} + +# The django-redis cache is used to establish concurrent locks using Redis. The +# django-rq settings will use the same instance/database by default. +# +# This "default" server is now used by RQ_QUEUES. +# >> See: nautobot.core.settings.RQ_QUEUES +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": parse_redis_connection(redis_database=0), + "TIMEOUT": 300, + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "PASSWORD": os.getenv("NAUTOBOT_REDIS_PASSWORD", ""), + }, + } +} + +# RQ_QUEUES is not set here because it just uses the default that gets imported +# up top via `from nautobot.core.settings import *`. + +# REDIS CACHEOPS +CACHEOPS_REDIS = parse_redis_connection(redis_database=1) + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. Nautobot will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "") + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = ["nautobot_ssot", "nautobot_ssot_servicenow"] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +PLUGINS_CONFIG = { + "nautobot_ssot": { + "hide_example_jobs": True, + }, + "nautobot_ssot_servicenow": { + "instance": os.getenv("SERVICENOW_INSTANCE", ""), + "username": os.getenv("SERVICENOW_USERNAME", ""), + "password": os.getenv("SERVICENOW_PASSWORD", ""), + }, +} diff --git a/nautobot_ssot/integrations/servicenow/__init__.py b/nautobot_ssot/integrations/servicenow/__init__.py new file mode 100644 index 00000000..3434e9da --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/__init__.py @@ -0,0 +1,41 @@ +"""Plugin declaration for nautobot_ssot_servicenow.""" +from nautobot.core.signals import nautobot_database_ready +from nautobot.extras.plugins import PluginConfig + +from .signals import nautobot_database_ready_callback + +try: + from importlib import metadata +except ImportError: + # Running on pre-3.8 Python; use importlib-metadata package + import importlib_metadata as metadata + +__version__ = metadata.version(__name__) + + +class NautobotSSOTServiceNowConfig(PluginConfig): + """Plugin configuration for the nautobot_ssot_servicenow plugin.""" + + name = "nautobot_ssot_servicenow" + verbose_name = "Nautobot SSoT ServiceNow" + version = __version__ + author = "Network to Code, LLC" + description = "Nautobot SSoT ServiceNow." + base_url = "ssot-servicenow" + required_settings = [] + min_version = "1.4.0" + max_version = "1.9999" + default_settings = {} + required_settings = [] + caching_config = {} + + home_view_name = "plugins:nautobot_ssot:dashboard" # a link to the ServiceNow job would be even better + config_view_name = "plugins:nautobot_ssot_servicenow:config" + + def ready(self): + """Callback when this plugin is loaded.""" + super().ready() + nautobot_database_ready.connect(nautobot_database_ready_callback, sender=self) + + +config = NautobotSSOTServiceNowConfig # pylint:disable=invalid-name diff --git a/nautobot_ssot/integrations/servicenow/data/mappings.yaml b/nautobot_ssot/integrations/servicenow/data/mappings.yaml new file mode 100644 index 00000000..6992136e --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/data/mappings.yaml @@ -0,0 +1,85 @@ +--- +company: + table: "core_company" + # Only load companies that are flagged as manufacturer: true, as those correspond to Nautobot Manufacturer records + table_query: + manufacturer: true + mappings: + - field: "name" + column: "name" + - field: "manufacturer" + column: "manufacturer" +product_model: + table: "cmdb_hardware_product_model" + parent: + modelname: "company" + field: "manufacturer_name" + column: "manufacturer" + mappings: + - field: "manufacturer_name" + reference: + key: "manufacturer" + table: "core_company" + column: "name" + - field: "model_name" + column: "name" + - field: "model_number" + column: "model_number" +location: + table: "cmn_location" + mappings: + - field: "name" + column: "name" + - field: "parent_location_name" + reference: + key: "parent" + table: "cmn_location" + column: "name" + - field: "full_name" + column: "full_name" + - field: "latitude" + column: "latitude" + - field: "longitude" + column: "longitude" +device: + table: "cmdb_ci_ip_switch" + parent: + modelname: "location" + field: "location_name" + column: "location" + mappings: + - field: "name" + column: "name" + - field: "location_name" + reference: + key: "location" + table: "cmn_location" + column: "name" + - field: "asset_tag" + column: "asset_tag" + - field: "manufacturer_name" + reference: + key: "manufacturer" + table: "core_company" + column: "name" + - field: "model_name" + reference: + key: "model_id" + table: "cmdb_hardware_product_model" + column: "name" + - field: "serial" + column: "serial_number" +interface: + table: "cmdb_ci_network_adapter" + parent: + modelname: "device" + field: "device_name" + column: "cmdb_ci" + mappings: + - field: "name" + column: "name" + - field: "device_name" + reference: + key: "cmdb_ci" + table: "cmdb_ci_ip_switch" + column: "name" diff --git a/nautobot_ssot/integrations/servicenow/diffsync/__init__.py b/nautobot_ssot/integrations/servicenow/diffsync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py new file mode 100644 index 00000000..bdd8187e --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py @@ -0,0 +1,232 @@ +"""DiffSync adapter class for Nautobot as source-of-truth.""" + +import datetime + +from diffsync import DiffSync +from diffsync.exceptions import ObjectNotFound + +from django.contrib.contenttypes.models import ContentType + +from nautobot.dcim.models import Device, DeviceType, Interface, Manufacturer, Region, Site +from nautobot.extras.choices import CustomFieldTypeChoices +from nautobot.extras.models import CustomField, Tag +from nautobot.utilities.choices import ColorChoices + +from . import models + + +class NautobotDiffSync(DiffSync): + """Nautobot adapter for DiffSync.""" + + company = models.Company + device = models.Device # child of location + interface = models.Interface # child of device + location = models.Location + product_model = models.ProductModel # child of company + + top_level = [ + "company", + "location", + ] + + def __init__(self, *args, job, sync, site_filter=None, **kwargs): + """Initialize the NautobotDiffSync.""" + super().__init__(*args, **kwargs) + self.job = job + self.sync = sync + self.site_filter = site_filter + + def load_manufacturers(self): + """Add Manufacturers and their descendant DeviceTypes as DiffSyncModel instances.""" + for mfr_record in Manufacturer.objects.all(): + mfr = self.company(diffsync=self, name=mfr_record.name, manufacturer=True, pk=mfr_record.pk) + self.add(mfr) + for dtype_record in DeviceType.objects.filter(manufacturer=mfr_record): + dtype = self.product_model( + diffsync=self, + manufacturer_name=mfr.name, + model_name=dtype_record.model, + model_number=dtype_record.model, + pk=dtype_record.pk, + ) + self.add(dtype) + mfr.add_child(dtype) + + self.job.log_info( + message=f"Loaded {len(self.get_all('company'))} manufacturer records and " + f"{len(self.get_all('product_model'))} device-type records from Nautobot." + ) + + def load_regions(self, parent_location=None): + """Recursively add Nautobot Region objects as DiffSync Location models.""" + if self.site_filter is not None: + # Load only direct ancestors of the given Site + regions = [] + ancestor = self.site_filter.region + while ancestor is not None: + regions.insert(0, ancestor) + ancestor = ancestor.parent + else: + parent_pk = parent_location.region_pk if parent_location else None + regions = Region.objects.filter(parent=parent_pk) + + for region_record in regions: + location = self.location( + diffsync=self, + name=region_record.name, + region_pk=region_record.pk, + ) + if region_record.parent: + location.parent_location_name = region_record.parent.name + if not parent_location: + parent_location = self.get(self.location, region_record.parent.name) + if parent_location: + parent_location.contained_locations.append(location) + self.add(location) + if self.site_filter is None: + # Recursively load children of the given Region + self.load_regions(parent_location=location) + + def load_sites(self): + """Add Nautobot Site objects as DiffSync Location models.""" + for location in self.get_all(self.location): + self.job.log_debug(f"Getting Sites associated with {location}") + for site_record in Site.objects.filter(region__name=location.name): + if self.site_filter is not None and site_record != self.site_filter: + self.job.log_debug(f"Skipping site {site_record} due to site filter") + continue + # A Site and a Region may share the same name; if so they become part of the same Location record. + try: + region_location = self.get(self.location, site_record.name) + region_location.site_pk = site_record.pk + except ObjectNotFound: + site_location = self.location( + diffsync=self, + name=site_record.name, + latitude=site_record.latitude or "", + longitude=site_record.longitude or "", + site_pk=site_record.pk, + ) + self.add(site_location) + if site_record.region: + if site_record.name != site_record.region.name: + region_location = self.get(self.location, site_record.region.name) + region_location.contained_locations.append(site_location) + site_location.parent_location_name = site_record.region.name + + self.job.log_info( + message=f"Loaded {len(self.get_all('location'))} aggregated site and region records from Nautobot." + ) + + def load_interface(self, interface_record, device_model): + """Import a single Nautobot Interface object as a DiffSync Interface model.""" + interface = self.interface( + diffsync=self, + name=interface_record.name, + device_name=device_model.name, + description=interface_record.description, + pk=interface_record.pk, + ) + self.add(interface) + device_model.add_child(interface) + + def load(self): + """Load data from Nautobot.""" + self.load_manufacturers() + # Import all Nautobot Region records as Locations + self.load_regions() + + # Import all Nautobot Site records as Locations + self.load_sites() + + for location in self.get_all(self.location): + if location.site_pk is None: + continue + for device_record in Device.objects.filter(site__pk=location.site_pk): + device = self.device( + diffsync=self, + name=device_record.name, + location_name=location.name, + asset_tag=device_record.asset_tag or "", + manufacturer_name=device_record.device_type.manufacturer.name, + model_name=device_record.device_type.model, + serial=device_record.serial, + pk=device_record.pk, + ) + self.add(device) + location.add_child(device) + + for interface_record in Interface.objects.filter(device=device_record): + self.load_interface(interface_record, device) + + self.job.log_info( + message=f"Loaded {len(self.get_all('device'))} device records and " + f"{len(self.get_all('interface'))} interface records from Nautobot." + ) + + def tag_involved_objects(self, target): + """Tag all objects that were successfully synced to the target.""" + # The ssot-synced-to-servicenow tag *should* have been created automatically during plugin installation + # (see nautobot_ssot_servicenow/signals.py) but maybe a user deleted it inadvertently, so be safe: + tag, _ = Tag.objects.get_or_create( + slug="ssot-synced-to-servicenow", + defaults={ + "name": "SSoT Synced to ServiceNow", + "description": "Object synced at some point from Nautobot to ServiceNow", + "color": ColorChoices.COLOR_LIGHT_GREEN, + }, + ) + # Ensure that the "ssot-synced-to-servicenow" custom field is present; as above, it *should* already exist. + custom_field, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_DATE, + name="ssot-synced-to-servicenow", + defaults={ + "label": "Last synced to ServiceNow on", + }, + ) + for model in [Device, DeviceType, Interface, Manufacturer, Region, Site]: + custom_field.content_types.add(ContentType.objects.get_for_model(model)) + + for modelname in [ + "company", + "device", + "interface", + "location", + "product_model", + ]: + for local_instance in self.get_all(modelname): + unique_id = local_instance.get_unique_id() + # Verify that the object now has a counterpart in the target DiffSync + try: + target.get(modelname, unique_id) + except ObjectNotFound: + continue + + self.tag_object(modelname, unique_id, tag, custom_field) + + def tag_object(self, modelname, unique_id, tag, custom_field): + """Apply the given tag and custom field to the identified object.""" + model_instance = self.get(modelname, unique_id) + today = datetime.date.today().isoformat() + + def _tag_object(nautobot_object): + """Apply custom field and tag to object, if applicable.""" + if hasattr(nautobot_object, "tags"): + nautobot_object.tags.add(tag) + if hasattr(nautobot_object, "cf"): + nautobot_object.cf[custom_field.name] = today + nautobot_object.validated_save() + + if modelname == "company": + _tag_object(Manufacturer.objects.get(pk=model_instance.pk)) + elif modelname == "device": + _tag_object(Device.objects.get(pk=model_instance.pk)) + elif modelname == "interface": + _tag_object(Interface.objects.get(pk=model_instance.pk)) + elif modelname == "location": + if model_instance.region_pk is not None: + _tag_object(Region.objects.get(pk=model_instance.region_pk)) + if model_instance.site_pk is not None: + _tag_object(Site.objects.get(pk=model_instance.site_pk)) + elif modelname == "product_model": + _tag_object(DeviceType.objects.get(pk=model_instance.pk)) diff --git a/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py b/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py new file mode 100644 index 00000000..f5f3821d --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py @@ -0,0 +1,299 @@ +"""DiffSync adapter for ServiceNow.""" +from base64 import b64encode +import json +import os + +from diffsync import DiffSync +from diffsync.enum import DiffSyncFlags +from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound +from jinja2 import Environment, FileSystemLoader +import yaml + +from . import models + + +class ServiceNowDiffSync(DiffSync): + """DiffSync adapter using pysnow to communicate with a ServiceNow server.""" + + company = models.Company + device = models.Device # child of location + interface = models.Interface # child of device + location = models.Location + product_model = models.ProductModel # child of company + + top_level = [ + "company", + "location", + ] + + DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), "data")) + + def __init__(self, *args, client=None, job=None, sync=None, site_filter=None, **kwargs): + """Initialize the ServiceNowDiffSync adapter.""" + super().__init__(*args, **kwargs) + self.client = client + self.job = job + self.sync = sync + self.site_filter = site_filter + self.sys_ids = {} + self.mapping_data = [] + + # Since a device may contain dozens or hundreds of interfaces, + # to improve performance when a device is created, we use ServiceNow's bulk/batch API to + # create all of these interfaces in a single API call. + self.interfaces_to_create_per_device = {} + + def load(self): + """Load data via pysnow.""" + self.mapping_data = self.load_yaml_datafile("mappings.yaml") + + for modelname, entry in self.mapping_data.items(): + if modelname == "location" and self.site_filter is not None: + # Load the specific record, if any, corresponding to the site_filter + record = ( + self.client.resource(api_path=f"/table/{entry['table']}") + .get(query={"name": self.site_filter.name}) + .one_or_none() + ) + if record: + location = self.load_record(entry["table"], record, self.location, entry["mappings"]) + # Load all of its ServiceNow ancestors as well + name_tokens = location.full_name.split("/") + ancestor_full_name = "" + for name_token in name_tokens[:-1]: + if ancestor_full_name: + ancestor_full_name += "/" + ancestor_full_name += name_token + record = ( + self.client.resource(api_path=f"/table/{entry['table']}") + .get(query={"full_name": ancestor_full_name}) + .one_or_none() + ) + if record: + self.load_record(entry["table"], record, self.location, entry["mappings"]) + # Load all Nautobot ancestor records as well + # This is so in case the Nautobot ancestors exist in ServiceNow but aren't linked to the record, + # we link them together instead of creating new, redundant ancestor records in ServiceNow. + ancestor = self.site_filter.region + while ancestor is not None: + try: + self.get(self.location, ancestor.name) + except ObjectNotFound: + record = ( + self.client.resource(api_path=f"/table/{entry['table']}") + .get(query={"name": ancestor.name}) + .one_or_none() + ) + if record: + self.load_record(entry["table"], record, self.location, entry["mappings"]) + ancestor = ancestor.parent + + self.job.log_info( + message=f"Loaded a total of {len(self.get_all('location'))} location records from ServiceNow." + ) + + else: + self.load_table(modelname, **entry) + + @classmethod + def load_yaml_datafile(cls, filename, config=None): + """Get the contents of the given YAML data file. + + Args: + filename (str): Filename within the 'data' directory. + config (dict): Data for Jinja2 templating. + """ + file_path = os.path.join(cls.DATA_DIR, filename) + if not os.path.isfile(file_path): + raise RuntimeError(f"No data file found at {file_path}") + if not config: + config = {} + env = Environment(loader=FileSystemLoader(cls.DATA_DIR), autoescape=True) + template = env.get_template(filename) + populated = template.render(config) + return yaml.safe_load(populated) + + def load_table(self, modelname, table, mappings, **kwargs): + """Load data from the ServiceNow "table" into the DiffSync model. + + Args: + modelname (str): DiffSync model class identifier, such as "location" or "device". + table (str): ServiceNow table name, such as "cmdb_ci_ip_switch" + mappings (list): List of dicts, each stating how to populate a field in the model. + **kwargs: Optional arguments, all of which default to False if unset: + + - parent (dict): Dict of {"modelname": ..., "field": ...} used to link table records back to their parents + """ + model_cls = getattr(self, modelname) + self.job.log_info(message=f"Loading ServiceNow table `{table}` into {modelname} instances...") + + if "parent" not in kwargs: + # Load the entire table + for record in self.client.all_table_entries(table): + self.load_record(table, record, model_cls, mappings, **kwargs) + else: + # Load items per parent object that we know/care about + # This is necessary because, for example, the cmdb_ci_network_adapter table contains network interfaces + # for ALL types of devices (servers, switches, firewalls, etc.) but we only have switches as parent objects + for parent in self.get_all(kwargs["parent"]["modelname"]): + for record in self.client.all_table_entries(table, {kwargs["parent"]["column"]: parent.sys_id}): + self.load_record(table, record, model_cls, mappings, **kwargs) + + self.job.log_info( + message=f"Loaded {len(self.get_all(modelname))} {modelname} records from ServiceNow table `{table}`." + ) + + def load_record(self, table, record, model_cls, mappings, **kwargs): + """Helper method to load_table().""" + self.sys_ids.setdefault(table, {})[record["sys_id"]] = record + + ids_attrs = self.map_record_to_attrs(record, mappings) + model = model_cls(**ids_attrs) + modelname = model.get_type() + + try: + self.add(model) + except ObjectAlreadyExists: + # The baseline data in a standard ServiceNow developer instance has a number of duplicate Location entries. + # For now, ignore the duplicate entry and continue + self.job.log_warning( + message=f'Ignoring apparent duplicate record for {modelname} "{model.get_unique_id()}".' + ) + + if "parent" in kwargs: + parent_uid = getattr(model, kwargs["parent"]["field"]) + if parent_uid is None: + self.job.log_warning( + message=f'Model {modelname} "{model.get_unique_id}" does not have a parent uid value ' + f"in field {kwargs['parent']['field']}" + ) + else: + parent_model = self.get(kwargs["parent"]["modelname"], parent_uid) + parent_model.add_child(model) + + return model + + def map_record_to_attrs(self, record, mappings): # TODO pylint: disable=too-many-branches + """Helper method to load_table().""" + attrs = {"sys_id": record["sys_id"]} + for mapping in mappings: + value = None + if "column" in mapping: + value = record[mapping["column"]] + elif "reference" in mapping: + # Reference by sys_id to a field in a record in another table + table = mapping["reference"]["table"] + if "key" in mapping["reference"]: + key = mapping["reference"]["key"] + if key not in record: + self.job.log_warning(message=f"Key `{key}` is not present in record `{record}`") + else: + sys_id = record[key] + else: + raise NotImplementedError + + if sys_id: + if sys_id not in self.sys_ids.get(table, {}): + referenced_record = self.client.get_by_sys_id(table, sys_id) + if referenced_record is None: + self.job.log_warning( + message=f"Record `{record.get('name', record)}` field `{mapping['field']}` " + f"references sys_id `{sys_id}`, but that was not found in table `{table}`" + ) + else: + self.sys_ids.setdefault(table, {})[sys_id] = referenced_record + + if sys_id in self.sys_ids.get(table, {}): + value = self.sys_ids[table][sys_id][mapping["reference"]["column"]] + else: + raise NotImplementedError + + attrs[mapping["field"]] = value + + return attrs + + def bulk_create_interfaces(self): + """Bulk-create interfaces for any newly created devices as a performance optimization.""" + if not self.interfaces_to_create_per_device: + return + + self.job.log_info(message="Beginning bulk creation of interfaces in ServiceNow for newly added devices...") + + sn_resource = self.client.resource(api_path="/v1/batch") + sn_mapping_entry = self.mapping_data["interface"] + + # One batch API request per new device, consisting of requests to create each interface that the device has + for request_id, device_name in enumerate(self.interfaces_to_create_per_device.keys()): + device = self.job.lookup_object("device", device_name) + if not self.interfaces_to_create_per_device[device_name]: + self.job.log_info(obj=device, message="No interfaces to create for this device, continuing") + continue + + request_data = { + "batch_request_id": str(request_id), + "rest_requests": [], + } + + for inner_request_id, interface in enumerate(self.interfaces_to_create_per_device[device_name]): + inner_request_payload = interface.map_data_to_sn_record( + data=dict(**interface.get_identifiers(), **interface.get_attrs()), + mapping_entry=sn_mapping_entry, + ) + inner_request_body = b64encode(json.dumps(inner_request_payload).encode("utf-8")).decode("utf-8") + inner_request_data = { + "id": str(inner_request_id), + "exclude_response_headers": True, + "headers": [ + {"name": "Content-Type", "value": "application/json"}, + {"name": "Accept", "value": "application/json"}, + ], + "url": f"/api/now/table/{sn_mapping_entry['table']}", + "method": "POST", + "body": inner_request_body, + } + request_data["rest_requests"].append(inner_request_data) + + self.job.log_debug( + f'Sending bulk API request to ServiceNow to create interfaces for device "{device_name}":' + f"\n```\n{json.dumps(request_data, indent=4)}\n```" + ) + + sn_response = sn_resource.request( + "POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + data=json.dumps(request_data), + ) + + # Get the wrapped requests.Response object from the returned pysnow.Response object + response = sn_response._response # pylint: disable=protected-access + response_data = response.json() + + if response.status_code != 200: + self.job.log_failure( + obj=device, + message=f"Got status code {response.status_code} from ServiceNow when bulk-creating interfaces:" + f"\n```\n{json.dumps(response_data, indent=4)}\n```", + ) + elif response_data["unserviced_requests"]: + self.job.log_warning( + obj=device, + message="ServiceNow indicated that parts of the bulk request for interface creation " + f"were not serviced:\n```\n{json.dumps(response_data['unserviced_requests'], indent=4)}\n```", + ) + else: + self.job.log_debug( + f"ServiceNow response: {response.status_code}\n```\n{json.dumps(response_data, indent=4)}\n```" + ) + self.job.log_success(obj=device, message="Interfaces successfully bulk-created.") + + self.job.log_info(message="Bulk creation of interfaces completed.") + + def sync_complete(self, source, diff, flags=DiffSyncFlags.NONE, logger=None): + """Callback after the `sync_from` operation has completed and updated this instance. + + Note that this callback is **only** triggered if the sync actually resulted in data changes. + If there are no detected changes, this callback will **not** be called. + """ + self.bulk_create_interfaces() + + source.tag_involved_objects(target=self) diff --git a/nautobot_ssot/integrations/servicenow/diffsync/models.py b/nautobot_ssot/integrations/servicenow/diffsync/models.py new file mode 100644 index 00000000..39d48a29 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/diffsync/models.py @@ -0,0 +1,281 @@ +"""DiffSyncModel subclasses for Nautobot-to-ServiceNow data sync.""" +from typing import List, Optional, Union +import uuid + +from diffsync import DiffSyncModel +from diffsync.enum import DiffSyncStatus + +# import pysnow +from nautobot_ssot_servicenow.third_party import pysnow + + +class ServiceNowCRUDMixin: + """Mixin class for all ServiceNow models, to support CRUD operations based on mappings.yaml.""" + + _sys_id_cache = {} + """Dict of table -> column_name -> value -> sys_id.""" + + def map_data_to_sn_record(self, data, mapping_entry, existing_record=None): + """Map create/update data from DiffSync to a corresponding ServiceNow data record.""" + record = existing_record or {} + for mapping in mapping_entry.get("mappings", []): + if mapping["field"] not in data: + continue + value = data[mapping["field"]] + if "column" in mapping: + record[mapping["column"]] = value + elif "reference" in mapping: + tablename = mapping["reference"]["table"] + sys_id = None + if "column" not in mapping["reference"]: + raise NotImplementedError + column_name = mapping["reference"]["column"] + if value is not None: + # Look in the cache first + sys_id = self._sys_id_cache.get(tablename, {}).get(column_name, {}).get(value, None) + if not sys_id: + target = self.diffsync.client.get_by_query(tablename, {mapping["reference"]["column"]: value}) + if target is None: + self.diffsync.job.log_warning(message=f"Unable to find reference target in {tablename}") + else: + sys_id = target["sys_id"] + self._sys_id_cache.setdefault(tablename, {}).setdefault(column_name, {})[value] = sys_id + + record[mapping["reference"]["key"]] = sys_id + else: + raise NotImplementedError + + self.diffsync.job.log_debug(f"Mapped data {data} to record {record}") + return record + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create a new instance, data-driven by mappings.""" + entry = diffsync.mapping_data[cls.get_type()] + + model = super().create(diffsync, ids=ids, attrs=attrs) + + sn_resource = diffsync.client.resource(api_path=f"/table/{entry['table']}") + sn_record = model.map_data_to_sn_record(data=dict(**ids, **attrs), mapping_entry=entry) + sn_resource.create(payload=sn_record) + + return model + + def update(self, attrs): + """Update an existing instance, data-driven by mappings.""" + entry = self.diffsync.mapping_data[self.get_type()] + + sn_resource = self.diffsync.client.resource(api_path=f"/table/{entry['table']}") + query = self.map_data_to_sn_record(data=self.get_identifiers(), mapping_entry=entry) + try: + record = sn_resource.get(query=query).one() + except pysnow.exceptions.MultipleResults: + self.diffsync.job.log_failure( + message=f"Unsure which record to update, as query {query} matched more than one item " + f"in table {entry['table']}" + ) + return None + + sn_record = self.map_data_to_sn_record(data=attrs, mapping_entry=entry, existing_record=record) + sn_resource.update(query=query, payload=sn_record) + + super().update(attrs) + return self + + # TODO delete() method + + +class Company(ServiceNowCRUDMixin, DiffSyncModel): + """ServiceNow Company model.""" + + _modelname = "company" + _identifiers = ("name",) + _attributes = ("manufacturer",) + _children = { + "product_model": "product_models", + } + + name: str + manufacturer: bool = False + + product_models: List["ProductModel"] = list() + + sys_id: Optional[str] = None + pk: Optional[uuid.UUID] = None + + +class ProductModel(ServiceNowCRUDMixin, DiffSyncModel): + """ServiceNow Hardware Product Model model.""" + + _modelname = "product_model" + _identifiers = ("manufacturer_name", "model_name", "model_number") + + manufacturer_name: Optional[str] # some ServiceNow products have no associated manufacturer? + # Nautobot has only one combined "model" field, but ServiceNow has both name and number + model_name: str + model_number: str + + sys_id: Optional[str] = None + pk: Optional[uuid.UUID] = None + + +class Location(ServiceNowCRUDMixin, DiffSyncModel): + """ServiceNow Location model.""" + + _modelname = "location" + _identifiers = ("name",) + _attributes = ( + "parent_location_name", + "latitude", + "longitude", + ) + _children = { + "device": "devices", + } + + name: str + + parent_location_name: Optional[str] + contained_locations: List["Location"] = list() + latitude: Union[float, str] = "" # can't use Optional[float] because an empty string doesn't map to None + longitude: Union[float, str] = "" + + devices: List["Device"] = list() + + sys_id: Optional[str] = None + region_pk: Optional[uuid.UUID] = None + site_pk: Optional[uuid.UUID] = None + + full_name: Optional[str] = None + + +class Device(ServiceNowCRUDMixin, DiffSyncModel): + """ServiceNow Device model.""" + + _modelname = "device" + _identifiers = ("name",) + # For now we do not store more of the device fields in ServiceNow: + # platform, model, role, vendor + # ...as we would need to sync these data models to ServiceNow as well, and we don't do that yet. + _attributes = ( + "location_name", + "asset_tag", + "manufacturer_name", + "model_name", + "serial", + ) + _children = { + "interface": "interfaces", + } + + name: str + + location_name: Optional[str] + asset_tag: Optional[str] + manufacturer_name: Optional[str] + model_name: Optional[str] + serial: Optional[str] + + platform: Optional[str] + role: Optional[str] + vendor: Optional[str] + + interfaces: List["Interface"] = list() + + sys_id: Optional[str] = None + pk: Optional[uuid.UUID] = None + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create a new Device instance, and set things up for eventual bulk-creation of its child Interfaces.""" + model = super().create(diffsync, ids=ids, attrs=attrs) + + diffsync.job.log_debug(f'New Device "{ids["name"]}" is being created, will bulk-create its interfaces later.') + diffsync.interfaces_to_create_per_device[ids["name"]] = [] + + return model + + # TODO delete() method + + +class Interface(ServiceNowCRUDMixin, DiffSyncModel): + """ServiceNow Interface model.""" + + _modelname = "interface" + _identifiers = ( + "device_name", + "name", + ) + _shortname = ("name",) + # ServiceNow currently stores very little data about interfaces that we are interested in + _attributes = () + + _children = {"ip_address": "ip_addresses"} + + name: str + device_name: str + + access_vlan: Optional[int] + active: Optional[bool] + allowed_vlans: List[str] = list() + description: Optional[str] + is_virtual: Optional[bool] + is_lag: Optional[bool] + is_lag_member: Optional[bool] + lag_members: List[str] = list() + mode: Optional[str] # TRUNK, ACCESS, L3, NONE + mtu: Optional[int] + parent: Optional[str] + speed: Optional[int] + switchport_mode: Optional[str] + type: Optional[str] + + ip_addresses: List["IPAddress"] = list() + + sys_id: Optional[str] = None + pk: Optional[uuid.UUID] = None + + @classmethod + def create(cls, diffsync, ids, attrs): + """Create an interface in isolation, or if the parent Device is new as well, defer for later bulk-creation.""" + if ids["device_name"] in diffsync.interfaces_to_create_per_device: + diffsync.job.log_debug( + f'Device "{ids["device_name"]}" was just created; deferring creation of interface "{ids["name"]}"' + ) + # copy-paste of DiffSyncModel's create() classmethod; + # we don't want to call super().create() here as that would be ServiceNowCRUDMixin.create(), + # which is what we're trying to avoid here! + model = cls(**ids, diffsync=diffsync, **attrs) + model.set_status(DiffSyncStatus.SUCCESS, "Deferred creation in ServiceNow") + diffsync.interfaces_to_create_per_device[ids["device_name"]].append(model) + else: + model = super().create(diffsync, ids=ids, attrs=attrs) + return model + + # TODO delete() method + + +class IPAddress(ServiceNowCRUDMixin, DiffSyncModel): + """An IPv4 or IPv6 address.""" + + _modelname = "ip_address" + _identifiers = ("address",) + _attributes = ( + "device_name", + "interface_name", + ) + + address: str # TODO: change to netaddr.IPAddress? + + device_name: Optional[str] + interface_name: Optional[str] + + sys_id: Optional[str] = None + pk: Optional[uuid.UUID] = None + + +Company.update_forward_refs() +Device.update_forward_refs() +Interface.update_forward_refs() +Location.update_forward_refs() +ProductModel.update_forward_refs() diff --git a/nautobot_ssot/integrations/servicenow/forms.py b/nautobot_ssot/integrations/servicenow/forms.py new file mode 100644 index 00000000..bee23d12 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/forms.py @@ -0,0 +1,28 @@ +"""User-facing forms for nautobot-ssot-servicenow.""" +from django import forms + +from nautobot.extras.models import SecretsGroup +from nautobot.utilities.forms import DynamicModelChoiceField + +from .models import SSOTServiceNowConfig + + +class SSOTServiceNowConfigForm(forms.ModelForm): + """Plugin configuration form for nautobot-ssot-servicenow.""" + + servicenow_instance = forms.CharField( + required=True, + help_text="ServiceNow instance name, will be used as <instance>.servicenow.com.", + ) + servicenow_secrets = DynamicModelChoiceField( + queryset=SecretsGroup.objects.all(), + required=True, + null_option="None", + help_text="Secrets group for authentication to ServiceNow. Should contain a REST username and REST password.", + ) + + class Meta: + """Meta class properties.""" + + model = SSOTServiceNowConfig + fields = ["servicenow_instance", "servicenow_secrets"] diff --git a/nautobot_ssot/integrations/servicenow/jobs.py b/nautobot_ssot/integrations/servicenow/jobs.py new file mode 100644 index 00000000..1015ed74 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/jobs.py @@ -0,0 +1,140 @@ +"""ServiceNow Data Target Job.""" +from django.core.exceptions import ObjectDoesNotExist +from django.templatetags.static import static +from django.urls import reverse + +from diffsync.enum import DiffSyncFlags + +from nautobot.dcim.models import Device, DeviceType, Interface, Manufacturer, Region, Site +from nautobot.extras.jobs import Job, BooleanVar, ObjectVar + +from nautobot_ssot.jobs.base import DataMapping, DataTarget + +from .diffsync.adapter_nautobot import NautobotDiffSync +from .diffsync.adapter_servicenow import ServiceNowDiffSync +from .servicenow import ServiceNowClient +from .utils import get_servicenow_parameters + + +name = "SSoT - ServiceNow" # pylint: disable=invalid-name + + +class ServiceNowDataTarget(DataTarget, Job): # pylint: disable=abstract-method + """Job syncing data from Nautobot to ServiceNow.""" + + debug = BooleanVar(description="Enable for more verbose logging.") + + log_unchanged = BooleanVar( + description="Create log entries even for unchanged objects", + default=False, + ) + + # TODO: not yet implemented + # delete_records = BooleanVar( + # description="Delete records from ServiceNow if not present in Nautobot", + # default=False, + # ) + + site_filter = ObjectVar( + description="Only sync records belonging to a single Site.", + model=Site, + default=None, + required=False, + ) + + class Meta: + """Metadata about this Job.""" + + name = "Nautobot ⟹ ServiceNow" + data_target = "ServiceNow" + data_target_icon = static("nautobot_ssot_servicenow/ServiceNow_logo.svg") + description = "Synchronize data from Nautobot into ServiceNow." + + @classmethod + def data_mappings(cls): + """List describing the data mappings involved in this DataTarget.""" + return ( + DataMapping("Device", reverse("dcim:device_list"), "IP Switch", None), + DataMapping("Device Type", reverse("dcim:devicetype_list"), "Hardware Product Model", None), + DataMapping("Interface", reverse("dcim:interface_list"), "Interface", None), + DataMapping("Manufacturer", reverse("dcim:manufacturer_list"), "Company", None), + DataMapping("Region", reverse("dcim:region_list"), "Location", None), + DataMapping("Site", reverse("dcim:site_list"), "Location", None), + ) + + @classmethod + def config_information(cls): + """Dictionary describing the configuration of this DataTarget.""" + configs = get_servicenow_parameters() + return { + "ServiceNow instance": configs.get("instance"), + "Username": configs.get("username"), + # Password is intentionally omitted! + } + + def sync_data(self): + """Sync a slew of Nautobot data into ServiceNow.""" + configs = get_servicenow_parameters() + snc = ServiceNowClient( + instance=configs.get("instance"), + username=configs.get("username"), + password=configs.get("password"), + worker=self, + ) + + self.log_info(message="Loading current data from ServiceNow...") + servicenow_diffsync = ServiceNowDiffSync( + client=snc, job=self, sync=self.sync, site_filter=self.kwargs.get("site_filter") + ) + servicenow_diffsync.load() + + self.log_info(message="Loading current data from Nautobot...") + nautobot_diffsync = NautobotDiffSync(job=self, sync=self.sync, site_filter=self.kwargs.get("site_filter")) + nautobot_diffsync.load() + + diffsync_flags = DiffSyncFlags.CONTINUE_ON_FAILURE + if self.kwargs.get("log_unchanged"): + diffsync_flags |= DiffSyncFlags.LOG_UNCHANGED_RECORDS + if not self.kwargs.get("delete_records"): + diffsync_flags |= DiffSyncFlags.SKIP_UNMATCHED_DST + + self.log_info(message="Calculating diffs...") + diff = servicenow_diffsync.diff_from(nautobot_diffsync, flags=diffsync_flags) + self.sync.diff = diff.dict() + self.sync.save() + + if not self.kwargs["dry_run"]: + self.log_info(message="Syncing from Nautobot to ServiceNow...") + servicenow_diffsync.sync_from(nautobot_diffsync, flags=diffsync_flags) + self.log_info(message="Sync complete") + + def log_debug(self, message): + """Conditionally log a debug message.""" + if self.kwargs.get("debug"): + super().log_debug(message) + + def lookup_object(self, model_name, unique_id): + """Look up a Nautobot object based on the DiffSync model name and unique ID.""" + obj = None + try: + if model_name == "company": + obj = Manufacturer.objects.get(name=unique_id) + elif model_name == "device": + obj = Device.objects.get(name=unique_id) + elif model_name == "interface": + device_name, interface_name = unique_id.split("__") + obj = Interface.objects.get(device__name=device_name, name=interface_name) + elif model_name == "location": + try: + obj = Site.objects.get(name=unique_id) + except Site.DoesNotExist: + obj = Region.objects.get(name=unique_id) + elif model_name == "product_model": + manufacturer, model, _ = unique_id.split("__") + obj = DeviceType.objects.get(manufacturer__name=manufacturer, model=model) + except ObjectDoesNotExist: + pass + return obj + + +jobs = [ServiceNowDataTarget] diff --git a/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py b/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py new file mode 100644 index 00000000..70cefe25 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py @@ -0,0 +1,38 @@ +# Generated by Django 3.1.12 on 2021-12-21 21:49 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("extras", "0018_joblog_data_migration"), + ] + + operations = [ + migrations.CreateModel( + name="SSOTServiceNowConfig", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True + ), + ), + ("servicenow_instance", models.CharField(blank=True, max_length=100)), + ( + "servicenow_secrets", + models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="extras.secretsgroup" + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/nautobot_ssot/integrations/servicenow/migrations/__init__.py b/nautobot_ssot/integrations/servicenow/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nautobot_ssot/integrations/servicenow/models.py b/nautobot_ssot/integrations/servicenow/models.py new file mode 100644 index 00000000..2fa5611c --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/models.py @@ -0,0 +1,42 @@ +"""Configuration data model for nautobot-ssot-servicenow.""" + +from django.db import models +from django.shortcuts import reverse + +from nautobot.core.models import BaseModel + + +class SSOTServiceNowConfig(BaseModel): + """Singleton data model describing the configuration of this plugin.""" + + def delete(self, *args, **kwargs): + """Cannot be deleted.""" + + @classmethod + def load(cls): + """Singleton instance getter.""" + if cls.objects.all().exists(): + return cls.objects.first() + return cls.objects.create() + + servicenow_instance = models.CharField( + max_length=100, + blank=True, + help_text="ServiceNow instance name, will be used as <instance>.servicenow.com.", + ) + + servicenow_secrets = models.ForeignKey( + to="extras.SecretsGroup", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text="Secrets group for authentication to ServiceNow. Should contain a REST username and REST password.", + ) + + def __str__(self): + """String representation of singleton instance.""" + return "SSoT ServiceNow Configuration" + + def get_absolute_url(self): # pylint: disable=no-self-use + """Get URL for the associated configuration view.""" + return reverse("plugins:nautobot_ssot_servicenow:config") diff --git a/nautobot_ssot/integrations/servicenow/servicenow.py b/nautobot_ssot/integrations/servicenow/servicenow.py new file mode 100644 index 00000000..af0c7249 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/servicenow.py @@ -0,0 +1,55 @@ +"""Interactions with ServiceNow APIs.""" +import logging + +# from pysnow import Client +from nautobot_ssot_servicenow.third_party.pysnow import Client + +# from pysnow.exceptions import MultipleResults +from nautobot_ssot_servicenow.third_party.pysnow.exceptions import MultipleResults +import requests # pylint: disable=wrong-import-order + + +logger = logging.getLogger(__name__) + + +class ServiceNowClient(Client): + """Extend the pysnow Client with additional use-case-specific functionality.""" + + def __init__(self, instance=None, username=None, password=None, worker=None): + """Create a ServiceNowClient with the appropriate environment parameters.""" + super().__init__(instance=instance, user=username, password=password) + + self.worker = worker + + # When getting records from ServiceNow, for reference fields, only return the sys_id value of the reference, + # rather than returning a dict of {"link": "https://.servicenow.com/...", "value": } + # We don't need the link for our purposes, and including it makes it harder to preserve idempotence. + self.parameters.exclude_reference_link = True + + def all_table_entries(self, table, query=None): + """Iterator over all records in a given table.""" + if not query: + query = {} + logger.debug("Getting all entries in table %s matching query %s", table, query) + yield from self.resource(api_path=f"/table/{table}").get(query=query, stream=True).all() + + def get_by_sys_id(self, table, sys_id): + """Get a record with a given sys_id from a given table.""" + return self.get_by_query(table, {"sys_id": sys_id}) + + def get_by_query(self, table, query): + """Get a specific record from a given table.""" + logger.debug("Querying table %s with query %s", table, query) + try: + result = self.resource(api_path=f"/table/{table}").get(query=query).one_or_none() + except requests.exceptions.HTTPError as exc: + # Raised if for example we get a 400 response because we're querying a nonexistent table + logger.error("HTTP error encountered: %s", exc) + return None + except MultipleResults: + logger.error('Multiple results unexpectedly returned when querying table "%s" with "%s"', table, query) + return None + + if not result: + logger.warning("Query %s did not match an object in table %s", query, table) + return result diff --git a/nautobot_ssot/integrations/servicenow/signals.py b/nautobot_ssot/integrations/servicenow/signals.py new file mode 100644 index 00000000..e361a3af --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/signals.py @@ -0,0 +1,43 @@ +"""Signal handlers for nautobot_ssot_servicenow.""" + +from nautobot.extras.choices import CustomFieldTypeChoices +from nautobot.utilities.choices import ColorChoices + + +def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disable=unused-argument + """Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready.""" + # pylint: disable=invalid-name + ContentType = apps.get_model("contenttypes", "ContentType") + CustomField = apps.get_model("extras", "CustomField") + Device = apps.get_model("dcim", "Device") + DeviceType = apps.get_model("dcim", "DeviceType") + Interface = apps.get_model("dcim", "Interface") + Manufacturer = apps.get_model("dcim", "Manufacturer") + Region = apps.get_model("dcim", "Region") + Site = apps.get_model("dcim", "Site") + Tag = apps.get_model("extras", "Tag") + + Tag.objects.get_or_create( + slug="ssot-synced-to-servicenow", + defaults={ + "name": "SSoT Synced to ServiceNow", + "description": "Object synced at some point from Nautobot to ServiceNow", + "color": ColorChoices.COLOR_LIGHT_GREEN, + }, + ) + custom_field, _ = CustomField.objects.get_or_create( + type=CustomFieldTypeChoices.TYPE_DATE, + name="ssot-synced-to-servicenow", + defaults={ + "label": "Last synced to ServiceNow on", + }, + ) + for content_type in [ + ContentType.objects.get_for_model(Device), + ContentType.objects.get_for_model(DeviceType), + ContentType.objects.get_for_model(Interface), + ContentType.objects.get_for_model(Manufacturer), + ContentType.objects.get_for_model(Region), + ContentType.objects.get_for_model(Site), + ]: + custom_field.content_types.add(content_type) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/__init__.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/__init__.py new file mode 100644 index 00000000..33e767dc --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from .client import Client +from .oauth_client import OAuthClient +from .query_builder import QueryBuilder +from .resource import Resource +from .params_builder import ParamsBuilder + +# Set default logging handler to avoid "No handler found" warnings. +import logging + +try: # Python 2.7+ + from logging import NullHandler +except ImportError: # pragma: no cover + + class NullHandler(logging.Handler): + def emit(self, record): + pass + + +logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/attachment.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/attachment.py new file mode 100644 index 00000000..d97927d4 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/attachment.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +import os +import warnings + +try: + import magic + + HAS_MAGIC = True +except ImportError: # pragma: no cover + HAS_MAGIC = False + warnings.warn( + "Missing the python-magic library; attachment content-type will be set to text/plain. " + "Try installing the python-magic-bin package if running on Mac or Windows" + ) + +from .exceptions import InvalidUsage + + +class Attachment(object): + """Attachment management + + :param resource: Table API resource to manage attachments for + :param table_name: Name of the table to use in the attachment API + """ + + def __init__(self, resource, table_name): + self.resource = resource + self.table_name = table_name + + def get(self, sys_id=None, limit=100): + """Returns a list of attachments + + :param sys_id: record sys_id to list attachments for + :param limit: override the default limit of 100 + :return: list of attachments + """ + + if sys_id: + return self.resource.get( + query={"table_sys_id": sys_id, "table_name": self.table_name} + ).all() + + return self.resource.get( + query={"table_name": self.table_name}, limit=limit + ).all() + + def upload(self, sys_id, file_path, name=None, multipart=False): + """Attaches a new file to the provided record + + :param sys_id: the sys_id of the record to attach the file to + :param file_path: local absolute path of the file to upload + :param name: custom name for the uploaded file (instead of basename) + :param multipart: whether or not to use multipart + :return: the inserted record + """ + + if not isinstance(multipart, bool): + raise InvalidUsage("Multipart must be of type bool") + + resource = self.resource + + if name is None: + name = os.path.basename(file_path) + + resource.parameters.add_custom( + {"table_name": self.table_name, "table_sys_id": sys_id, "file_name": name} + ) + + data = open(file_path, "rb").read() + headers = {} + + if multipart: + headers["Content-Type"] = "multipart/form-data" + path_append = "/upload" + else: + headers["Content-Type"] = ( + magic.from_file(file_path, mime=True) if HAS_MAGIC else "text/plain" + ) + path_append = "/file" + + return resource.request( + method="POST", data=data, headers=headers, path_append=path_append + ) + + def delete(self, sys_id): + """Deletes the provided attachment record + + :param sys_id: attachment sys_id + :return: delete result + """ + + return self.resource.delete(query={"sys_id": sys_id}) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py new file mode 100644 index 00000000..7e67d16f --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- + +import logging +import inspect +import warnings + +import requests +from nautobot_ssot_servicenow.third_party import pysnow + +from requests.auth import HTTPBasicAuth +from .legacy_request import LegacyRequest +from .exceptions import InvalidUsage +from .resource import Resource +from .url_builder import URLBuilder +from .params_builder import ParamsBuilder + +logger = logging.getLogger("pysnow") + + +class Client(object): + """User-created Client object. + + :param instance: Instance name, used to construct host + :param host: Host can be passed as an alternative to instance + :param user: User name + :param password: Password + :param raise_on_empty: Whether or not to raise an exception on 404 (no matching records), defaults to True + :param request_params: Request params to send with requests globally (deprecated) + :param use_ssl: Enable or disable the use of SSL, defaults to True + :param session: Optional :class:`requests.Session` object to use instead of passing user/pass to :class:`Client` + :raises: + - InvalidUsage: On argument validation error + """ + + def __init__( + self, + instance=None, + host=None, + user=None, + password=None, + raise_on_empty=None, + request_params=None, + use_ssl=True, + session=None, + ): + + if (host and instance) is not None: + raise InvalidUsage( + "Arguments 'instance' and 'host' are mutually exclusive, you cannot use both." + ) + + if type(use_ssl) is not bool: + raise InvalidUsage("Argument 'use_ssl' must be of type bool") + + if raise_on_empty is None: + self.raise_on_empty = True + elif type(raise_on_empty) is bool: + warnings.warn( + "The use of the `raise_on_empty` argument is deprecated and will be removed in a " + "future release.", + DeprecationWarning, + ) + + self.raise_on_empty = raise_on_empty + else: + raise InvalidUsage("Argument 'raise_on_empty' must be of type bool") + + if not (host or instance): + raise InvalidUsage("You must supply either 'instance' or 'host'") + + if not isinstance(self, pysnow.OAuthClient): + if not (user and password) and not session: + raise InvalidUsage( + "You must supply either username and password or a session object" + ) + elif (user and session) is not None: + raise InvalidUsage( + "Provide either username and password or a session, not both." + ) + + self.parameters = ParamsBuilder() + + if request_params is not None: + warnings.warn( + "The use of the `request_params` argument is deprecated and will be removed in a " + "future release. Please use Client.parameters instead.", + DeprecationWarning, + ) + + self.parameters.add_custom(request_params) + + self.request_params = request_params or {} + self.instance = instance + self.host = host + self._user = user + self._password = password + self.use_ssl = use_ssl + self.base_url = URLBuilder.get_base_url(use_ssl, instance, host) + + if not isinstance(self, pysnow.OAuthClient): + self.session = self._get_session(session) + else: + self.session = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + + def close(self): + self.session.close() + + def _get_session(self, session): + """Creates a new session with basic auth, unless one was provided, and sets headers. + + :param session: (optional) Session to re-use + :return: + - :class:`requests.Session` object + """ + + if not session: + logger.debug("(SESSION_CREATE) User: %s" % self._user) + s = requests.Session() + s.auth = HTTPBasicAuth(self._user, self._password) + else: + logger.debug("(SESSION_CREATE) Object: %s" % session) + s = session + + s.headers.update( + { + "content-type": "application/json", + "accept": "application/json", + "User-Agent": "pysnow", + } + ) + + return s + + def _legacy_request(self, method, table, **kwargs): + """Returns a :class:`LegacyRequest` object, compatible with Client.query and Client.insert + + :param method: HTTP method + :param table: Table to operate on + :return: + - :class:`LegacyRequest` object + """ + + warnings.warn( + "`%s` is deprecated and will be removed in a future release. " + "Please use `resource()` instead." % inspect.stack()[1][3], + DeprecationWarning, + ) + + return LegacyRequest( + method, + table, + request_params=self.request_params, + raise_on_empty=self.raise_on_empty, + session=self.session, + instance=self.instance, + base_url=self.base_url, + **kwargs + ) + + def resource(self, api_path=None, base_path="/api/now", chunk_size=None, **kwargs): + """Creates a new :class:`Resource` object after validating paths + + :param api_path: Path to the API to operate on + :param base_path: (optional) Base path override + :param chunk_size: Response stream parser chunk size (in bytes) + :param **kwargs: Pass request.request parameters to the Resource object + :return: + - :class:`Resource` object + :raises: + - InvalidUsage: If a path fails validation + """ + + for path in [api_path, base_path]: + URLBuilder.validate_path(path) + + return Resource( + api_path=api_path, + base_path=base_path, + parameters=self.parameters, + chunk_size=chunk_size or 8192, + session=self.session, + base_url=self.base_url, + **kwargs + ) + + def query(self, table, **kwargs): + """Query (GET) request wrapper. + + :param table: table to perform query on + :param kwargs: Keyword arguments passed along to `Request` + :return: + - List of dictionaries containing the matching records + """ + + return self._legacy_request("GET", table, **kwargs) + + def insert(self, table, payload, **kwargs): + """Insert (POST) request wrapper + + :param table: table to insert on + :param payload: update payload (dict) + :param kwargs: Keyword arguments passed along to `Request` + :return: + - Dictionary containing the created record + """ + + r = self._legacy_request("POST", table, **kwargs) + return r.insert(payload) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/criterion.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/criterion.py new file mode 100644 index 00000000..318aa199 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/criterion.py @@ -0,0 +1,506 @@ +import inspect +import six +import pytz +from datetime import datetime + +from .enums import ( + Boolean, + Equality, + DateTimeOn, + Order, +) +from .exceptions import QueryTypeError + + +class Term(object): + @staticmethod + def wrap_constant(value, types, list_type=False): + if list_type: + if isinstance(value, (list, tuple)): + if all(type(x) in types for x in value): + return ListValueWrapper(value, types) + else: + caller = inspect.currentframe().f_back.f_code.co_name + raise QueryTypeError( + "Expected value to be a list of type %s, not %s" + % (types, type(value)) + ) + else: + caller = inspect.currentframe().f_back.f_code.co_name + raise QueryTypeError( + "Invalid type passed to %s() , expected list or tuple" % (caller) + ) + elif isinstance(value, ValueWrapper) and ( + value.type_ in types or (value.type_ == list and list_type) + ): + return value + # allow other types than datetime, as long as they have strftime + elif hasattr(value, "strftime") and datetime in types: + return DateTimeValueWrapper(value) + elif not type(value) in types: + caller = inspect.currentframe().f_back.f_code.co_name + raise QueryTypeError( + "Invalid type passed to %s() , expected: %s" % (caller, types) + ) + elif isinstance(value, int): + return IntValueWrapper(value) + elif isinstance(value, str): + return StringValueWrapper(value) + else: + return value + + def eq(self, other): + return self == other + + def gt(self, other): + return self > other + + def gte(self, other): + return self >= other + + def lt(self, other): + return self < other + + def lte(self, other): + return self <= other + + def ne(self, other): + return self != other + + def is_empty(self): + return IsEmptyCriterion(self) + + def is_not_empty(self): + return NotEmptyCriterion(self) + + def is_empty_string(self): + return IsEmptyStringCriterion(self) + + def between(self, lower, upper): + return BetweenCriterion( + self, + self.wrap_constant(lower, types=[int, datetime]), + self.wrap_constant(upper, types=[int, datetime]), + ) + + def starts_with(self, other): + return BasicCriterion( + "STARTSWITH", self, self.wrap_constant(other, types=[str]) + ) + + def ends_with(self, other): + return BasicCriterion("ENDSWITH", self, self.wrap_constant(other, types=[str])) + + def contains(self, other): + return self.like(other) + + def not_contains(self, other): + return self.not_like(other) + + def like(self, other): + return BasicCriterion("LIKE", self, self.wrap_constant(other, types=[str])) + + def not_like(self, other): + return BasicCriterion("NOT LIKE", self, self.wrap_constant(other, types=[str])) + + def is_in(self, other): + return BasicCriterion( + "IN", self, self.wrap_constant(other, types=[int, str], list_type=True) + ) + + def not_in(self, other): + return BasicCriterion( + "NOT IN", self, self.wrap_constant(other, types=[int, str], list_type=True) + ) + + def is_anything(self, other): + return IsAnythingCriterion(self) + + def is_same(self, other): + return BasicCriterion("SAMEAS", self, self.wrap_constant(other, types=[Field])) + + def is_different(self, other): + return BasicCriterion("NSAMEAS", self, self.wrap_constant(other, types=[Field])) + + def __eq__(self, other): + return BasicCriterion( + Equality.eq, self, self.wrap_constant(other, types=[int, str]) + ) + + def __ne__(self, other): + return BasicCriterion( + Equality.ne, self, self.wrap_constant(other, types=[int, str]) + ) + + def __gt__(self, other): + return BasicCriterion( + Equality.gt, self, self.wrap_constant(other, types=[int, datetime]) + ) + + def __ge__(self, other): + return BasicCriterion( + Equality.gte, self, self.wrap_constant(other, types=[int, datetime]) + ) + + def __lt__(self, other): + return BasicCriterion( + Equality.lt, self, self.wrap_constant(other, types=[int, datetime]) + ) + + def __le__(self, other): + return BasicCriterion( + Equality.lte, self, self.wrap_constant(other, types=[int, datetime]) + ) + + # DateTime only + def on(self, other): + return DateTimeOnCriterion( + self, self.wrap_constant(other, types=[datetime, DateTimeOn]) + ) + + def not_on(self, other): + return DateTimeNotOnCriterion( + self, self.wrap_constant(other, types=[datetime, DateTimeOn]) + ) + + # End DateTime only + + def order(self, direction): + return OrderCriterion(self, direction) + + def __str__(self): + return self.get_query() + + def get_query(self, **kwargs): + raise NotImplementedError() + + +class Criterion(Term): + def AND(self, other): + return self & other + + def OR(self, other): + return self | other + + def NQ(self, other): + return self ^ other + + def __and__(self, other): + return BasicCriterion(Boolean.and_, self, other) + + def __or__(self, other): + return BasicCriterion(Boolean.or_, self, other) + + def __xor__(self, other): + """ + While not really an XOR operation, this allows us to use Python's bitwise operators + """ + return BasicCriterion(Boolean.nq_, self, other) + + @staticmethod + def any(terms=()): + crit = EmptyCriterion() + + for term in terms: + crit |= term + + return crit + + @staticmethod + def all(terms=()): + crit = EmptyCriterion() + + for term in terms: + crit &= term + + return crit + + def get_query(self, **kwargs): + raise NotImplementedError() + + +class EmptyCriterion(object): + def __and__(self, other): + return other + + def __or__(self, other): + return other + + def __xor__(self, other): + return other + + +class BasicCriterion(Criterion): + def __init__(self, comparator, left, right): + """ + A wrapper for a basic criterion such as equality or inequality. + This wraps three parts, a left and right term and a comparator which + defines the type of comparison. + + + :param comparator: + Type: Comparator + This defines the type of comparison, such as {quote}={quote} or + {quote}>{quote}. + :param left: + The term on the left side of the expression. + :param right: + The term on the right side of the expression. + """ + super(BasicCriterion, self).__init__() + self.comparator = comparator + self.left = left + self.right = right + + def get_query(self, **kwargs): + return "{left}{comparator}{right}".format( + comparator=getattr(self.comparator, "value", self.comparator), + left=self.left.get_query(**kwargs), + right=self.right.get_query(**kwargs), + ) + + +class IsEmptyCriterion(Criterion): + def __init__(self, term): + super(IsEmptyCriterion, self).__init__() + self.term = term + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + return "{}ISEMPTY".format(term) + + +class NotEmptyCriterion(Criterion): + def __init__(self, term): + super(NotEmptyCriterion, self).__init__() + self.term = term + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + return "{}ISNOTEMPTY".format(term) + + +class IsAnythingCriterion(Criterion): + def __init__(self, term): + super(IsAnythingCriterion, self).__init__() + self.term = term + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + return "{}ANYTHING".format(term) + + +class IsEmptyStringCriterion(Criterion): + def __init__(self, term): + super(IsEmptyStringCriterion, self).__init__() + self.term = term + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + return "{}EMPTYSTRING".format(term) + + +class BetweenCriterion(Criterion): + def __init__(self, term, start, end): + super(BetweenCriterion, self).__init__() + self.term = term + self.start = start + self.end = end + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + start = self.start.get_query(**kwargs) + end = self.end.get_query(**kwargs) + + if isinstance(self.start, DateTimeValueWrapper) and isinstance( + self.end, DateTimeValueWrapper + ): + dt_between = "%s@%s" % (start, end) + elif isinstance(self.start, IntValueWrapper) and isinstance( + self.end, IntValueWrapper + ): + dt_between = "%d@%d" % (start, end) + else: + raise QueryTypeError( + "Expected both `start` and `end` of type `int` " + "or instance of `datetime`, not %s and %s" % (type(start), type(end)) + ) + + return "{}BETWEEN{}".format(term, dt_between) + + +class DateTimeOnCriterion(Criterion): + def __init__(self, term, criteria): + super(DateTimeOnCriterion, self).__init__() + self.term = term + self.criteria = criteria + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + if isinstance(self.criteria, DateTimeOn): + return "{term}ON{criteria}".format(term=term, criteria=self.criteria.value) + else: + return "{term}ONcustom@{start}@{end}".format( + term=term, + start=self.criteria.get_query(date_only=True, extra_param="start"), + end=self.criteria.get_query(date_only=True, extra_param="end"), + ) + + +class DateTimeNotOnCriterion(Criterion): + def __init__(self, term, criteria): + super(DateTimeNotOnCriterion, self).__init__() + self.term = term + self.criteria = criteria + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + if isinstance(self.criteria, DateTimeOn): + return "{term}NOTON{criteria}".format( + term=term, criteria=self.criteria.value + ) + else: + return "{term}NOTONcustom@{start}@{end}".format( + term=term, + start=self.criteria.get_query(date_only=True, extra_param="start"), + end=self.criteria.get_query(date_only=True, extra_param="end"), + ) + + +class OrderCriterion(Criterion): + def __init__(self, term, direction): + super(OrderCriterion, self).__init__() + self.term = term + self.direction = direction + + def get_query(self, **kwargs): + term = self.term.get_query(**kwargs) + if self.direction == Order.asc or ( + isinstance(self.direction, six.string_types) + and self.direction.lower() == "asc" + ): + return "ORDERBY{term}".format(term=term) + elif self.direction == Order.desc or ( + isinstance(self.direction, six.string_types) + and self.direction.lower() == "desc" + ): + return "ORDERBYDESC{term}".format(term=term) + else: + raise QueryTypeError( + "Expected 'asc', 'desc', or an instance of Order, not %s" + % (type(self.direction)) + ) + + +class ValueWrapper(Term): + def __init__(self, type_): + self.type_ = type_ + + +class IntValueWrapper(ValueWrapper): + def __init__(self, value): + super(IntValueWrapper, self).__init__(int) + self.value = value + + def get_query(self, **kwargs): + if isinstance(self.value, int): + return self.value + else: + raise QueryTypeError( + "Expected value to be an instance of `int`, not %s" % type(self.value) + ) + + +class StringValueWrapper(ValueWrapper): + def __init__(self, value): + super(StringValueWrapper, self).__init__(str) + self.value = value + + def get_query(self, **kwargs): + if isinstance(self.value, six.string_types): + return self.value + else: + raise QueryTypeError( + "Expected value to be an instance of `str`, not %s" % type(self.value) + ) + + +class DateTimeValueWrapper(ValueWrapper): + def __init__(self, value): + super(DateTimeValueWrapper, self).__init__(datetime) + self.value = value + + def get_query(self, date_only=False, extra_param=None, **kwargs): + if hasattr(self.value, "strftime"): + datetime_ = datetime_as_utc(self.value) + if date_only: + value = datetime_.strftime("%Y-%m-%d") + else: + value = datetime_.strftime("%Y-%m-%d %H:%M:%S") + + if extra_param: + value += '", "{}'.format(extra_param) + + return 'javascript:gs.dateGenerate("{}")'.format(value) + else: + raise QueryTypeError( + "Expected value to be an instance of `datetime`, not %s" + % type(self.value) + ) + + +class ListValueWrapper(ValueWrapper): + def __init__(self, value, types): + super(ListValueWrapper, self).__init__(list) + self.value = value + self.types = types + + def get_query(self, **kwargs): + if isinstance(self.value, (list, tuple)) and all( + type(x) in self.types for x in self.value + ): + return ",".join(map(str, self.value)) + else: + raise QueryTypeError( + "Expected value to be a list of type %s, not %s" + % (self.types, type(self.value)) + ) + + +class Field(Criterion): + def __init__(self, name): + super(Field, self).__init__() + self.name = name + + def get_query(self, **kwargs): + return self.name + + +class Table(object): + """ + Allows the following: + + ``` + incident = Table(incident) + criterion = incident.company.eq('3dasd3') + ``` + """ + + # Could be used to automate resource creation? + def __init__(self, name): + self.table_name = name + + def field(self, name): + return Field(name) + + def __getattr__(self, name): + return self.field(name) + + def __getitem__(self, name): + return self.field(name) + + +def datetime_as_utc(date_obj): + if date_obj.tzinfo is not None and date_obj.tzinfo.utcoffset(date_obj) is not None: + return date_obj.astimezone(pytz.UTC) + return date_obj diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/enums.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/enums.py new file mode 100644 index 00000000..a4614bb3 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/enums.py @@ -0,0 +1,112 @@ +from enum import Enum + + +class Order(Enum): + asc = "ASC" + desc = "DESC" + + +class Comparator(Enum): + pass + + +class Equality(Comparator): + eq = "=" + ne = "!=" + gt = ">" + gte = ">=" + lt = "<" + lte = "<=" + + +class Boolean(Comparator): + and_ = "^" + or_ = "^OR" + nq_ = "^NQ" + + +# Retrieved by inspecting the `ON` dropdown +class DateTimeOn(Enum): + today = "Today@javascript:gs.beginningOfToday()@javascript:gs.endOfToday()" + yesterday = ( + "Yesterday@javascript:gs.beginningOfYesterday()@javascript:gs.endOfYesterday()" + ) + tomorrow = ( + "Tomorrow@javascript:gs.beginningOfTomorrow()@javascript:gs.endOfTomorrow()" + ) + this_week = ( + "This week@javascript:gs.beginningOfThisWeek()@javascript:gs.endOfThisWeek()" + ) + last_week = ( + "Last week@javascript:gs.beginningOfLastWeek()@javascript:gs.endOfLastWeek()" + ) + next_week = ( + "Next week@javascript:gs.beginningOfNextWeek()@javascript:gs.endOfNextWeek()" + ) + this_month = ( + "This month@javascript:gs.beginningOfThisMonth()@javascript:gs.endOfThisMonth()" + ) + last_month = ( + "Last month@javascript:gs.beginningOfLastMonth()@javascript:gs.endOfLastMonth()" + ) + next_month = ( + "Next month@javascript:gs.beginningOfNextMonth()@javascript:gs.endOfNextMonth()" + ) + last_3_months = "Last 3 months@javascript:gs.beginningOfLast3Months()@javascript:gs.endOfLast3Months()" + last_6_months = "Last 6 months@javascript:gs.beginningOfLast6Months()@javascript:gs.endOfLast6Months()" + last_9_months = "Last 9 months@javascript:gs.beginningOfLast9Months()@javascript:gs.endOfLast9Months()" + last_12_months = "Last 12 months@javascript:gs.beginningOfLast12Months()@javascript:gs.endOfLast12Months()" + this_quarter = "This quarter@javascript:gs.beginningOfThisQuarter()@javascript:gs.endOfThisQuarter()" + last_quarter = "Last quarter@javascript:gs.beginningOfLastQuarter()@javascript:gs.endOfLastQuarter()" + last_2_quarters = "Last 2 quarters@javascript:gs.beginningOfLast2Quarters()@javascript:gs.endOfLast2Quarters()" + next_quarter = "Next quarter@javascript:gs.beginningOfNextQuarter()@javascript:gs.endOfNextQuarter()" + next_2_quarter = "Next 2 quarters@javascript:gs.beginningOfNext2Quarters()@javascript:gs.endOfNext2Quarters()" + this_year = ( + "This year@javascript:gs.beginningOfThisYear()@javascript:gs.endOfThisYear()" + ) + next_year = ( + "Next year@javascript:gs.beginningOfNextYear()@javascript:gs.endOfNextYear()" + ) + last_yesr = ( + "Last year@javascript:gs.beginningOfLastYear()@javascript:gs.endOfLastYear()" + ) + last_2_years = "Last 2 years@javascript:gs.beginningOfLast2Years()@javascript:gs.endOfLast2Years()" + last_7_days = "Last 7 days@javascript:gs.beginningOfLast7Days()@javascript:gs.endOfLast7Days()" + last_30_days = "Last 30 days@javascript:gs.beginningOfLast30Days()@javascript:gs.endOfLast30Days()" + last_60_days = "Last 60 days@javascript:gs.beginningOfLast60Days()@javascript:gs.endOfLast60Days()" + last_90_days = "Last 90 days@javascript:gs.beginningOfLast90Days()@javascript:gs.endOfLast90Days()" + last_120_days = "Last 120 days@javascript:gs.beginningOfLast120Days()@javascript:gs.endOfLast120Days()" + this_hour = "Current hour@javascript:gs.beginningOfCurrentHour()@javascript:gs.endOfCurrentHour()" + last_hour = ( + "Last hour@javascript:gs.beginningOfLastHour()@javascript:gs.endOfLastHour()" + ) + last_2_hours = "Last 2 hours@javascript:gs.beginningOfLast2Hours()@javascript:gs.endOfLast2Hours()" + this_minute = "Current minute@javascript:gs.beginningOfCurrentMinute()@javascript:gs.endOfCurrentMinute()" + last_minute = "Last minute@javascript:gs.beginningOfLastMinute()@javascript:gs.endOfLastMinute()" + last_15_minutes = "Last 15 minutes@javascript:gs.beginningOfLast15Minutes()@javascript:gs.endOfLast15Minutes()" + last_30_minutes = "Last 30 minutes@javascript:gs.beginningOfLast30Minutes()@javascript:gs.endOfLast30Minutes()" + last_45_minutes = "Last 45 minutes@javascript:gs.beginningOfLast45Minutes()@javascript:gs.endOfLast45Minutes()" + one_year_ago = "One year ago@javascript:gs.beginningOfOneYearAgo()@javascript:gs.endOfOneYearAgo()" + + +class RelativeEquality(Comparator): + ee = "EE" + eq = "EE" # on + gt = "GT" # after + gte = "GE" # on or after + lt = "LT" # before + lte = "LTE" # on or before + + +class RelativeTimeWindows(Enum): + minutes = "minute" + hours = "hour" + days = "dayofweek" + months = "month" + quarters = "quarter" + years = "year" + + +class RelativeDirection(Enum): + ago = "ago" + ahead = "ahead" diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/exceptions.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/exceptions.py new file mode 100644 index 00000000..9bff3b98 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/exceptions.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + + +class PysnowException(Exception): + pass + + +class InvalidUsage(PysnowException): + pass + + +class UnexpectedResponseFormat(PysnowException): + pass + + +class ResponseError(PysnowException): + message = "" + detail = "" + + def __init__(self, error): + if "message" in error: + self.message = error["message"] or self.message + if "detail" in error: + self.detail = error["detail"] or self.detail + + def __str__(self): + return "Error in response. Message: %s, Details: %s" % ( + self.message, + self.detail, + ) + + +class MissingResult(PysnowException): + pass + + +class NoResults(PysnowException): + pass + + +class EmptyContent(PysnowException): + pass + + +class MultipleResults(PysnowException): + pass + + +class MissingToken(PysnowException): + pass + + +class TokenCreateError(PysnowException): + def __init__(self, error, description, status_code): + self.error = error + self.description = description + self.snow_status_code = status_code + + +class QueryTypeError(PysnowException): + pass + + +class QueryMissingField(PysnowException): + pass + + +class QueryEmpty(PysnowException): + pass + + +class QueryExpressionError(PysnowException): + pass + + +class QueryMultipleExpressions(PysnowException): + pass diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_exceptions.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_exceptions.py new file mode 100644 index 00000000..1d1f559f --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_exceptions.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + + +class InvalidUsage(Exception): + pass + + +class QueryTypeError(TypeError): + pass + + +class QueryMissingField(Exception): + pass + + +class QueryEmpty(Exception): + pass + + +class QueryExpressionError(Exception): + pass + + +class QueryMultipleExpressions(Exception): + pass + + +class MissingResult(Exception): + pass + + +class MissingToken(Exception): + pass + + +class UnexpectedResponseFormat(Exception): + pass + + +class ReportUnavailable(Exception): + pass + + +class UnexpectedResponse(Exception): + """Provides detailed information about a server error response + + :param code_expected: Expected HTTP status code + :param code_actual: Actual HTTP status code + :param http_method: HTTP method used + :param error_summary: Summary of what went wrong + :param error_details: Details about the error + """ + + def __init__( + self, code_expected, code_actual, http_method, error_summary, error_details + ): + if code_expected == code_actual: + message = "Unexpected response on HTTP %s from server: %s" % ( + http_method, + error_summary, + ) + else: + message = "Unexpected HTTP %s response code. Expected %d, got %d" % ( + http_method, + code_expected, + code_actual, + ) + + super(UnexpectedResponse, self).__init__(message) + self.status_code = code_actual + self.error_summary = error_summary + self.error_details = error_details + + +class NoResults(Exception): + pass + + +class MultipleResults(Exception): + pass + + +class NoRequestExecuted(Exception): + pass diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_request.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_request.py new file mode 100644 index 00000000..326317f9 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/legacy_request.py @@ -0,0 +1,451 @@ +# -*- coding: utf-8 -*- + +import itertools +import json +import os +import six +import ntpath +import warnings + +from .query_builder import QueryBuilder + +from .legacy_exceptions import ( + NoRequestExecuted, + MultipleResults, + NoResults, + InvalidUsage, + UnexpectedResponse, + MissingResult, +) + + +class LegacyRequest(object): + base_path = "api/now" + + def __init__(self, method, table, **kwargs): + """Takes arguments used to perform a HTTP request + + :param method: HTTP request method + :param table: table to operate on + """ + self.method = method + self.table = table + self.url_link = None # Updated when a linked request is iterated on + self.base_url = kwargs.pop("base_url") + self.request_params = kwargs.pop("request_params") + self.raise_on_empty = kwargs.pop("raise_on_empty") + self.session = kwargs.pop("session") + self._last_response = None + + if method in ("GET", "DELETE"): + self.query = kwargs.pop("query") + + @property + def last_response(self): + """Return _last_response after making sure an inner `requests.request` has been performed + + :raise: + :NoRequestExecuted: If no request has been executed + :return: + - last response + """ + if self._last_response is None: + raise NoRequestExecuted("%s hasn't been executed" % self) + return self._last_response + + @last_response.setter + def last_response(self, response): + """ Sets last_response property + :param response: `requests.request` response + """ + self._last_response = response + + @property + def count(self): + """ Returns the number of records the query would yield""" + self.request_params.update({"sysparm_count": True}) + response = self.session.get( + self._get_stats_url(), + params=self._get_formatted_query( + fields=list(), limit=None, order_by=list(), offset=None + ), + ) + + content = self._get_content(response) + + return int(content["stats"]["count"]) + + @property + def status_code(self): + """Return last_response.status_code after making sure an inner `requests.request` has been performed + + :return: status_code of last_response + """ + return self.last_response.status_code + + def _all_inner(self, fields, limit, order_by, offset): + """Yields all records for the query and follows links if present on the response after validating + + :return: List of records with content + """ + response = self.session.get( + self._get_table_url(), + params=self._get_formatted_query(fields, limit, order_by, offset), + ) + + yield self._get_content(response) + while "next" in response.links: + self.url_link = response.links["next"]["url"] + response = self.session.get(self.url_link) + yield self._get_content(response) + + def get_all(self, fields=list(), limit=None, order_by=list(), offset=None): + """DEPRECATED - see get_multiple()""" + warnings.warn( + "get_all() is deprecated, please use get_multiple() instead", + DeprecationWarning, + ) + return self.get_multiple(fields, limit, order_by, offset) + + def get_multiple(self, fields=list(), limit=None, order_by=list(), offset=None): + """Wrapper method that takes whatever was returned by the _all_inner() generators and chains it in one result + + The response can be sorted by passing a list of fields to order_by. + + Example: + get_multiple(order_by=['category', '-created_on']) would sort the category field in ascending order, + with a secondary sort by created_on in descending order. + + :param fields: List of fields to return in the result + :param limit: Limits the number of records returned + :param order_by: Sort response based on certain fields + :param offset: A number of records to skip before returning records (for pagination) + :return: + - Iterable chain object + """ + return itertools.chain.from_iterable( + self._all_inner(fields, limit, order_by, offset) + ) + + def get_one(self, fields=list()): + """Convenience function for queries returning only one result. Validates response before returning. + + :param fields: List of fields to return in the result + :raise: + :MultipleResults: if more than one match is found + :return: + - Record content + """ + response = self.session.get( + self._get_table_url(), + params=self._get_formatted_query( + fields, limit=None, order_by=list(), offset=None + ), + ) + + content = self._get_content(response) + l = len(content) + if l > 1: + raise MultipleResults("Multiple results for get_one()") + + if len(content) == 0: + return {} + + return content[0] + + def insert(self, payload): + """Inserts a new record with the payload passed as an argument + + :param payload: The record to create (dict) + :return: + - Created record + """ + response = self.session.post(self._get_table_url(), data=json.dumps(payload)) + return self._get_content(response) + + def delete(self): + """Deletes the queried record and returns response content after response validation + + :raise: + :NoResults: if query returned no results + :NotImplementedError: if query returned more than one result (currently not supported) + :return: + - Delete response content (Generally always {'Success': True}) + """ + try: + result = self.get_one() + if "sys_id" not in result: + raise NoResults() + except MultipleResults: + raise MultipleResults("Deletion of multiple records is not supported") + except NoResults as e: + e.args = ("Cannot delete a non-existing record",) + raise + + response = self.session.delete(self._get_table_url(sys_id=result["sys_id"])) + return self._get_content(response) + + def update(self, payload): + """Updates the queried record with `payload` and returns the updated record after validating the response + + :param payload: Payload to update the record with + :raise: + :NoResults: if query returned no results + :MultipleResults: if query returned more than one result (currently not supported) + :return: + - The updated record + """ + try: + result = self.get_one() + if "sys_id" not in result: + raise NoResults() + except MultipleResults: + raise MultipleResults("Update of multiple records is not supported") + except NoResults as e: + e.args = ("Cannot update a non-existing record",) + raise + + if not isinstance(payload, dict): + raise InvalidUsage("Update payload must be of type dict") + + response = self.session.put( + self._get_table_url(sys_id=result["sys_id"]), data=json.dumps(payload) + ) + return self._get_content(response) + + def clone(self, reset_fields=list()): + """Clones the queried record + + :param reset_fields: Fields to reset + :raise: + :NoResults: if query returned no results + :MultipleResults: if query returned more than one result (currently not supported) + :UnexpectedResponse: informs the user about what likely went wrong + :return: + - The cloned record + """ + + if not isinstance(reset_fields, list): + raise InvalidUsage("reset_fields must be a `list` of fields") + + try: + response = self.get_one() + if "sys_id" not in response: + raise NoResults() + except MultipleResults: + raise MultipleResults("Cloning multiple records is not supported") + except NoResults as e: + e.args = ("Cannot clone a non-existing record",) + raise + + payload = {} + + # Iterate over fields in the result + for field in response: + # Ignore fields in reset_fields + if field in reset_fields: + continue + + item = response[field] + # Check if the item is of type dict and has a sys_id ref (value) + if isinstance(item, dict) and "value" in item: + payload[field] = item["value"] + else: + payload[field] = item + + try: + return self.insert(payload) + except UnexpectedResponse as e: + if e.status_code == 403: + # User likely attempted to clone a record without resetting a unique field + e.args = ( + "Unable to create clone. Make sure unique fields has been reset.", + ) + raise + + def attach(self, file): + """Attaches the queried record with `file` and returns the response after validating the response + + :param file: File to attach to the record + :raise: + :NoResults: if query returned no results + :MultipleResults: if query returned more than one result (currently not supported) + :return: + - The attachment record metadata + """ + try: + result = self.get_one() + if "sys_id" not in result: + raise NoResults() + except MultipleResults: + raise MultipleResults( + "Attaching a file to multiple records is not supported" + ) + except NoResults: + raise NoResults("Attempted to attach file to a non-existing record") + + if not os.path.isfile(file): + raise InvalidUsage( + "Attachment '%s' must be an existing regular file" % file + ) + + response = self.session.post( + self._get_attachment_url("upload"), + data={ + "table_name": self.table, + "table_sys_id": result["sys_id"], + "file_name": ntpath.basename(file), + }, + files={"file": open(file, "rb")}, + headers={"content-type": None}, # Temporarily override header + ) + return self._get_content(response) + + def _get_content(self, response): + """Checks for errors in the response. Returns response content, in bytes. + + :param response: response object + :raise: + :UnexpectedResponse: if the server responded with an unexpected response + :return: + - ServiceNow response content + """ + method = response.request.method + self.last_response = response + + server_error = {"summary": None, "details": None} + + try: + content_json = response.json() + if "error" in content_json: + e = content_json["error"] + if "message" in e: + server_error["summary"] = e["message"] + if "detail" in e: + server_error["details"] = e["detail"] + except ValueError: + content_json = {} + + if method == "DELETE": + # Make sure the delete operation returned the expected response + if response.status_code == 204: + return {"success": True} + else: + raise UnexpectedResponse( + 204, + response.status_code, + method, + server_error["summary"], + server_error["details"], + ) + # Make sure the POST operation returned the expected response + elif method == "POST" and response.status_code != 201: + raise UnexpectedResponse( + 201, + response.status_code, + method, + server_error["summary"], + server_error["details"], + ) + # It seems that Helsinki and later returns status 200 instead of 404 on empty result sets + if ( + "result" in content_json and len(content_json["result"]) == 0 + ) or response.status_code == 404: + if self.raise_on_empty is True: + raise NoResults("Query yielded no results") + elif "error" in content_json: + raise UnexpectedResponse( + 200, + response.status_code, + method, + server_error["summary"], + server_error["details"], + ) + + if "result" not in content_json: + raise MissingResult( + "The request was successful but the content didn't contain the expected 'result'" + ) + + return content_json["result"] + + def _get_table_url(self, **kwargs): + return self._get_url("table", item=self.table, **kwargs) + + def _get_attachment_url(self, action): + return self._get_url("attachment", item=action) + + def _get_stats_url(self): + return self._get_url("stats", item=self.table) + + def _get_url(self, resource, item, sys_id=None): + """Takes table and sys_id (if present), and returns a URL + + :param resource: API resource + :param item: API resource item + :param sys_id: Record sys_id + :return: + - url string + """ + + url_str = "%(base_url)s/%(base_path)s/%(resource)s/%(item)s" % ( + { + "base_url": self.base_url, + "base_path": self.base_path, + "resource": resource, + "item": item, + } + ) + + if sys_id: + return "%s/%s" % (url_str, sys_id) + + return url_str + + def _get_formatted_query(self, fields, limit, order_by, offset): + """ + Converts the query to a ServiceNow-interpretable format + :return: + - ServiceNow query + """ + + if not isinstance(order_by, list): + raise InvalidUsage("Argument order_by should be a `list` of fields") + + if not isinstance(fields, list): + raise InvalidUsage("Argument fields should be a `list` of fields") + + if isinstance(self.query, QueryBuilder): + sysparm_query = str(self.query) + elif isinstance(self.query, dict): # Dict-type query + sysparm_query = "^".join( + ["%s=%s" % (k, v) for k, v in six.iteritems(self.query)] + ) + elif isinstance(self.query, six.string_types): # String-type query + sysparm_query = self.query + else: + raise InvalidUsage( + "Query must be instance of %s, %s or %s" % (QueryBuilder, str, dict) + ) + + for field in order_by: + if field[0] == "-": + sysparm_query += "^ORDERBYDESC%s" % field[1:] + else: + sysparm_query += "^ORDERBY%s" % field + + params = {"sysparm_query": sysparm_query} + params.update(self.request_params) + + if limit is not None: + params.update( + {"sysparm_limit": limit, "sysparm_suppress_pagination_header": True} + ) + + if offset is not None: + params.update({"sysparm_offset": offset}) + + if len(fields) > 0: + params.update({"sysparm_fields": ",".join(fields)}) + + return params diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/oauth_client.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/oauth_client.py new file mode 100644 index 00000000..2d231d5b --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/oauth_client.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- + +import warnings +import logging + +from oauthlib.oauth2 import LegacyApplicationClient +from oauthlib.oauth2.rfc6749.errors import OAuth2Error +from requests_oauthlib import OAuth2Session + +from .client import Client +from .exceptions import InvalidUsage, MissingToken, TokenCreateError + +logger = logging.getLogger("pysnow") + + +class OAuthClient(Client): + """Pysnow `Client` with extras for oauth session and token handling. + + :param client_id: client_id from ServiceNow + :param client_secret: client_secret from ServiceNow + :param token_updater: function called when a token has been refreshed + :param kwargs: kwargs passed along to :class:`pysnow.Client` + """ + + token = None + + def __init__( + self, client_id=None, client_secret=None, token_updater=None, **kwargs + ): + + if not (client_secret and client_id): + raise InvalidUsage("You must supply a client_id and client_secret") + + if kwargs.get("session") or kwargs.get("user"): + warnings.warn( + "pysnow.OAuthClient manages sessions internally, " + "provided user / password credentials or sessions will be ignored." + ) + + # Forcibly set session, user and password + kwargs["user"] = None + kwargs["password"] = None + + super(OAuthClient, self).__init__(**kwargs) + + self.token_updater = token_updater + self.client_id = client_id + self.client_secret = client_secret + + self.token_url = "%s/oauth_token.do" % self.base_url + + def _get_oauth_session(self): + """Creates a new OAuth session + + :return: + - OAuth2Session object + """ + + return self._get_session( + OAuth2Session( + client_id=self.client_id, + token=self.token, + token_updater=self.token_updater, + auto_refresh_url=self.token_url, + auto_refresh_kwargs={ + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + ) + ) + + def set_token(self, token): + """Validate and set token + + :param token: the token (dict) to set + """ + + if not token: + self.token = None + return + + expected_keys = [ + "token_type", + "refresh_token", + "access_token", + "scope", + "expires_in", + "expires_at", + ] + if not isinstance(token, dict) or not set(token) >= set(expected_keys): + raise InvalidUsage( + "Expected a token dictionary containing the following keys: {0}".format( + expected_keys + ) + ) + + # Set sanitized token + self.token = dict((k, v) for k, v in token.items() if k in expected_keys) + + def _legacy_request(self, *args, **kwargs): + """Makes sure token has been set, then calls parent to create a new :class:`pysnow.LegacyRequest` object + + :param args: args to pass along to _legacy_request() + :param kwargs: kwargs to pass along to _legacy_request() + :return: + - :class:`pysnow.LegacyRequest` object + :raises: + - MissingToken: If token hasn't been set + """ + + if isinstance(self.token, dict): + self.session = self._get_oauth_session() + return super(OAuthClient, self)._legacy_request(*args, **kwargs) + + raise MissingToken( + "You must set_token() before creating a legacy request with OAuthClient" + ) + + def resource(self, api_path=None, base_path="/api/now", chunk_size=None): + """Overrides :meth:`resource` provided by :class:`pysnow.Client` with extras for OAuth + + :param api_path: Path to the API to operate on + :param base_path: (optional) Base path override + :param chunk_size: Response stream parser chunk size (in bytes) + :return: + - :class:`Resource` object + :raises: + - InvalidUsage: If a path fails validation + """ + + if isinstance(self.token, dict): + self.session = self._get_oauth_session() + return super(OAuthClient, self).resource(api_path, base_path, chunk_size) + + raise MissingToken( + "You must set_token() before creating a resource with OAuthClient" + ) + + def generate_token(self, user, password): + """Takes user and password credentials and generates a new token + + :param user: user + :param password: password + :return: + - dictionary containing token data + :raises: + - TokenCreateError: If there was an error generating the new token + """ + + logger.debug("(TOKEN_CREATE) :: User: %s" % user) + + session = OAuth2Session( + client=LegacyApplicationClient(client_id=self.client_id) + ) + + try: + return dict( + session.fetch_token( + token_url=self.token_url, + username=user, + password=password, + client_id=self.client_id, + client_secret=self.client_secret, + ) + ) + except OAuth2Error as exception: + raise TokenCreateError( + "Error creating user token", + exception.description, + exception.status_code, + ) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/params_builder.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/params_builder.py new file mode 100644 index 00000000..a84c68fe --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/params_builder.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- + +import six + +from .criterion import Criterion +from .query_builder import QueryBuilder + +from .exceptions import InvalidUsage + + +class ParamsBuilder(object): + """Provides an interface for setting / getting common ServiceNow sysparms.""" + + def __init__(self): + self._custom_params = {} + + self._sysparms = { + "sysparm_query": "", + "sysparm_limit": 10000, + "sysparm_offset": None, + "sysparm_display_value": False, + "sysparm_suppress_pagination_header": False, + "sysparm_exclude_reference_link": False, + "sysparm_view": "", + "sysparm_fields": [], + } + + @staticmethod + def stringify_query(query): + """Stringifies the query (dict or QueryBuilder) into a ServiceNow-compatible format + + :return: + - ServiceNow-compatible string-type query + """ + + if isinstance(query, QueryBuilder) or isinstance(query, Criterion): + # Get string-representation of the passed :class:`pysnow.QueryBuilder` object + return str(query) + elif isinstance(query, dict): + # Dict-type query + return "^".join(["%s=%s" % (k, v) for k, v in six.iteritems(query)]) + elif isinstance(query, six.string_types): + # Regular string-type query + return query + else: + raise InvalidUsage( + "Query must be of type string, dict, QueryBuilder, or Criterion" + ) + + def add_custom(self, params): + """Adds new custom parameter after making sure it's of type dict. + + :param params: Dictionary containing one or more parameters + """ + + if isinstance(params, dict) is False: + raise InvalidUsage("custom parameters must be of type `dict`") + + self._custom_params.update(params) + + @property + def custom_params(self): + """Returns a dictionary of added custom parameters""" + return self._custom_params + + @property + def display_value(self): + """Maps to `sysparm_display_value`""" + return self._sysparms["sysparm_display_value"] + + @display_value.setter + def display_value(self, value): + """Sets `sysparm_display_value` + + :param value: Bool or 'all' + """ + + if not (isinstance(value, bool) or value == "all"): + raise InvalidUsage("Display value can be of type bool or value 'all'") + + self._sysparms["sysparm_display_value"] = value + + @property + def query(self): + """Maps to `sysparm_query`""" + return self._sysparms["sysparm_query"] + + @query.setter + def query(self, query): + """Validates, stringifies and sets `sysparm_query` + + :param query: String, dict or QueryBuilder + """ + + self._sysparms["sysparm_query"] = self.stringify_query(query) + + @property + def limit(self): + """Maps to `sysparm_limit`""" + return self._sysparms["sysparm_limit"] + + @limit.setter + def limit(self, limit): + """Sets `sysparm_limit` + + :param limit: Size limit (int) + """ + + if not isinstance(limit, int) or isinstance(limit, bool): + raise InvalidUsage("limit size must be of type integer") + + self._sysparms["sysparm_limit"] = limit + + @property + def offset(self): + """Maps to `sysparm_offset`""" + return self._sysparms["sysparm_offset"] + + @offset.setter + def offset(self, offset): + """Sets `sysparm_offset`, usually used to accomplish pagination + + :param offset: Number of records to skip before fetching records + :raise: + :InvalidUsage: if offset is of an unexpected type + """ + + if not isinstance(offset, int) or isinstance(offset, bool): + raise InvalidUsage("Offset must be an integer") + + self._sysparms["sysparm_offset"] = offset + + @property + def fields(self): + """Maps to `sysparm_fields`""" + return self._sysparms["sysparm_fields"] + + @fields.setter + def fields(self, fields): + """Sets `sysparm_fields` after joining the given list of `fields` + + :param fields: List of fields to include in the response + :raise: + :InvalidUsage: if fields is of an unexpected type + """ + + if not isinstance(fields, list): + raise InvalidUsage("fields must be of type `list`") + + self._sysparms["sysparm_fields"] = ",".join(fields) + + @property + def exclude_reference_link(self): + """Maps to `sysparm_exclude_reference_link`""" + return self._sysparms["sysparm_exclude_reference_link"] + + @exclude_reference_link.setter + def exclude_reference_link(self, exclude): + """Sets `sysparm_exclude_reference_link` to a bool value + + :param exclude: bool + """ + if not isinstance(exclude, bool): + raise InvalidUsage("exclude_reference_link must be of type bool") + + self._sysparms["sysparm_exclude_reference_link"] = exclude + + @property + def suppress_pagination_header(self): + """Maps to `sysparm_suppress_pagination_header`""" + return self._sysparms["sysparm_suppress_pagination_header"] + + @suppress_pagination_header.setter + def suppress_pagination_header(self, suppress): + """Enables or disables pagination header by setting `sysparm_suppress_pagination_header` + + :param suppress: bool + """ + if not isinstance(suppress, bool): + raise InvalidUsage("suppress_pagination_header must be of type bool") + + self._sysparms["sysparm_suppress_pagination_header"] = suppress + + def as_dict(self): + """Constructs query params compatible with :class:`requests.Request` + + :return: + - Dictionary containing query parameters + """ + + sysparms = self._sysparms + sysparms.update(self._custom_params) + + return sysparms diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/query_builder.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/query_builder.py new file mode 100644 index 00000000..f285d4f2 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/query_builder.py @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- + +import inspect +import six +import pytz + +from .exceptions import ( + QueryEmpty, + QueryExpressionError, + QueryMissingField, + QueryMultipleExpressions, + QueryTypeError, +) + + +class QueryBuilder(object): + """Query builder - for constructing advanced ServiceNow queries""" + + def __init__(self): + self._query = [] + self.current_field = None + self.c_oper = None + self.l_oper = None + + def field(self, field): + """Sets the field to operate on + + :param field: field (str) to operate on + """ + + self.current_field = field + return self + + def order_descending(self): + """Sets ordering of field descending""" + + self._query.append("ORDERBYDESC{0}".format(self.current_field)) + self.c_oper = inspect.currentframe().f_back.f_code.co_name + return self + + def order_ascending(self): + """Sets ordering of field ascending""" + + self._query.append("ORDERBY{0}".format(self.current_field)) + self.c_oper = inspect.currentframe().f_back.f_code.co_name + return self + + def starts_with(self, starts_with): + """Adds new `STARTSWITH` condition + + :param starts_with: Match field starting with the provided value + """ + + return self._add_condition("STARTSWITH", starts_with, types=[str]) + + def ends_with(self, ends_with): + """Adds new `ENDSWITH` condition + + :param ends_with: Match field ending with the provided value + """ + + return self._add_condition("ENDSWITH", ends_with, types=[str]) + + def contains(self, contains): + """Adds new `LIKE` condition + + :param contains: Match field containing the provided value + """ + + return self._add_condition("LIKE", contains, types=[str]) + + def not_contains(self, not_contains): + """Adds new `NOT LIKE` condition + + :param not_contains: Match field not containing the provided value + """ + + return self._add_condition("NOT LIKE", not_contains, types=[str]) + + def is_empty(self): + """Adds new `ISEMPTY` condition""" + + return self._add_condition("ISEMPTY", "", types=[str, int]) + + def is_not_empty(self): + """Adds new `ISNOTEMPTY` condition""" + + return self._add_condition("ISNOTEMPTY", "", types=[str, int]) + + def equals(self, data): + """Adds new `IN` or `=` condition depending on if a list or string was provided + + :param data: string or list of values + :raise: + - QueryTypeError: if `data` is of an unexpected type + """ + + if isinstance(data, six.string_types): + return self._add_condition("=", data, types=[int, str]) + elif isinstance(data, list): + return self._add_condition("IN", ",".join(map(str, data)), types=[str]) + + raise QueryTypeError( + "Expected value of type `str` or `list`, not %s" % type(data) + ) + + def not_equals(self, data): + """Adds new `NOT IN` or `!=` condition depending on if a list or string was provided + + :param data: string or list of values + :raise: + - QueryTypeError: if `data` is of an unexpected type + """ + + if isinstance(data, six.string_types): + return self._add_condition("!=", data, types=[int, str]) + elif isinstance(data, list): + return self._add_condition("NOT IN", ",".join(data), types=[str]) + + raise QueryTypeError( + "Expected value of type `str` or `list`, not %s" % type(data) + ) + + def greater_than(self, greater_than): + """Adds new `>` condition + + :param greater_than: str or datetime compatible object (naive UTC datetime or tz-aware datetime) + :raise: + - QueryTypeError: if `greater_than` is of an unexpected type + """ + + if hasattr(greater_than, "strftime"): + greater_than = datetime_as_utc(greater_than).strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(greater_than, six.string_types): + raise QueryTypeError( + "Expected value of type `int` or instance of `datetime`, not %s" + % type(greater_than) + ) + + return self._add_condition(">", greater_than, types=[int, str]) + + def greater_than_or_equal(self, greater_than): + """Adds new `>=` condition + + :param greater_than: str or datetime compatible object (naive UTC datetime or tz-aware datetime) + :raise: + - QueryTypeError: if `greater_than` is of an unexpected type + """ + + if hasattr(greater_than, "strftime"): + greater_than = datetime_as_utc(greater_than).strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(greater_than, six.string_types): + raise QueryTypeError( + "Expected value of type `int` or instance of `datetime`, not %s" + % type(greater_than) + ) + + return self._add_condition(">=", greater_than, types=[int, str]) + + def less_than(self, less_than): + """Adds new `<` condition + + :param less_than: str or datetime compatible object (naive UTC datetime or tz-aware datetime) + :raise: + - QueryTypeError: if `less_than` is of an unexpected type + """ + + if hasattr(less_than, "strftime"): + less_than = datetime_as_utc(less_than).strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(less_than, six.string_types): + raise QueryTypeError( + "Expected value of type `int` or instance of `datetime`, not %s" + % type(less_than) + ) + + return self._add_condition("<", less_than, types=[int, str]) + + def less_than_or_equal(self, less_than): + """Adds new `<=` condition + + :param less_than: str or datetime compatible object (naive UTC datetime or tz-aware datetime) + :raise: + - QueryTypeError: if `less_than` is of an unexpected type + """ + + if hasattr(less_than, "strftime"): + less_than = datetime_as_utc(less_than).strftime("%Y-%m-%d %H:%M:%S") + elif isinstance(less_than, six.string_types): + raise QueryTypeError( + "Expected value of type `int` or instance of `datetime`, not %s" + % type(less_than) + ) + + return self._add_condition("<=", less_than, types=[int, str]) + + def between(self, start, end): + """Adds new `BETWEEN` condition + + :param start: int or datetime compatible object (in SNOW user's timezone) + :param end: int or datetime compatible object (in SNOW user's timezone) + :raise: + - QueryTypeError: if start or end arguments is of an invalid type + """ + + if hasattr(start, "strftime") and hasattr(end, "strftime"): + dt_between = ( + 'javascript:gs.dateGenerate("%(start)s")' + "@" + 'javascript:gs.dateGenerate("%(end)s")' + ) % { + "start": start.strftime("%Y-%m-%d %H:%M:%S"), + "end": end.strftime("%Y-%m-%d %H:%M:%S"), + } + elif isinstance(start, int) and isinstance(end, int): + dt_between = "%d@%d" % (start, end) + else: + raise QueryTypeError( + "Expected `start` and `end` of type `int` " + "or instance of `datetime`, not %s and %s" % (type(start), type(end)) + ) + + return self._add_condition("BETWEEN", dt_between, types=[str]) + + def AND(self): + """Adds an and-operator""" + return self._add_logical_operator("^") + + def OR(self): + """Adds an or-operator""" + return self._add_logical_operator("^OR") + + def NQ(self): + """Adds a NQ-operator (new query)""" + return self._add_logical_operator("^NQ") + + def _add_condition(self, operator, operand, types): + """Appends condition to self._query after performing validation + + :param operator: operator (str) + :param operand: operand + :param types: allowed types + :raise: + - QueryMissingField: if a field hasn't been set + - QueryMultipleExpressions: if a condition already has been set + - QueryTypeError: if the value is of an unexpected type + """ + + if not self.current_field: + raise QueryMissingField("Conditions requires a field()") + + elif not type(operand) in types: + caller = inspect.currentframe().f_back.f_code.co_name + raise QueryTypeError( + "Invalid type passed to %s() , expected: %s" % (caller, types) + ) + + elif self.c_oper: + raise QueryMultipleExpressions("Expected logical operator after expression") + + self.c_oper = inspect.currentframe().f_back.f_code.co_name + + self._query.append( + "%(current_field)s%(operator)s%(operand)s" + % { + "current_field": self.current_field, + "operator": operator, + "operand": operand, + } + ) + + return self + + def _add_logical_operator(self, operator): + """Adds a logical operator in query + + :param operator: logical operator (str) + :raise: + - QueryExpressionError: if a expression hasn't been set + """ + + if not self.c_oper: + raise QueryExpressionError( + "Logical operators must be preceded by an expression" + ) + + self.current_field = None + self.c_oper = None + + self.l_oper = inspect.currentframe().f_back.f_code.co_name + self._query.append(operator) + return self + + def __str__(self): + """String representation of the query object + + :raise: + - QueryEmpty: if there's no conditions defined + - QueryMissingField: if field() hasn't been set + - QueryExpressionError: if a expression hasn't been set + + :return: + - String-type query + """ + + if len(self._query) == 0: + raise QueryEmpty("At least one condition is required") + elif self.current_field is None: + raise QueryMissingField("Logical operator expects a field()") + elif self.c_oper is None: + raise QueryExpressionError("field() expects an expression") + + return str().join(self._query) + + +def datetime_as_utc(date_obj): + if date_obj.tzinfo is not None and date_obj.tzinfo.utcoffset(date_obj) is not None: + return date_obj.astimezone(pytz.UTC) + return date_obj diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/request.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/request.py new file mode 100644 index 00000000..1054c9b1 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/request.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- + +import logging +import json +import six + +from .response import Response +from .exceptions import InvalidUsage + +logger = logging.getLogger("pysnow") + + +class SnowRequest(object): + """Creates a new :class:`SnowRequest` object. + + :param parameters: :class:`params_builder.ParamsBuilder` object + :param session: :class:`request.Session` object + :param url_builder: :class:`url_builder.URLBuilder` object + """ + + def __init__( + self, + parameters=None, + session=None, + url_builder=None, + chunk_size=None, + resource=None, + timeout=60, + ): + self._parameters = parameters + self._url_builder = url_builder + self._session = session + self._chunk_size = chunk_size + self._resource = resource + self._timeout = timeout + + self._url = url_builder.get_url() + + def _get_response(self, method, **kwargs): + """Response wrapper - creates a :class:`requests.Response` object and passes along to :class:`pysnow.Response` + for validation and parsing. + + :param args: args to pass along to _send() + :param kwargs: kwargs to pass along to _send() + :return: + - :class:`pysnow.Response` object + """ + + params = self._parameters.as_dict() + use_stream = kwargs.pop("stream", False) + + logger.debug( + "(REQUEST_SEND) Method: %s, Resource: %s" % (method, self._resource) + ) + + response = self._session.request( + method, self._url, stream=use_stream, params=params, timeout=self._timeout, **kwargs + ) + response.raw.decode_content = True + + logger.debug( + "(RESPONSE_RECEIVE) Code: %d, Resource: %s" + % (response.status_code, self._resource) + ) + + return Response( + response=response, + resource=self._resource, + chunk_size=self._chunk_size, + stream=use_stream, + ) + + def _get_custom_endpoint(self, value): + if isinstance(value, dict) and "value" in value: + value = value["value"] + elif not isinstance(value, six.string_types): + raise InvalidUsage( + "Argument 'path_append' must be a string in the following format: " + "/path-to-append[/.../...]" + ) + + segment = value if value.startswith("/") else "/{0}".format(value) + return self._url_builder.get_appended_custom(segment) + + def get(self, *args, **kwargs): + """Fetches one or more records + + :return: + - :class:`pysnow.Response` object + """ + + query = kwargs.pop("query", {}) if len(args) == 0 else args[0] + + if isinstance(query, dict): + for key, value in query.items(): + if isinstance(value, dict): + query[key] = value["value"] + + self._parameters.query = query + self._parameters.limit = kwargs.pop("limit", 10000) + self._parameters.offset = kwargs.pop("offset", 0) + self._parameters.fields = kwargs.pop("fields", []) + if "display_value" in kwargs: + self._parameters.display_value = kwargs.pop("display_value") + if "exclude_reference_link" in kwargs: + self._parameters.exclude_reference_link = kwargs.pop( + "exclude_reference_link" + ) + if "suppress_pagination_header" in kwargs: + self._parameters.suppress_pagination_header = kwargs.pop( + "suppress_pagination_header" + ) + + return self._get_response("GET", stream=kwargs.pop("stream", False)) + + def create(self, payload): + """Creates a new record + + :param payload: Dictionary payload + :return: + - Dictionary of the inserted record + """ + + return self._get_response("POST", data=json.dumps(payload)) + + def update(self, query, payload): + """Updates a record + + :param query: Dictionary, string or :class:`QueryBuilder` object + :param payload: Dictionary payload + :return: + - Dictionary of the updated record + """ + + if not isinstance(payload, dict): + raise InvalidUsage("Update payload must be of type dict") + + record = self.get(query=query).one() + + self._url = self._get_custom_endpoint(record["sys_id"]) + return self._get_response("PUT", data=json.dumps(payload)) + + def delete(self, query): + """Deletes a record + + :param query: Dictionary, string or :class:`QueryBuilder` object + :return: + - Dictionary containing status of the delete operation + """ + + record = self.get(query=query).one() + self._url = self._get_custom_endpoint(record["sys_id"]) + + return self._get_response("DELETE").one() + + def custom(self, method, path_append=None, **kwargs): + """Creates a custom request + + :param method: HTTP method + :param path_append: (optional) append path to resource.api_path + :param headers: (optional) Dictionary of headers to add or override + :param kwargs: kwargs to pass along to :class:`requests.Request` + :return: + - :class:`pysnow.Response` object + """ + if path_append is not None: + self._url = self._get_custom_endpoint(path_append) + + return self._get_response(method, **kwargs) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/resource.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/resource.py new file mode 100644 index 00000000..1c884343 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/resource.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- + +import logging + +from copy import copy, deepcopy + +from .request import SnowRequest +from .attachment import Attachment +from .url_builder import URLBuilder +from .exceptions import InvalidUsage + +logger = logging.getLogger("pysnow") + + +class Resource(object): + r"""Creates a new :class:`Resource` object + + Resources provides a natural way of interfacing with ServiceNow APIs. + + :param base_path: Base path + :param api_path: API path + :param chunk_size: Response stream parser chunk size (in bytes) + :param \*\*kwargs: Arguments to pass along to :class:`Request` + """ + + def __init__( + self, base_url=None, base_path=None, api_path=None, parameters=None, **kwargs + ): + + self._base_url = base_url + self._base_path = base_path + self._api_path = api_path + self._url_builder = URLBuilder(base_url, base_path, api_path) + + self.kwargs = kwargs + self.parameters = deepcopy(parameters) + + logger.debug( + "(RESOURCE_ADD) Object: %s, chunk_size: %d" + % (self, kwargs.get("chunk_size")) + ) + + def __repr__(self): + return "<%s [%s] at %s>" % (self.__class__.__name__, self.path, hex(id(self))) + + @property + def path(self): + """Get current path relative to base URL + + :return: resource path + """ + + return "%s" % self._base_path + self._api_path + + @property + def attachments(self): + """Provides an `Attachment` API for this resource. + Enables easy listing, deleting and creating new attachments. + + :return: Attachment object + """ + + resource = copy(self) + resource._url_builder = URLBuilder( + self._base_url, self._base_path, "/attachment" + ) + + path = self._api_path.strip("/").split("/") + + if path[0] != "table": + raise InvalidUsage("The attachment API can only be used with the table API") + + return Attachment(resource, path[1]) + + @property + def _request(self): + """Request wrapper + + :return: SnowRequest object + """ + + parameters = copy(self.parameters) + + return SnowRequest( + url_builder=self._url_builder, + parameters=parameters, + resource=self, + **self.kwargs + ) + + def get_record_link(self, sys_id): + """Provides full URL to the provided sys_id + + :param sys_id: sys_id to generate URL for + :return: full sys_id URL + """ + + return "%s/%s" % (self._url_builder.get_url(), sys_id) + + def get(self, *args, **kwargs): + """Queries the API resource + + :param args: + - :param query: Dictionary, string or :class:`QueryBuilder` object + defaults to empty dict (all) + + :param kwargs: + - :param limit: Limits the number of records returned + - :param fields: List of fields to include in the response + created_on in descending order. + - :param offset: Number of records to skip before returning records + - :param stream: Whether or not to use streaming / generator response interface + + :return: + - :class:`Response` object + """ + + return self._request.get(*args, **kwargs) + + def create(self, payload): + """Creates a new record in the API resource + + :param payload: Dictionary containing key-value fields of the new record + :return: + - Dictionary of the inserted record + """ + + return self._request.create(payload) + + def update(self, query, payload): + """Updates a record in the API resource + + :param query: Dictionary, string or :class:`QueryBuilder` object + :param payload: Dictionary containing key-value fields of the record to be updated + :return: + - Dictionary of the updated record + """ + + return self._request.update(query, payload) + + def delete(self, query): + """Deletes matching record + + :param query: Dictionary, string or :class:`QueryBuilder` object + :return: + - Dictionary containing information about deletion result + """ + + return self._request.delete(query) + + def request(self, method, path_append=None, headers=None, **kwargs): + """Create a custom request + + :param method: HTTP method to use + :param path_append: (optional) relative to :attr:`api_path` + :param headers: (optional) Dictionary of headers to add or override + :param kwargs: kwargs to pass along to :class:`requests.Request` + :return: + - :class:`Response` object + """ + + return self._request.custom( + method, path_append=path_append, headers=headers, **kwargs + ) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/response.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/response.py new file mode 100644 index 00000000..d7e951a0 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/response.py @@ -0,0 +1,280 @@ +# -*- coding: utf-8 -*- + +import ijson + +from ijson.common import ObjectBuilder +from itertools import chain +from .exceptions import ( + ResponseError, + NoResults, + InvalidUsage, + MultipleResults, + EmptyContent, + MissingResult, +) + + +class Response(object): + """Takes a :class:`requests.Response` object and performs deserialization and validation. + + :param response: :class:`requests.Response` object + :param resource: parent :class:`resource.Resource` object + :param chunk_size: Read and return up to this size (in bytes) in the stream parser + """ + + def __init__(self, response, resource, chunk_size=8192, stream=False): + self._response = response + self._chunk_size = chunk_size + self._count = 0 + self._resource = resource + self._stream = stream + + @property + def headers(self): + return self._response.headers + + @property + def count(self): + return self._count + + @count.setter + def count(self, count): + if not isinstance(count, int) or isinstance(count, bool): + raise TypeError("Count must be an integer") + + self._count = count + + def __getitem__(self, key): + return self.one().get(key) + + def __repr__(self): + return "<%s [%d - %s]>" % ( + self.__class__.__name__, + self._response.status_code, + self._response.request.method, + ) + + def _parse_response(self): + """Looks for `result.item` (array), `result` (object) and `error` (object) keys and parses + the raw response content (stream of bytes) + + :raise: + - ResponseError: If there's an error in the response + - MissingResult: If no result nor error was found + """ + + response = self._get_response() + + has_result_single = False + has_result_many = False + has_error = False + + builder = ObjectBuilder() + + for prefix, event, value in ijson.parse( + response.raw, buf_size=self._chunk_size + ): + if (prefix, event) == ("error", "start_map"): + # Matched ServiceNow `error` object at the root + has_error = True + elif prefix == "result" and event in ["start_map", "start_array"]: + # Matched ServiceNow `result` + if event == "start_map": # Matched object + has_result_single = True + elif event == "start_array": # Matched array + has_result_many = True + + if has_result_many: + # Build the result + if (prefix, event) == ("result.item", "end_map"): + # Reached end of object. Set count and yield + builder.event(event, value) + self.count += 1 + yield getattr(builder, "value") + elif prefix.startswith("result.item"): + # Build the result object + builder.event(event, value) + elif has_result_single: + if (prefix, event) == ("result", "end_map"): + # Reached end of the result object. Set count and yield. + builder.event(event, value) + self.count += 1 + yield getattr(builder, "value") + elif prefix.startswith("result"): + # Build the error object + builder.event(event, value) + elif has_error: + if (prefix, event) == ("error", "end_map"): + # Reached end of the error object - raise ResponseError exception + raise ResponseError(getattr(builder, "value")) + elif prefix.startswith("error"): + # Build the error object + builder.event(event, value) + + if (has_result_single or has_result_many) and self.count == 0: # Results empty + return + + if not ( + has_result_single or has_result_many or has_error + ): # None of the expected keys were found + raise MissingResult( + "The expected `result` key was missing in the response. Cannot continue" + ) + + def _get_response(self): + response = self._response + + # Raise an HTTPError if we hit a non-200 status code + response.raise_for_status() + + if response.request.method == "GET" and response.status_code == 202: + # GET request with a "202: no content" response: Raise NoContent Exception. + raise EmptyContent( + "Unexpected empty content in response for GET request: {}".format( + response.request.url + ) + ) + + return response + + def _get_streamed_response(self): + """Parses byte stream (memory efficient) + + :return: Parsed JSON + """ + + yield self._parse_response() + + def _get_buffered_response(self): + """Returns a buffered response + + :return: Buffered response + """ + + response = self._get_response() + + if response.request.method == "DELETE" and response.status_code == 204: + return [{"status": "record deleted"}], 1 + + result = self._response.json().get("result", None) + + if result is None: + raise MissingResult( + "The expected `result` key was missing in the response. Cannot continue" + ) + + length = 0 + + if isinstance(result, list): + length = len(result) + elif isinstance(result, dict): + result = [result] + length = 1 + + return result, length + + def all(self): + """Returns a chained generator response containing all matching records + + :return: + - Iterable response + """ + + if self._stream: + return chain.from_iterable(self._get_streamed_response()) + + return self._get_buffered_response()[0] + + def first(self): + """Return the first record or raise an exception if the result doesn't contain any data + + :return: + - Dictionary containing the first item in the response content + + :raise: + - NoResults: If no results were found + """ + + if not self._stream: + raise InvalidUsage("first() is only available when stream=True") + + try: + content = next(self.all()) + except StopIteration: + raise NoResults("No records found") + + return content + + def first_or_none(self): + """Return the first record or None + + :return: + - Dictionary containing the first item or None + """ + + try: + return self.first() + except NoResults: + return None + + def one(self): + """Return exactly one record or raise an exception. + + :return: + - Dictionary containing the only item in the response content + + :raise: + - MultipleResults: If more than one records are present in the content + - NoResults: If the result is empty + """ + + result, count = self._get_buffered_response() + + if count == 0: + raise NoResults("No records found") + elif count > 1: + raise MultipleResults("Expected single-record result, got multiple") + + return result[0] + + def one_or_none(self): + """Return at most one record or raise an exception. + + :return: + - Dictionary containing the matching record or None + + :raise: + - MultipleResults: If more than one records are present in the content + """ + + try: + return self.one() + except NoResults: + return None + + def update(self, payload): + """Convenience method for updating a fetched record + + :param payload: update payload + :return: update response object + """ + + return self._resource.update({"sys_id": self["sys_id"]}, payload) + + def delete(self): + """Convenience method for deleting a fetched record + + :return: delete response object + """ + + return self._resource.delete({"sys_id": self["sys_id"]}) + + def upload(self, *args, **kwargs): + """Convenience method for attaching files to a fetched record + + :param args: args to pass along to `Attachment.upload` + :param kwargs: kwargs to pass along to `Attachment.upload` + :return: upload response object + """ + + return self._resource.attachments.upload(self["sys_id"], *args, **kwargs) diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/url_builder.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/url_builder.py new file mode 100644 index 00000000..dd35d311 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/url_builder.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +import re +import six +from .exceptions import InvalidUsage + + +class URLBuilder(object): + def __init__(self, base_url, base_path, api_path): + self.base_url = base_url + self.base_path = base_path + self.api_path = api_path + self.full_path = base_path + api_path + + self._resource_url = "%(base_url)s%(full_path)s" % ( + {"base_url": base_url, "full_path": self.full_path} + ) + + @staticmethod + def validate_path(path): + """Validates the provided path + + :param path: path to validate (string) + :raise: + :InvalidUsage: If validation fails. + """ + + if not isinstance(path, six.string_types) or not re.match( + "^/(?:[._a-zA-Z0-9-]/?)+[^/]$", path + ): + raise InvalidUsage( + "Path validation failed - Expected: '/[/component], got: %s" + % path + ) + + return True + + @staticmethod + def get_base_url(use_ssl, instance=None, host=None): + """Formats the base URL either `host` or `instance` + + :return: Base URL string + """ + + if instance is not None: + host = ("%s.service-now.com" % instance).rstrip("/") + + if use_ssl is True: + return "https://%s" % host + + return "http://%s" % host + + def get_appended_custom(self, path_component): + """Validates the provided path_component, then returns it appended to :prop:`_resource_url` + + :param path_component: Path component string + :return: Full URL to the custom resource + """ + self.validate_path(path_component) + + return self._resource_url + path_component + + def get_url(self): + """Returns :prop:`_resource_url` + + :return: :prop:`_resource_url` + """ + return self._resource_url diff --git a/nautobot_ssot/integrations/servicenow/urls.py b/nautobot_ssot/integrations/servicenow/urls.py new file mode 100644 index 00000000..b4451cb3 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/urls.py @@ -0,0 +1,10 @@ +"""URL patterns for nautobot-ssot-servicenow.""" +from django.urls import path + +from . import views + +app_name = "nautobot_ssot_servicenow" + +urlpatterns = [ + path("config/", views.SSOTServiceNowConfigView.as_view(), name="config"), +] diff --git a/nautobot_ssot/integrations/servicenow/utils.py b/nautobot_ssot/integrations/servicenow/utils.py new file mode 100644 index 00000000..cd1c489e --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/utils.py @@ -0,0 +1,41 @@ +"""Utility/helper functions for nautobot-ssot-servicenow.""" +import logging + +from django.conf import settings + +from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices + +from .models import SSOTServiceNowConfig + + +logger = logging.getLogger(__name__) + + +def get_servicenow_parameters(): + """Get a dictionary containing the instance, username, and password for connecting to ServiceNow.""" + db_config = SSOTServiceNowConfig.load() + settings_config = settings.PLUGINS_CONFIG.get("nautobot_ssot_servicenow", {}) + result = { + "instance": settings_config.get("instance", db_config.servicenow_instance), + "username": settings_config.get("username", ""), + "password": settings_config.get("password", ""), + } + if not result["username"]: + try: + result["username"] = db_config.servicenow_secrets.get_secret_value( + SecretsGroupAccessTypeChoices.TYPE_REST, + SecretsGroupSecretTypeChoices.TYPE_USERNAME, + obj=db_config, + ) + except Exception as exc: # pylint: disable=broad-except + logger.error("Unable to retrieve ServiceNow username: %s", exc) + if not result["password"]: + try: + result["password"] = db_config.servicenow_secrets.get_secret_value( + SecretsGroupAccessTypeChoices.TYPE_REST, + SecretsGroupSecretTypeChoices.TYPE_PASSWORD, + obj=db_config, + ) + except Exception as exc: # pylint: disable=broad-except + logger.error("Unable to retrieve ServiceNow username: %s", exc) + return result diff --git a/nautobot_ssot/integrations/servicenow/views.py b/nautobot_ssot/integrations/servicenow/views.py new file mode 100644 index 00000000..403ca84d --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/views.py @@ -0,0 +1,32 @@ +"""UI view classes and methods for nautobot-ssot-servicenow.""" +from django.contrib import messages +from django.views.generic import UpdateView + +from nautobot.utilities.forms import restrict_form_fields + +from .forms import SSOTServiceNowConfigForm +from .models import SSOTServiceNowConfig + + +class SSOTServiceNowConfigView(UpdateView): + """Plugin-level configuration view for nautobot-ssot-servicenow.""" + + form_class = SSOTServiceNowConfigForm + template_name = "nautobot_ssot_servicenow/config.html" + + def get_object(self, queryset=None): # pylint: disable=unused-argument,no-self-use + """Retrieve the SSOTServiceNowConfig singleton instance.""" + return SSOTServiceNowConfig.load() + + def get_context_data(self, **kwargs): + """Get all necessary context for the view.""" + context = super().get_context_data(**kwargs) + restrict_form_fields(context["form"], self.request.user) + context["editing"] = True + context["obj"] = self.get_object() + return context + + def form_valid(self, form): + """Callback when the form is submitted successfully.""" + messages.success(self.request, "Successfully updated configuration") + return super().form_valid(form) diff --git a/nautobot_ssot/static/nautobot_ssot_servicenow/ServiceNow_logo.svg b/nautobot_ssot/static/nautobot_ssot_servicenow/ServiceNow_logo.svg new file mode 100644 index 00000000..7913d06c --- /dev/null +++ b/nautobot_ssot/static/nautobot_ssot_servicenow/ServiceNow_logo.svg @@ -0,0 +1,14 @@ + +ServiceNow logoA cloud computing and enterprise software provider based in Santa Clara, California, United Statesimage/svg+xml + + + + + + + + + + + + diff --git a/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html new file mode 100644 index 00000000..86c81a60 --- /dev/null +++ b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html @@ -0,0 +1,12 @@ +{% extends "generic/object_edit.html" %} + +{% block header %} + {% if settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.instance or settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.username or settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.password %} + + {% endif %} +{% endblock header %} diff --git a/nautobot_ssot/tests/servicenow/__init__.py b/nautobot_ssot/tests/servicenow/__init__.py new file mode 100644 index 00000000..a359aeca --- /dev/null +++ b/nautobot_ssot/tests/servicenow/__init__.py @@ -0,0 +1 @@ +"""Unit tests for nautobot_ssot_servicenow plugin.""" diff --git a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py new file mode 100644 index 00000000..b6b51288 --- /dev/null +++ b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py @@ -0,0 +1,78 @@ +"""Unit tests for the Nautobot DiffSync adapter.""" + +from unittest import mock +import uuid + +from django.conf import settings +from django.contrib.contenttypes.models import ContentType + +from nautobot.dcim.models import Device, DeviceRole, DeviceType, Interface, Manufacturer, Region, Site +from nautobot.extras.models import Job, JobResult, Status +from nautobot.utilities.testing import TransactionTestCase + +from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot_servicenow.diffsync.adapter_nautobot import NautobotDiffSync + + +if "job_logs" in settings.DATABASES: + settings.DATABASES["job_logs"] = settings.DATABASES["job_logs"].copy() + settings.DATABASES["job_logs"]["TEST"] = {"MIRROR": "default"} + + +class NautobotDiffSyncTestCase(TransactionTestCase): + """Test the NautobotDiffSync adapter class.""" + + databases = ("default", "job_logs") + + def setUp(self): + """Per-test-case data setup.""" + status_active = Status.objects.get(slug="active") + + region_1 = Region.objects.create(name="Region 1", slug="region-1") + region_2 = Region.objects.create(name="Region 2", slug="region-2", parent=region_1) + region_3 = Region.objects.create(name="Site/Region", slug="site-region", parent=region_1) + + site_1 = Site.objects.create(region=region_2, name="Site 1", slug="site-1", status=status_active) + site_2 = Site.objects.create(region=region_3, name="Site/Region", slug="site-region", status=status_active) + + manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") + device_type = DeviceType.objects.create(manufacturer=manufacturer, model="CSR 1000v", slug="csr1000v") + device_role = DeviceRole.objects.create(name="Router", slug="router") + + device_1 = Device.objects.create( + name="csr1", device_type=device_type, device_role=device_role, site=site_1, status=status_active + ) + device_2 = Device.objects.create( + name="csr2", device_type=device_type, device_role=device_role, site=site_2, status=status_active + ) + + Interface.objects.create(device=device_1, name="eth1") + Interface.objects.create(device=device_1, name="eth2") + Interface.objects.create(device=device_2, name="eth1") + Interface.objects.create(device=device_2, name="eth2") + + # Override the JOB_LOGS to None so that the Log Objects are created in the default database. + # This change is required as JOB_LOGS is a `fake` database pointed at the default. The django + # database cleanup will fail and cause tests to fail as this is not a real database. + @mock.patch("nautobot.extras.models.models.JOB_LOGS", None) + def test_data_loading(self): + """Test the load() function.""" + job = ServiceNowDataTarget() + job.job_result = JobResult.objects.create( + name=job.class_path, obj_type=ContentType.objects.get_for_model(Job), user=None, job_id=uuid.uuid4() + ) + nds = NautobotDiffSync(job=job, sync=None) + nds.load() + + self.assertEqual( + ["Region 1", "Region 2", "Site 1", "Site/Region"], + sorted(loc.get_unique_id() for loc in nds.get_all("location")), + ) + self.assertEqual( + ["csr1", "csr2"], + sorted(dev.get_unique_id() for dev in nds.get_all("device")), + ) + self.assertEqual( + ["csr1__eth1", "csr1__eth2", "csr2__eth1", "csr2__eth2"], + sorted(intf.get_unique_id() for intf in nds.get_all("interface")), + ) diff --git a/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py b/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py new file mode 100644 index 00000000..e00e7665 --- /dev/null +++ b/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py @@ -0,0 +1,618 @@ +"""Unit tests for the ServiceNowDiffSync adapter class.""" + +import uuid + +from django.contrib.contenttypes.models import ContentType + +from nautobot.extras.models import Job, JobResult +from nautobot.utilities.testing import TransactionTestCase + +from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot_servicenow.diffsync.adapter_servicenow import ServiceNowDiffSync + + +class MockServiceNowClient: + """Mock version of the ServiceNowClient class using canned data.""" + + def get_by_sys_id(self, table, sys_id): # pylint: disable=unused-argument,no-self-use + """Get a record with a given sys_id from a given table.""" + return None + + def all_table_entries(self, table, query=None): # pylint: disable=no-self-use + """Iterator over all records in a given table.""" + + if table == "cmn_location": + yield from [ + { + "country": "", + "parent": "", + "city": "", + "latitude": "", + "sys_updated_on": "2021-07-12 20:19:23", + "sys_id": "7200ad3d2f153010fe08351ef699b69a", + "sys_updated_by": "admin", + "stock_room": "false", + "street": "", + "sys_created_on": "2021-07-12 20:19:23", + "contact": "", + "phone_territory": "", + "company": "", + "lat_long_error": "", + "state": "", + "sys_created_by": "admin", + "longitude": "", + "zip": "", + "sys_mod_count": "0", + "sys_tags": "", + "time_zone": "", + "full_name": "Asia", + "fax_phone": "", + "phone": "", + "name": "Asia", + "coordinates_retrieved_on": "", + }, + { + "country": "Japan", + "parent": "7200ad3d2f153010fe08351ef699b69a", + "city": "Japan", + "latitude": "36.204824", + "sys_updated_on": "2021-07-12 20:19:30", + "sys_id": "0d9561b437d0200044e0bfc8bcbe5d32", + "sys_updated_by": "admin", + "stock_room": "false", + "street": "", + "sys_created_on": "2012-02-17 17:57:16", + "contact": "", + "phone_territory": "dcb7e002eb1201007128a5fc5206fe64", + "company": "81fd65ecac1d55eb42a426568fc87a63", + "lat_long_error": "", + "state": "", + "sys_created_by": "admin", + "longitude": "138.252924", + "zip": "", + "sys_mod_count": "1", + "sys_tags": "", + "time_zone": "", + "full_name": "Asia/Japan", + "fax_phone": "", + "phone": "", + "name": "Japan", + "coordinates_retrieved_on": "", + }, + { + "country": "Japan", + "parent": "0d9561b437d0200044e0bfc8bcbe5d32", + "city": "Tokyo", + "latitude": "35.6894875", + "sys_updated_on": "2012-02-19 17:11:11", + "sys_id": "821c169bac1d55eb68ede6e36aa35112", + "sys_updated_by": "admin", + "stock_room": "false", + "street": "", + "sys_created_on": "2010-11-25 08:17:47", + "contact": "", + "phone_territory": "", + "company": "81fd65ecac1d55eb42a426568fc87a63", + "lat_long_error": "", + "state": "", + "sys_created_by": "dariusz.maint", + "longitude": "139.6917064", + "zip": "", + "sys_mod_count": "3", + "sys_tags": "", + "time_zone": "", + "full_name": "Asia/Japan/Tokyo", + "fax_phone": "", + "phone": "", + "name": "Tokyo", + "coordinates_retrieved_on": "", + }, + { + "country": "China", + "parent": "7200ad3d2f153010fe08351ef699b69a", + "city": "China", + "latitude": "35.86166", + "sys_updated_on": "2021-07-12 20:19:26", + "sys_id": "8195ad7437d0200044e0bfc8bcbe5d8f", + "sys_updated_by": "admin", + "stock_room": "false", + "street": "", + "sys_created_on": "2012-02-17 17:57:15", + "contact": "", + "phone_territory": "4cb7e002eb1201007128a5fc5206fe0b", + "company": "81fdf9ebac1d55eb4cb89f136a082555", + "lat_long_error": "", + "state": "", + "sys_created_by": "admin", + "longitude": "104.195397", + "zip": "", + "sys_mod_count": "1", + "sys_tags": "", + "time_zone": "", + "full_name": "Asia/China", + "fax_phone": "", + "phone": "", + "name": "China", + "coordinates_retrieved_on": "", + }, + { + "country": "", + "parent": "8195ad7437d0200044e0bfc8bcbe5d8f", + "city": "", + "latitude": "", + "sys_updated_on": "2021-07-14 21:39:47", + "sys_id": "84a54c662f513010fe08351ef699b624", + "sys_updated_by": "admin", + "stock_room": "false", + "street": "", + "sys_created_on": "2021-07-14 21:39:47", + "contact": "", + "phone_territory": "", + "company": "", + "lat_long_error": "", + "state": "", + "sys_created_by": "admin", + "longitude": "", + "zip": "", + "sys_mod_count": "0", + "sys_tags": "", + "time_zone": "", + "full_name": "Asia/China/hkg", + "fax_phone": "", + "phone": "", + "name": "hkg", + "coordinates_retrieved_on": "", + }, + ] + elif table == "cmdb_ci_ip_switch": + if query and query["location"] == "84a54c662f513010fe08351ef699b624": # hkg + yield from [ + { + "attested_date": "", + "can_switch": "false", + "stack": "false", + "operational_status": "1", + "cpu_manufacturer": "", + "sys_updated_on": "2021-07-14 21:45:09", + "discovery_source": "", + "first_discovered": "", + "due_in": "", + "can_partitionvlans": "false", + "gl_account": "", + "invoice_number": "", + "sys_created_by": "admin", + "ram": "", + "warranty_expiration": "", + "cpu_speed": "", + "owned_by": "", + "checked_out": "", + "firmware_manufacturer": "", + "disk_space": "", + "sys_domain_path": "/", + "discovery_proto_id": "", + "maintenance_schedule": "", + "cost_center": "", + "attested_by": "", + "dns_domain": "", + "assigned": "", + "life_cycle_stage": "", + "purchase_date": "", + "short_description": "", + "managed_by": "", + "range": "", + "firmware_version": "", + "can_print": "false", + "last_discovered": "", + "ports": "", + "sys_class_name": "cmdb_ci_ip_switch", + "cpu_count": "1", + "manufacturer": "", + "life_cycle_stage_status": "", + "vendor": "", + "can_route": "false", + "model_number": "", + "assigned_to": "", + "start_date": "", + "bandwidth": "", + "serial_number": "", + "support_group": "", + "correlation_id": "", + "unverified": "false", + "attributes": "", + "asset": "a2d60ce62f513010fe08351ef699b618", + "skip_sync": "false", + "device_type": "", + "attestation_score": "", + "sys_updated_by": "admin", + "sys_created_on": "2021-07-14 21:40:27", + "cpu_type": "", + "sys_domain": "global", + "install_date": "", + "asset_tag": "", + "hardware_substatus": "", + "fqdn": "", + "stack_mode": "", + "change_control": "", + "internet_facing": "true", + "physical_interface_count": "", + "delivery_date": "", + "hardware_status": "installed", + "channels": "", + "install_status": "1", + "supported_by": "", + "name": "hkg-leaf-01", + "subcategory": "IP", + "default_gateway": "", + "assignment_group": "", + "managed_by_group": "", + "can_hub": "false", + "sys_id": "f9c500a62f513010fe08351ef699b65b", + "po_number": "", + "checked_in": "", + "sys_class_path": "/!!/!2/!!/!,", + "mac_address": "", + "company": "", + "justification": "", + "department": "", + "snmp_sys_location": "", + "comments": "", + "cost": "", + "sys_mod_count": "1", + "monitor": "false", + "ip_address": "", + "model_id": "aa722dbd2f153010fe08351ef699b605", + "duplicate_of": "", + "sys_tags": "", + "cost_cc": "USD", + "discovery_proto_type": "", + "order_date": "", + "schedule": "", + "environment": "", + "due": "", + "attested": "false", + "location": "84a54c662f513010fe08351ef699b624", + "category": "Resource", + "fault_count": "0", + "lease_id": "", + }, + { + "attested_date": "", + "can_switch": "false", + "stack": "false", + "operational_status": "1", + "cpu_manufacturer": "", + "sys_updated_on": "2021-07-14 21:45:07", + "discovery_source": "", + "first_discovered": "", + "due_in": "", + "can_partitionvlans": "false", + "gl_account": "", + "invoice_number": "", + "sys_created_by": "admin", + "ram": "", + "warranty_expiration": "", + "cpu_speed": "", + "owned_by": "", + "checked_out": "", + "firmware_manufacturer": "", + "disk_space": "", + "sys_domain_path": "/", + "discovery_proto_id": "", + "maintenance_schedule": "", + "cost_center": "", + "attested_by": "", + "dns_domain": "", + "assigned": "", + "life_cycle_stage": "", + "purchase_date": "", + "short_description": "", + "managed_by": "", + "range": "", + "firmware_version": "", + "can_print": "false", + "last_discovered": "", + "ports": "", + "sys_class_name": "cmdb_ci_ip_switch", + "cpu_count": "1", + "manufacturer": "", + "life_cycle_stage_status": "", + "vendor": "", + "can_route": "false", + "model_number": "", + "assigned_to": "", + "start_date": "", + "bandwidth": "", + "serial_number": "", + "support_group": "", + "correlation_id": "", + "unverified": "false", + "attributes": "", + "asset": "9ed6c8e62f513010fe08351ef699b6c8", + "skip_sync": "false", + "device_type": "", + "attestation_score": "", + "sys_updated_by": "admin", + "sys_created_on": "2021-07-14 21:40:36", + "cpu_type": "", + "sys_domain": "global", + "install_date": "", + "asset_tag": "", + "hardware_substatus": "", + "fqdn": "", + "stack_mode": "", + "change_control": "", + "internet_facing": "true", + "physical_interface_count": "", + "delivery_date": "", + "hardware_status": "installed", + "channels": "", + "install_status": "1", + "supported_by": "", + "name": "hkg-leaf-02", + "subcategory": "IP", + "default_gateway": "", + "assignment_group": "", + "managed_by_group": "", + "can_hub": "false", + "sys_id": "c4d540a62f513010fe08351ef699b602", + "po_number": "", + "checked_in": "", + "sys_class_path": "/!!/!2/!!/!,", + "mac_address": "", + "company": "", + "justification": "", + "department": "", + "snmp_sys_location": "", + "comments": "", + "cost": "", + "sys_mod_count": "1", + "monitor": "false", + "ip_address": "", + "model_id": "aa722dbd2f153010fe08351ef699b605", + "duplicate_of": "", + "sys_tags": "", + "cost_cc": "USD", + "discovery_proto_type": "", + "order_date": "", + "schedule": "", + "environment": "", + "due": "", + "attested": "false", + "location": "84a54c662f513010fe08351ef699b624", + "category": "Resource", + "fault_count": "0", + "lease_id": "", + }, + ] + else: + yield from [] + elif table == "cmdb_ci_network_adapter": + if query and query["cmdb_ci"] == "f9c500a62f513010fe08351ef699b65b": # hkg-leaf-01 + yield from [ + { + "mac_manufacturer": "", + "attested_date": "", + "skip_sync": "false", + "operational_status": "1", + "sys_updated_on": "2021-07-14 21:40:27", + "attestation_score": "", + "discovery_source": "", + "first_discovered": "", + "sys_updated_by": "admin", + "due_in": "", + "sys_created_on": "2021-07-14 21:40:27", + "sys_domain": "global", + "install_date": "", + "gl_account": "", + "invoice_number": "", + "sys_created_by": "admin", + "warranty_expiration": "", + "asset_tag": "", + "cmdb_ci": "f9c500a62f513010fe08351ef699b65b", + "fqdn": "", + "change_control": "", + "owned_by": "", + "checked_out": "", + "sys_domain_path": "/", + "dhcp_enabled": "false", + "delivery_date": "", + "maintenance_schedule": "", + "install_status": "1", + "cost_center": "", + "attested_by": "", + "supported_by": "", + "dns_domain": "", + "name": "Ethernet1", + "assigned": "", + "life_cycle_stage": "", + "purchase_date": "", + "subcategory": "Network", + "short_description": "", + "virtual": "false", + "assignment_group": "", + "managed_by": "", + "managed_by_group": "", + "can_print": "false", + "last_discovered": "", + "sys_class_name": "cmdb_ci_network_adapter", + "manufacturer": "", + "sys_id": "f9c500a62f513010fe08351ef699b65d", + "po_number": "", + "checked_in": "", + "netmask": "255.255.255.0", + "sys_class_path": "/!!/!8", + "life_cycle_stage_status": "", + "mac_address": "", + "vendor": "", + "alias": "", + "company": "", + "justification": "", + "model_number": "", + "department": "", + "assigned_to": "", + "start_date": "", + "comments": "", + "cost": "", + "sys_mod_count": "0", + "monitor": "false", + "serial_number": "", + "ip_address": "", + "model_id": "", + "duplicate_of": "", + "sys_tags": "", + "cost_cc": "USD", + "order_date": "", + "schedule": "", + "support_group": "", + "environment": "", + "due": "", + "attested": "false", + "correlation_id": "", + "unverified": "false", + "attributes": "", + "location": "", + "asset": "", + "category": "Hardware", + "fault_count": "0", + "ip_default_gateway": "", + "lease_id": "", + }, + { + "mac_manufacturer": "", + "attested_date": "", + "skip_sync": "false", + "operational_status": "1", + "sys_updated_on": "2021-07-14 21:40:28", + "attestation_score": "", + "discovery_source": "", + "first_discovered": "", + "sys_updated_by": "admin", + "due_in": "", + "sys_created_on": "2021-07-14 21:40:28", + "sys_domain": "global", + "install_date": "", + "gl_account": "", + "invoice_number": "", + "sys_created_by": "admin", + "warranty_expiration": "", + "asset_tag": "", + "cmdb_ci": "f9c500a62f513010fe08351ef699b65b", + "fqdn": "", + "change_control": "", + "owned_by": "", + "checked_out": "", + "sys_domain_path": "/", + "dhcp_enabled": "false", + "delivery_date": "", + "maintenance_schedule": "", + "install_status": "1", + "cost_center": "", + "attested_by": "", + "supported_by": "", + "dns_domain": "", + "name": "Ethernet2", + "assigned": "", + "life_cycle_stage": "", + "purchase_date": "", + "subcategory": "Network", + "short_description": "", + "virtual": "false", + "assignment_group": "", + "managed_by": "", + "managed_by_group": "", + "can_print": "false", + "last_discovered": "", + "sys_class_name": "cmdb_ci_network_adapter", + "manufacturer": "", + "sys_id": "4ac500a62f513010fe08351ef699b65f", + "po_number": "", + "checked_in": "", + "netmask": "255.255.255.0", + "sys_class_path": "/!!/!8", + "life_cycle_stage_status": "", + "mac_address": "", + "vendor": "", + "alias": "", + "company": "", + "justification": "", + "model_number": "", + "department": "", + "assigned_to": "", + "start_date": "", + "comments": "", + "cost": "", + "sys_mod_count": "0", + "monitor": "false", + "serial_number": "", + "ip_address": "", + "model_id": "", + "duplicate_of": "", + "sys_tags": "", + "cost_cc": "USD", + "order_date": "", + "schedule": "", + "support_group": "", + "environment": "", + "due": "", + "attested": "false", + "correlation_id": "", + "unverified": "false", + "attributes": "", + "location": "", + "asset": "", + "category": "Hardware", + "fault_count": "0", + "ip_default_gateway": "", + "lease_id": "", + }, + ] + else: + yield from [] + else: + yield from [] + + +class ServiceNowDiffSyncTestCase(TransactionTestCase): + """Test the ServiceNowDiffSync adapter class.""" + + databases = ("default", "job_logs") + + def test_data_loading(self): + """Test the load() function.""" + job = ServiceNowDataTarget() + job.job_result = JobResult.objects.create( + name=job.class_path, obj_type=ContentType.objects.get_for_model(Job), user=None, job_id=uuid.uuid4() + ) + snds = ServiceNowDiffSync(job=job, sync=None, client=MockServiceNowClient()) + snds.load() + + self.assertEqual( + ["Asia", "China", "Japan", "Tokyo", "hkg"], + sorted(loc.get_unique_id() for loc in snds.get_all("location")), + ) + japan = snds.get("location", "Japan") + self.assertEqual("Asia", japan.parent_location_name) + self.assertEqual("0d9561b437d0200044e0bfc8bcbe5d32", japan.sys_id) + self.assertEqual([], japan.devices) + + tokyo = snds.get("location", "Tokyo") + self.assertEqual("Japan", tokyo.parent_location_name) + self.assertEqual([], tokyo.devices) + + hkg = snds.get("location", "hkg") + self.assertEqual("China", hkg.parent_location_name) + self.assertEqual(["hkg-leaf-01", "hkg-leaf-02"], hkg.devices) + + self.assertEqual( + ["hkg-leaf-01", "hkg-leaf-02"], + sorted(dev.get_unique_id() for dev in snds.get_all("device")), + ) + + hkg_leaf_01 = snds.get("device", "hkg-leaf-01") + self.assertEqual("hkg", hkg_leaf_01.location_name) + self.assertEqual(["hkg-leaf-01__Ethernet1", "hkg-leaf-01__Ethernet2"], hkg_leaf_01.interfaces) + + self.assertEqual( + ["hkg-leaf-01__Ethernet1", "hkg-leaf-01__Ethernet2"], + sorted(intf.get_unique_id() for intf in snds.get_all("interface")), + ) diff --git a/nautobot_ssot/tests/servicenow/test_basic.py b/nautobot_ssot/tests/servicenow/test_basic.py new file mode 100644 index 00000000..8d4f9ef3 --- /dev/null +++ b/nautobot_ssot/tests/servicenow/test_basic.py @@ -0,0 +1,16 @@ +"""Basic tests that do not require Django.""" +import unittest +import os +import toml + +from nautobot_ssot_servicenow import __version__ as project_version + + +class TestVersion(unittest.TestCase): + """Test Version is the same.""" + + def test_version(self): + """Verify that pyproject.toml version is same as version specified in the package.""" + parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + poetry_version = toml.load(os.path.join(parent_path, "pyproject.toml"))["tool"]["poetry"]["version"] + self.assertEqual(project_version, poetry_version) diff --git a/nautobot_ssot/tests/servicenow/test_jobs.py b/nautobot_ssot/tests/servicenow/test_jobs.py new file mode 100644 index 00000000..7d21f62a --- /dev/null +++ b/nautobot_ssot/tests/servicenow/test_jobs.py @@ -0,0 +1,149 @@ +"""Test the Job class in this plugin.""" +import os +from unittest import mock + +from django.test import TestCase, override_settings +from django.urls import reverse + +from nautobot.dcim.models import Device, DeviceRole, DeviceType, Interface, Manufacturer, Region, Site +from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices +from nautobot.extras.models import Secret, SecretsGroup, SecretsGroupAssociation, Status + +from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot_servicenow.models import SSOTServiceNowConfig + + +class ServiceNowDataTargetJobTestCase(TestCase): + """Test the ServiceNowDataTarget Job.""" + + def test_metadata(self): + """Verify correctness of the Job Meta attributes.""" + self.assertEqual("Nautobot ⟹ ServiceNow", ServiceNowDataTarget.name) + self.assertEqual("Nautobot ⟹ ServiceNow", ServiceNowDataTarget.Meta.name) + self.assertEqual("ServiceNow", ServiceNowDataTarget.Meta.data_target) + self.assertEqual("Synchronize data from Nautobot into ServiceNow.", ServiceNowDataTarget.description) + self.assertEqual("Synchronize data from Nautobot into ServiceNow.", ServiceNowDataTarget.Meta.description) + + def test_data_mappings(self): + """Verify correctness of the data_mappings() API.""" + mappings = ServiceNowDataTarget.data_mappings() + + self.assertEqual("Device", mappings[0].source_name) + self.assertEqual(reverse("dcim:device_list"), mappings[0].source_url) + self.assertEqual("IP Switch", mappings[0].target_name) + self.assertIsNone(mappings[0].target_url) + + self.assertEqual("Device Type", mappings[1].source_name) + self.assertEqual(reverse("dcim:devicetype_list"), mappings[1].source_url) + self.assertEqual("Hardware Product Model", mappings[1].target_name) + self.assertIsNone(mappings[1].target_url) + + self.assertEqual("Interface", mappings[2].source_name) + self.assertEqual(reverse("dcim:interface_list"), mappings[2].source_url) + self.assertEqual("Interface", mappings[2].target_name) + self.assertIsNone(mappings[2].target_url) + + self.assertEqual("Manufacturer", mappings[3].source_name) + self.assertEqual(reverse("dcim:manufacturer_list"), mappings[3].source_url) + self.assertEqual("Company", mappings[3].target_name) + self.assertIsNone(mappings[3].target_url) + + self.assertEqual("Region", mappings[4].source_name) + self.assertEqual(reverse("dcim:region_list"), mappings[4].source_url) + self.assertEqual("Location", mappings[4].target_name) + self.assertIsNone(mappings[4].target_url) + + self.assertEqual("Site", mappings[5].source_name) + self.assertEqual(reverse("dcim:site_list"), mappings[5].source_url) + self.assertEqual("Location", mappings[5].target_name) + self.assertIsNone(mappings[5].target_url) + + @override_settings( + PLUGINS_CONFIG={"nautobot_ssot_servicenow": {"instance": "dev12345", "username": "admin", "password": ""}} + ) + def test_config_information_settings(self): + """Verify the config_information() API for configs provided in Django settings.""" + config_information = ServiceNowDataTarget.config_information() + self.assertEqual( + config_information, + { + "ServiceNow instance": "dev12345", + "Username": "admin", + # password should NOT be present! + }, + ) + + @override_settings(PLUGINS_CONFIG={}) + @mock.patch.dict(os.environ, {"SNOW_USERNAME": "someuser", "SNOW_PASSWORD": "notsosecret"}) + def test_config_information_db(self): + """Verify the config_information() API for configs provided in the database.""" + db_config = SSOTServiceNowConfig.load() + db_config.servicenow_instance = "dev98765" + user_secret = Secret.objects.create( + name="ServiceNow Username", + slug="servicenow-username", + provider="environment-variable", + parameters={"variable": "SNOW_USERNAME"}, + ) + password_secret = Secret.objects.create( + name="ServiceNow Password", + slug="servicenow-password", + provider="environment-variable", + parameters={"variable": "SNOW_PASSWORD"}, + ) + db_config.servicenow_secrets = SecretsGroup.objects.create( + name="ServiceNow Secrets", + slug="servicenow-secrets", + ) + SecretsGroupAssociation.objects.create( + group=db_config.servicenow_secrets, + secret=user_secret, + access_type=SecretsGroupAccessTypeChoices.TYPE_REST, + secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, + ) + SecretsGroupAssociation.objects.create( + group=db_config.servicenow_secrets, + secret=password_secret, + access_type=SecretsGroupAccessTypeChoices.TYPE_REST, + secret_type=SecretsGroupSecretTypeChoices.TYPE_PASSWORD, + ) + db_config.save() + + config_information = ServiceNowDataTarget.config_information() + self.assertEqual( + config_information, + { + "ServiceNow instance": "dev98765", + "Username": "someuser", + # password should NOT be present! + }, + ) + + def test_lookup_object(self): + """Validate the lookup_object() API.""" + region = Region.objects.create(name="My Region", slug="my-region") + site = Site.objects.create(name="My Site", slug="my-site", status=Status.objects.get(slug="active")) + manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") + device_type = DeviceType.objects.create(manufacturer=manufacturer, model="CSR 1000v", slug="csr1000v") + device_role = DeviceRole.objects.create(name="Router", slug="router") + device = Device.objects.create( + name="mydevice", + device_role=device_role, + device_type=device_type, + site=site, + status=Status.objects.get(slug="active"), + ) + interface = Interface.objects.create(device=device, name="eth0") + + job = ServiceNowDataTarget() + + self.assertEqual(job.lookup_object("location", "My Region"), region) + self.assertEqual(job.lookup_object("location", "My Site"), site) + self.assertEqual(job.lookup_object("device", "mydevice"), device) + self.assertEqual(job.lookup_object("interface", "mydevice__eth0"), interface) + + self.assertIsNone(job.lookup_object("location", "no such region")) + self.assertIsNone(job.lookup_object("location", "no such site")) + self.assertIsNone(job.lookup_object("device", "no-such-device")) + self.assertIsNone(job.lookup_object("interface", "no-such-device__no-such-interface")) + self.assertIsNone(job.lookup_object("nosuchmodel", "")) From 21acb3f6b68cbe501379cbfe2eb7edcf0eef2eab Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Tue, 13 Jun 2023 09:45:10 +0000 Subject: [PATCH 02/11] fix: Integrate ServiceNow files --- .flake8 | 1 + .pydocstyle.ini | 4 +- development/creds.example.env | 2 + development/development.env | 4 + development/nautobot_config.py | 4 + development/servicenow/Dockerfile | 79 ------------------- development/servicenow/creds.example.env | 16 ---- development/servicenow/dev.env | 16 ---- .../servicenow/docker-compose.base.yml | 40 ---------- development/servicenow/docker-compose.dev.yml | 20 ----- .../servicenow/docker-compose.docs.yml | 11 --- .../docker-compose.requirements.yml | 25 ------ development/servicenow/nautobot_config.py | 78 ------------------ nautobot_ssot/__init__.py | 4 + .../integrations/servicenow/__init__.py | 42 +--------- .../servicenow/diffsync/adapter_nautobot.py | 3 +- .../servicenow/diffsync/adapter_servicenow.py | 1 + .../servicenow/diffsync/models.py | 16 ++-- .../servicenow/migrations/0001_initial.py | 38 --------- .../servicenow/migrations/__init__.py | 0 .../integrations/servicenow/models.py | 2 +- .../integrations/servicenow/servicenow.py | 4 +- .../integrations/servicenow/signals.py | 9 ++- .../servicenow/third_party/pysnow/client.py | 2 +- nautobot_ssot/integrations/servicenow/urls.py | 4 +- .../migrations/0005_ssotservicenowconfig.py | 72 +++++++++++++++++ nautobot_ssot/models.py | 9 +++ .../nautobot_ssot_servicenow/config.html | 4 +- nautobot_ssot/tests/servicenow/__init__.py | 2 +- .../tests/servicenow/test_adapter_nautobot.py | 4 +- .../servicenow/test_adapter_servicenow.py | 4 +- nautobot_ssot/tests/servicenow/test_basic.py | 16 ---- nautobot_ssot/tests/servicenow/test_jobs.py | 4 +- poetry.lock | 38 ++++++++- pyproject.toml | 44 +++++++++++ 35 files changed, 212 insertions(+), 410 deletions(-) delete mode 100644 development/servicenow/Dockerfile delete mode 100644 development/servicenow/creds.example.env delete mode 100644 development/servicenow/dev.env delete mode 100644 development/servicenow/docker-compose.base.yml delete mode 100644 development/servicenow/docker-compose.dev.yml delete mode 100644 development/servicenow/docker-compose.docs.yml delete mode 100644 development/servicenow/docker-compose.requirements.yml delete mode 100644 development/servicenow/nautobot_config.py delete mode 100644 nautobot_ssot/integrations/servicenow/migrations/0001_initial.py delete mode 100644 nautobot_ssot/integrations/servicenow/migrations/__init__.py create mode 100644 nautobot_ssot/migrations/0005_ssotservicenowconfig.py delete mode 100644 nautobot_ssot/tests/servicenow/test_basic.py diff --git a/.flake8 b/.flake8 index 1587fc6c..0f6a3770 100644 --- a/.flake8 +++ b/.flake8 @@ -4,3 +4,4 @@ ignore = E501, W503 exclude = .venv + nautobot_ssot/integrations/servicenow/third_party diff --git a/.pydocstyle.ini b/.pydocstyle.ini index 541cc510..03f5e208 100644 --- a/.pydocstyle.ini +++ b/.pydocstyle.ini @@ -2,10 +2,10 @@ convention = google inherit = false match = (?!__init__).*\.py -match-dir = (?!tests|migrations|development)[^\.].* +match-dir = (?!tests|migrations|development|third_party)[^\.].* # D212 is enabled by default in google convention, and complains if we have a docstring like: # """ # My docstring is on the line after the opening quotes instead of on the same line as them. # """ # We've discussed and concluded that we consider this to be a valid style choice. -add_ignore = D212 \ No newline at end of file +add_ignore = D212 diff --git a/development/creds.example.env b/development/creds.example.env index 6f4aa74e..99ecca2e 100644 --- a/development/creds.example.env +++ b/development/creds.example.env @@ -27,3 +27,5 @@ MYSQL_PASSWORD=${NAUTOBOT_DB_PASSWORD} # NAUTOBOT_CONFIG=development/nautobot_config.py NAUTOBOT_SSOT_INFOBLOX_PASSWORD="changeme" + +SERVICENOW_PASSWORD="changeme" diff --git a/development/development.env b/development/development.env index 6d1d4f5e..100495ea 100644 --- a/development/development.env +++ b/development/development.env @@ -54,3 +54,7 @@ NAUTOBOT_SSOT_INFOBLOX_URL="https://infoblox.example.com" NAUTOBOT_SSOT_INFOBLOX_USERNAME="changeme" NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL="True" # NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION="" + +NAUTOBOT_SSOT_ENABLE_SERVICENOW="True" +SERVICENOW_INSTANCE="" +SERVICENOW_USERNAME="" diff --git a/development/nautobot_config.py b/development/nautobot_config.py index 491a7737..3ae91bc1 100644 --- a/development/nautobot_config.py +++ b/development/nautobot_config.py @@ -139,6 +139,7 @@ PLUGINS_CONFIG = { "nautobot_ssot": { "enable_infoblox": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_INFOBLOX")), + "enable_servicenow": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SERVICENOW")), "hide_example_jobs": is_truthy(os.getenv("NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS")), "infoblox_default_status": os.getenv("NAUTOBOT_SSOT_INFOBLOX_DEFAULT_STATUS", "active"), "infoblox_enable_rfc1918_network_containers": is_truthy( @@ -157,6 +158,9 @@ "infoblox_username": os.getenv("NAUTOBOT_SSOT_INFOBLOX_USERNAME"), "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", True)), "infoblox_wapi_version": os.getenv("NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION", "v2.12"), + "servicenow_instance": os.getenv("SERVICENOW_INSTANCE", ""), + "servicenow_password": os.getenv("SERVICENOW_PASSWORD", ""), + "servicenow_username": os.getenv("SERVICENOW_USERNAME", ""), }, } diff --git a/development/servicenow/Dockerfile b/development/servicenow/Dockerfile deleted file mode 100644 index cb501301..00000000 --- a/development/servicenow/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# ------------------------------------------------------------------------------------- -# Nautobot App Developement Dockerfile Template -# Version: 1.0.0 -# -# Apps that need to add additional steps or packages can do in the section below. -# ------------------------------------------------------------------------------------- -# !!! USE CAUTION WHEN MODIFYING LINES BELOW - -# Accepts a desired Nautobot version as build argument, default to 1.4.0 -ARG NAUTOBOT_VER="1.4" - -# Accepts a desired Python version as build argument, default to 3.8 -ARG PYTHON_VER="3.8" - -# Retreive published development image of Nautobot base which should include most CI dependencies -FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER} - -# Runtime argument and environment setup -ARG NAUTOBOT_ROOT=/opt/nautobot - -ENV prometheus_multiproc_dir=/prom_cache -ENV NAUTOBOT_ROOT ${NAUTOBOT_ROOT} - -# Install Poetry manually via its installer script; -# We might be using an older version of Nautobot that includes an older version of Poetry -# and CI and local development may have a newer version of Poetry -# Since this is only used for development and we don't ship this container, pinning Poetry back is not expressly necessary -# We also don't need virtual environments in container -RUN curl -sSL https://install.python-poetry.org -o /tmp/install-poetry.py && \ - python /tmp/install-poetry.py && \ - rm -f /tmp/install-poetry.py && \ - poetry config virtualenvs.create false - -# !!! USE CAUTION WHEN MODIFYING LINES ABOVE -# ------------------------------------------------------------------------------------- -# App-specifc system build/test dependencies. -# -# Example: LDAP requires `libldap2-dev` to be apt-installed before the Python package. -# ------------------------------------------------------------------------------------- -# --> Start safe to modify section - -# Uncomment the line below if you are apt-installing any package. -# RUN apt update -# RUN apt install libldap2-dev - -# --> Stop safe to modify section -# ------------------------------------------------------------------------------------- -# Install Nautobot App -# ------------------------------------------------------------------------------------- -# !!! USE CAUTION WHEN MODIFYING LINES BELOW - -# Copy in the source code -WORKDIR /source -COPY . /source - -# Get container's installed Nautobot version as a forced constraint -# NAUTOBOT_VER may be a branch name and not a published release therefor we need to get the installed version -# so pip can use it to recognize local constraints. -RUN pip show nautobot | grep "^Version: " | sed -e 's/Version: /nautobot==/' > constraints.txt - -# Use Poetry to grab dev dependencies from the lock file -# Can be improved in Poetry 1.2 which allows `poetry install --only dev` -# -# We can't use the entire freeze as it takes forever to resolve with rigidly fixed non-direct dependencies, -# especially those that are only direct to Nautobot but the container included versions slightly mismatch -RUN poetry export -f requirements.txt --without-hashes --output poetry_freeze_base.txt -RUN poetry export -f requirements.txt --dev --without-hashes --output poetry_freeze_all.txt -RUN sort poetry_freeze_base.txt poetry_freeze_all.txt | uniq -u > poetry_freeze_dev.txt - -# Install all local project as editable, constrained on Nautobot version, to get any additional -# direct dependencies of the app -RUN pip install -c constraints.txt -e . - -# Install any dev dependencies frozen from Poetry -# Can be improved in Poetry 1.2 which allows `poetry install --only dev` -RUN pip install -c constraints.txt -r poetry_freeze_dev.txt - -COPY development/nautobot_config.py ${NAUTOBOT_ROOT}/nautobot_config.py -# !!! USE CAUTION WHEN MODIFYING LINES ABOVE diff --git a/development/servicenow/creds.example.env b/development/servicenow/creds.example.env deleted file mode 100644 index 8583b0ae..00000000 --- a/development/servicenow/creds.example.env +++ /dev/null @@ -1,16 +0,0 @@ -NAUTOBOT_DB_PASSWORD=notverysecurepwd -NAUTOBOT_REDIS_PASSWORD=notverysecurepwd -NAUTOBOT_SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj -NAUTOBOT_CREATE_SUPERUSER=true -NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 -NAUTOBOT_SUPERUSER_PASSWORD=admin -# Needed for Postgres, must match the values above -PGPASSWORD=notverysecurepwd -POSTGRES_DB=nautobot -POSTGRES_PASSWORD=notverysecurepwd -POSTGRES_USER=nautobot -# Needed for Redis, must match the values above -REDIS_PASSWORD=notverysecurepwd -# NAUTOBOT_DB_HOST=localhost -# NAUTOBOT_REDIS_HOST=localhost -# NAUTOBOT_ROOT=./development diff --git a/development/servicenow/dev.env b/development/servicenow/dev.env deleted file mode 100644 index a557b394..00000000 --- a/development/servicenow/dev.env +++ /dev/null @@ -1,16 +0,0 @@ -NAUTOBOT_ALLOWED_HOSTS=* -NAUTOBOT_DEBUG=True -NAUTOBOT_METRICS_ENABLED=True -NAUTOBOT_ROOT=/opt/nautobot -NAUTOBOT_DB_NAME=nautobot -NAUTOBOT_DB_HOST=postgres -NAUTOBOT_DB_USER=nautobot -NAUTOBOT_REDIS_HOST=redis -NAUTOBOT_REDIS_PORT=6379 -# NAUTOBOT_REDIS_SSL=True -# Uncomment NAUTOBOT_REDIS_SSL if using SSL -SUPERUSER_EMAIL=admin@example.com -SUPERUSER_NAME=admin - -NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=1500 -NAUTOBOT_CELERY_TASK_TIME_LIMIT=1800 diff --git a/development/servicenow/docker-compose.base.yml b/development/servicenow/docker-compose.base.yml deleted file mode 100644 index 17b0ee9c..00000000 --- a/development/servicenow/docker-compose.base.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -x-nautobot-build: &nautobot-build - build: - args: - NAUTOBOT_VER: "${NAUTOBOT_VER}" - PYTHON_VER: "${PYTHON_VER}" - context: "../" - dockerfile: "development/Dockerfile" -x-nautobot-base: &nautobot-base - image: "nautobot-ssot-servicenow/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" - env_file: - - "dev.env" - - "creds.env" - tty: true - -version: "3.4" -services: - nautobot: - ports: - - "8080:8080" - depends_on: - - "postgres" - - "redis" - <<: *nautobot-build - <<: *nautobot-base - worker: - entrypoint: "nautobot-server rqworker" - depends_on: - - "nautobot" - healthcheck: - disable: true - <<: *nautobot-base - celery_worker: - entrypoint: "nautobot-server celery worker -l INFO" - depends_on: - - "nautobot" - - "redis" - healthcheck: - disable: true - <<: *nautobot-base diff --git a/development/servicenow/docker-compose.dev.yml b/development/servicenow/docker-compose.dev.yml deleted file mode 100644 index 610c0782..00000000 --- a/development/servicenow/docker-compose.dev.yml +++ /dev/null @@ -1,20 +0,0 @@ -# We can't remove volumes in a compose override, for the test configuration using the final containers -# we don't want the volumes so this is the default override file to add the volumes in the dev case -# any override will need to include these volumes to use them. -# see: https://github.com/docker/compose/issues/3729 ---- -version: "3.4" -services: - nautobot: - command: "nautobot-server runserver 0.0.0.0:8080 --insecure" - volumes: - - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" - - "../:/source" - worker: - volumes: - - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" - - "../:/source" - celery_worker: - volumes: - - "./nautobot_config.py:/opt/nautobot/nautobot_config.py" - - "../:/source" diff --git a/development/servicenow/docker-compose.docs.yml b/development/servicenow/docker-compose.docs.yml deleted file mode 100644 index a09bafa1..00000000 --- a/development/servicenow/docker-compose.docs.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -version: "3.4" -services: - docs: - image: "nautobot-ssot-servicenow/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}" - entrypoint: "mkdocs serve -v -a 0.0.0.0:8080" - volumes: - - "../docs:/source/docs:ro" - - "../mkdocs.yml:/source/mkdocs.yml:ro" - ports: - - "8001:8080" diff --git a/development/servicenow/docker-compose.requirements.yml b/development/servicenow/docker-compose.requirements.yml deleted file mode 100644 index 175cd297..00000000 --- a/development/servicenow/docker-compose.requirements.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -version: "3.4" -services: - postgres: - image: "postgres:13-alpine" - env_file: - - "dev.env" - - "creds.env" - volumes: - - "postgres_data:/var/lib/postgresql/data" - ports: - - "5432:5432" - redis: - image: "redis:6-alpine" - command: - - "sh" - - "-c" # this is to evaluate the $REDIS_PASSWORD from the env - - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD" - env_file: - - "dev.env" - - "creds.env" - ports: - - "6379:6379" -volumes: - postgres_data: {} diff --git a/development/servicenow/nautobot_config.py b/development/servicenow/nautobot_config.py deleted file mode 100644 index ed1a4242..00000000 --- a/development/servicenow/nautobot_config.py +++ /dev/null @@ -1,78 +0,0 @@ -######################### -# # -# Required settings # -# # -######################### - -import os -import sys - -from nautobot.core.settings import * # noqa: F401,F403 -from nautobot.core.settings_funcs import parse_redis_connection - -TESTING = len(sys.argv) > 1 and sys.argv[1] == "test" - -# This is a list of valid fully-qualified domain names (FQDNs) for the Nautobot server. Nautobot will not permit write -# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. -# -# Example: ALLOWED_HOSTS = ['nautobot.example.com', 'nautobot.internal.local'] -ALLOWED_HOSTS = os.getenv("NAUTOBOT_ALLOWED_HOSTS").split(" ") - -# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: -# https://docs.djangoproject.com/en/stable/ref/settings/#databases -DATABASES = { - "default": { - "NAME": os.getenv("NAUTOBOT_DB_NAME", "nautobot"), # Database name - "USER": os.getenv("NAUTOBOT_DB_USER", ""), # Database username - "PASSWORD": os.getenv("NAUTOBOT_DB_PASSWORD", ""), # Datbase password - "HOST": os.getenv("NAUTOBOT_DB_HOST", "localhost"), # Database server - "PORT": os.getenv("NAUTOBOT_DB_PORT", ""), # Database port (leave blank for default) - "CONN_MAX_AGE": os.getenv("NAUTOBOT_DB_TIMEOUT", 300), # Database timeout - "ENGINE": "django.db.backends.postgresql", # Database driver (Postgres only supported!) - } -} - -# The django-redis cache is used to establish concurrent locks using Redis. The -# django-rq settings will use the same instance/database by default. -# -# This "default" server is now used by RQ_QUEUES. -# >> See: nautobot.core.settings.RQ_QUEUES -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": parse_redis_connection(redis_database=0), - "TIMEOUT": 300, - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "PASSWORD": os.getenv("NAUTOBOT_REDIS_PASSWORD", ""), - }, - } -} - -# RQ_QUEUES is not set here because it just uses the default that gets imported -# up top via `from nautobot.core.settings import *`. - -# REDIS CACHEOPS -CACHEOPS_REDIS = parse_redis_connection(redis_database=1) - -# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. -# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and -# symbols. Nautobot will not run without this defined. For more information, see -# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = os.getenv("NAUTOBOT_SECRET_KEY", "") - -# Enable installed plugins. Add the name of each plugin to the list. -PLUGINS = ["nautobot_ssot", "nautobot_ssot_servicenow"] - -# Plugins configuration settings. These settings are used by various plugins that the user may have installed. -# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. -PLUGINS_CONFIG = { - "nautobot_ssot": { - "hide_example_jobs": True, - }, - "nautobot_ssot_servicenow": { - "instance": os.getenv("SERVICENOW_INSTANCE", ""), - "username": os.getenv("SERVICENOW_USERNAME", ""), - "password": os.getenv("SERVICENOW_PASSWORD", ""), - }, -} diff --git a/nautobot_ssot/__init__.py b/nautobot_ssot/__init__.py index 06597377..145e224d 100644 --- a/nautobot_ssot/__init__.py +++ b/nautobot_ssot/__init__.py @@ -29,6 +29,7 @@ class NautobotSSOTPluginConfig(PluginConfig): max_version = "1.9999" default_settings = { "enable_infoblox": False, + "enable_servicenow": False, "hide_example_jobs": True, "infoblox_default_status": "", "infoblox_enable_rfc1918_network_containers": False, @@ -43,6 +44,9 @@ class NautobotSSOTPluginConfig(PluginConfig): "infoblox_username": "", "infoblox_verify_ssl": True, "infoblox_wapi_version": "", + "servicenow_instance": "", + "servicenow_password": "", + "servicenow_username": "", } caching_config = {} diff --git a/nautobot_ssot/integrations/servicenow/__init__.py b/nautobot_ssot/integrations/servicenow/__init__.py index 3434e9da..b145df64 100644 --- a/nautobot_ssot/integrations/servicenow/__init__.py +++ b/nautobot_ssot/integrations/servicenow/__init__.py @@ -1,41 +1 @@ -"""Plugin declaration for nautobot_ssot_servicenow.""" -from nautobot.core.signals import nautobot_database_ready -from nautobot.extras.plugins import PluginConfig - -from .signals import nautobot_database_ready_callback - -try: - from importlib import metadata -except ImportError: - # Running on pre-3.8 Python; use importlib-metadata package - import importlib_metadata as metadata - -__version__ = metadata.version(__name__) - - -class NautobotSSOTServiceNowConfig(PluginConfig): - """Plugin configuration for the nautobot_ssot_servicenow plugin.""" - - name = "nautobot_ssot_servicenow" - verbose_name = "Nautobot SSoT ServiceNow" - version = __version__ - author = "Network to Code, LLC" - description = "Nautobot SSoT ServiceNow." - base_url = "ssot-servicenow" - required_settings = [] - min_version = "1.4.0" - max_version = "1.9999" - default_settings = {} - required_settings = [] - caching_config = {} - - home_view_name = "plugins:nautobot_ssot:dashboard" # a link to the ServiceNow job would be even better - config_view_name = "plugins:nautobot_ssot_servicenow:config" - - def ready(self): - """Callback when this plugin is loaded.""" - super().ready() - nautobot_database_ready.connect(nautobot_database_ready_callback, sender=self) - - -config = NautobotSSOTServiceNowConfig # pylint:disable=invalid-name +"""Base module for ServiceNow integration.""" diff --git a/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py b/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py index bdd8187e..97a62bf6 100644 --- a/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py +++ b/nautobot_ssot/integrations/servicenow/diffsync/adapter_nautobot.py @@ -1,3 +1,4 @@ +# pylint: disable=duplicate-code """DiffSync adapter class for Nautobot as source-of-truth.""" import datetime @@ -167,7 +168,7 @@ def load(self): def tag_involved_objects(self, target): """Tag all objects that were successfully synced to the target.""" # The ssot-synced-to-servicenow tag *should* have been created automatically during plugin installation - # (see nautobot_ssot_servicenow/signals.py) but maybe a user deleted it inadvertently, so be safe: + # (see nautobot_ssot/integrations/servicenow/signals.py) but maybe a user deleted it inadvertently, so be safe: tag, _ = Tag.objects.get_or_create( slug="ssot-synced-to-servicenow", defaults={ diff --git a/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py b/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py index f5f3821d..a71e67b2 100644 --- a/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py +++ b/nautobot_ssot/integrations/servicenow/diffsync/adapter_servicenow.py @@ -1,3 +1,4 @@ +# pylint: disable=duplicate-code """DiffSync adapter for ServiceNow.""" from base64 import b64encode import json diff --git a/nautobot_ssot/integrations/servicenow/diffsync/models.py b/nautobot_ssot/integrations/servicenow/diffsync/models.py index 39d48a29..6b7098e3 100644 --- a/nautobot_ssot/integrations/servicenow/diffsync/models.py +++ b/nautobot_ssot/integrations/servicenow/diffsync/models.py @@ -6,7 +6,7 @@ from diffsync.enum import DiffSyncStatus # import pysnow -from nautobot_ssot_servicenow.third_party import pysnow +from nautobot_ssot.integrations.servicenow.third_party import pysnow class ServiceNowCRUDMixin: @@ -98,7 +98,7 @@ class Company(ServiceNowCRUDMixin, DiffSyncModel): name: str manufacturer: bool = False - product_models: List["ProductModel"] = list() + product_models: List["ProductModel"] = [] sys_id: Optional[str] = None pk: Optional[uuid.UUID] = None @@ -136,11 +136,11 @@ class Location(ServiceNowCRUDMixin, DiffSyncModel): name: str parent_location_name: Optional[str] - contained_locations: List["Location"] = list() + contained_locations: List["Location"] = [] latitude: Union[float, str] = "" # can't use Optional[float] because an empty string doesn't map to None longitude: Union[float, str] = "" - devices: List["Device"] = list() + devices: List["Device"] = [] sys_id: Optional[str] = None region_pk: Optional[uuid.UUID] = None @@ -180,7 +180,7 @@ class Device(ServiceNowCRUDMixin, DiffSyncModel): role: Optional[str] vendor: Optional[str] - interfaces: List["Interface"] = list() + interfaces: List["Interface"] = [] sys_id: Optional[str] = None pk: Optional[uuid.UUID] = None @@ -217,12 +217,12 @@ class Interface(ServiceNowCRUDMixin, DiffSyncModel): access_vlan: Optional[int] active: Optional[bool] - allowed_vlans: List[str] = list() + allowed_vlans: List[str] = [] description: Optional[str] is_virtual: Optional[bool] is_lag: Optional[bool] is_lag_member: Optional[bool] - lag_members: List[str] = list() + lag_members: List[str] = [] mode: Optional[str] # TRUNK, ACCESS, L3, NONE mtu: Optional[int] parent: Optional[str] @@ -230,7 +230,7 @@ class Interface(ServiceNowCRUDMixin, DiffSyncModel): switchport_mode: Optional[str] type: Optional[str] - ip_addresses: List["IPAddress"] = list() + ip_addresses: List["IPAddress"] = [] sys_id: Optional[str] = None pk: Optional[uuid.UUID] = None diff --git a/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py b/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py deleted file mode 100644 index 70cefe25..00000000 --- a/nautobot_ssot/integrations/servicenow/migrations/0001_initial.py +++ /dev/null @@ -1,38 +0,0 @@ -# Generated by Django 3.1.12 on 2021-12-21 21:49 - -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ("extras", "0018_joblog_data_migration"), - ] - - operations = [ - migrations.CreateModel( - name="SSOTServiceNowConfig", - fields=[ - ( - "id", - models.UUIDField( - default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True - ), - ), - ("servicenow_instance", models.CharField(blank=True, max_length=100)), - ( - "servicenow_secrets", - models.ForeignKey( - blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="extras.secretsgroup" - ), - ), - ], - options={ - "abstract": False, - }, - ), - ] diff --git a/nautobot_ssot/integrations/servicenow/migrations/__init__.py b/nautobot_ssot/integrations/servicenow/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/nautobot_ssot/integrations/servicenow/models.py b/nautobot_ssot/integrations/servicenow/models.py index 2fa5611c..9ce80a77 100644 --- a/nautobot_ssot/integrations/servicenow/models.py +++ b/nautobot_ssot/integrations/servicenow/models.py @@ -39,4 +39,4 @@ def __str__(self): def get_absolute_url(self): # pylint: disable=no-self-use """Get URL for the associated configuration view.""" - return reverse("plugins:nautobot_ssot_servicenow:config") + return reverse("plugins:nautobot_ssot:servicenow_config") diff --git a/nautobot_ssot/integrations/servicenow/servicenow.py b/nautobot_ssot/integrations/servicenow/servicenow.py index af0c7249..b154e234 100644 --- a/nautobot_ssot/integrations/servicenow/servicenow.py +++ b/nautobot_ssot/integrations/servicenow/servicenow.py @@ -2,10 +2,10 @@ import logging # from pysnow import Client -from nautobot_ssot_servicenow.third_party.pysnow import Client +from nautobot_ssot.integrations.servicenow.third_party.pysnow import Client # from pysnow.exceptions import MultipleResults -from nautobot_ssot_servicenow.third_party.pysnow.exceptions import MultipleResults +from nautobot_ssot.integrations.servicenow.third_party.pysnow.exceptions import MultipleResults import requests # pylint: disable=wrong-import-order diff --git a/nautobot_ssot/integrations/servicenow/signals.py b/nautobot_ssot/integrations/servicenow/signals.py index e361a3af..d2db35d7 100644 --- a/nautobot_ssot/integrations/servicenow/signals.py +++ b/nautobot_ssot/integrations/servicenow/signals.py @@ -1,9 +1,16 @@ -"""Signal handlers for nautobot_ssot_servicenow.""" +# pylint: disable=duplicate-code +"""Signal handlers for ServiceNow integration.""" +from nautobot.core.signals import nautobot_database_ready from nautobot.extras.choices import CustomFieldTypeChoices from nautobot.utilities.choices import ColorChoices +def register_signals(sender): + """Register signals for Infoblox integration.""" + nautobot_database_ready.connect(nautobot_database_ready_callback, sender=sender) + + def nautobot_database_ready_callback(sender, *, apps, **kwargs): # pylint: disable=unused-argument """Callback function triggered by the nautobot_database_ready signal when the Nautobot database is fully ready.""" # pylint: disable=invalid-name diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py b/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py index 7e67d16f..938b4cca 100644 --- a/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/client.py @@ -5,7 +5,7 @@ import warnings import requests -from nautobot_ssot_servicenow.third_party import pysnow +from nautobot_ssot.integrations.servicenow.third_party import pysnow from requests.auth import HTTPBasicAuth from .legacy_request import LegacyRequest diff --git a/nautobot_ssot/integrations/servicenow/urls.py b/nautobot_ssot/integrations/servicenow/urls.py index b4451cb3..f50629e7 100644 --- a/nautobot_ssot/integrations/servicenow/urls.py +++ b/nautobot_ssot/integrations/servicenow/urls.py @@ -3,8 +3,6 @@ from . import views -app_name = "nautobot_ssot_servicenow" - urlpatterns = [ - path("config/", views.SSOTServiceNowConfigView.as_view(), name="config"), + path("servicenow/config/", views.SSOTServiceNowConfigView.as_view(), name="servicenow_config"), ] diff --git a/nautobot_ssot/migrations/0005_ssotservicenowconfig.py b/nautobot_ssot/migrations/0005_ssotservicenowconfig.py new file mode 100644 index 00000000..0e52e0bc --- /dev/null +++ b/nautobot_ssot/migrations/0005_ssotservicenowconfig.py @@ -0,0 +1,72 @@ +# Generated by Django 3.2.16 on 2023-06-13 09:15 + +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models +from django.db.migrations.recorder import MigrationRecorder + +import django.db.models.deletion +import uuid + + +_APP_LABEL = "nautobot_ssot" +_OLD_APP_LABEL = "nautobot_plugin_ssot_servicenow" +_MODEL_NAME = "SSOTServiceNowConfig" + + +def _move_data(apps, schema_editor): + old_migration = { + "app": _OLD_APP_LABEL, + "name": "0001_initial", + } + if not MigrationRecorder.Migration.objects.filter(**old_migration).exists(): + return + + new_model_name = _MODEL_NAME.lower() + new_model = apps.get_model(_APP_LABEL, new_model_name) + new_table_name = new_model._meta.db_table + old_table_name = new_table_name.replace(f"{_APP_LABEL}_", f"{_OLD_APP_LABEL}_") + + with schema_editor.connection.cursor() as cursor: + # Table names are from trusted source (this script) + cursor.execute(f"INSERT INTO {new_table_name} SELECT * FROM {old_table_name};") # nosec + + # Update the content type to point to the new model + old_content_type = ContentType.objects.get(app_label=_OLD_APP_LABEL, model=_MODEL_NAME.lower()) + old_content_type.app_label = _APP_LABEL + old_content_type.model = new_model_name + old_content_type.save() + + with schema_editor.connection.cursor() as cursor: + cursor.execute(f"DROP TABLE {old_table_name} CASCADE;") + + +class Migration(migrations.Migration): + dependencies = [ + ("extras", "0053_relationship_required_on"), + ("nautobot_ssot", "0004_sync_summary"), + ] + + operations = [ + migrations.CreateModel( + name=_MODEL_NAME, + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True + ), + ), + ("servicenow_instance", models.CharField(blank=True, max_length=100)), + ( + "servicenow_secrets", + models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to="extras.secretsgroup" + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.RunPython(_move_data), + ] diff --git a/nautobot_ssot/models.py b/nautobot_ssot/models.py index 47b0cebd..4b4d5f0b 100644 --- a/nautobot_ssot/models.py +++ b/nautobot_ssot/models.py @@ -30,6 +30,8 @@ from nautobot.extras.models import JobResult from nautobot.extras.utils import extras_features +from nautobot_ssot.integrations.servicenow.models import SSOTServiceNowConfig + from .choices import SyncLogEntryActionChoices, SyncLogEntryStatusChoices @@ -188,3 +190,10 @@ def get_status_class(self): SyncLogEntryStatusChoices.STATUS_FAILURE: "warning", SyncLogEntryStatusChoices.STATUS_ERROR: "danger", }.get(self.status) + + +__all__ = ( + "SSOTServiceNowConfig", + "Sync", + "SyncLogEntry", +) diff --git a/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html index 86c81a60..d0637eb4 100644 --- a/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html +++ b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html @@ -1,12 +1,12 @@ {% extends "generic/object_edit.html" %} {% block header %} - {% if settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.instance or settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.username or settings.PLUGINS_CONFIG.nautobot_ssot_servicenow.password %} + {% if settings.PLUGINS_CONFIG.nautobot_ssot.servicenow_instance or settings.PLUGINS_CONFIG.nautobot_ssot.servicenow_username or settings.PLUGINS_CONFIG.nautobot_ssot.servicenow_password %} {% endif %} {% endblock header %} diff --git a/nautobot_ssot/tests/servicenow/__init__.py b/nautobot_ssot/tests/servicenow/__init__.py index a359aeca..855bfcdd 100644 --- a/nautobot_ssot/tests/servicenow/__init__.py +++ b/nautobot_ssot/tests/servicenow/__init__.py @@ -1 +1 @@ -"""Unit tests for nautobot_ssot_servicenow plugin.""" +"""Unit tests for ServiceNow integration.""" diff --git a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py index b6b51288..9ee3a286 100644 --- a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py +++ b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py @@ -10,8 +10,8 @@ from nautobot.extras.models import Job, JobResult, Status from nautobot.utilities.testing import TransactionTestCase -from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget -from nautobot_ssot_servicenow.diffsync.adapter_nautobot import NautobotDiffSync +from nautobot_ssot.integrations.servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot.integrations.servicenow.diffsync.adapter_nautobot import NautobotDiffSync if "job_logs" in settings.DATABASES: diff --git a/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py b/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py index e00e7665..3e450273 100644 --- a/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py +++ b/nautobot_ssot/tests/servicenow/test_adapter_servicenow.py @@ -7,8 +7,8 @@ from nautobot.extras.models import Job, JobResult from nautobot.utilities.testing import TransactionTestCase -from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget -from nautobot_ssot_servicenow.diffsync.adapter_servicenow import ServiceNowDiffSync +from nautobot_ssot.integrations.servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot.integrations.servicenow.diffsync.adapter_servicenow import ServiceNowDiffSync class MockServiceNowClient: diff --git a/nautobot_ssot/tests/servicenow/test_basic.py b/nautobot_ssot/tests/servicenow/test_basic.py deleted file mode 100644 index 8d4f9ef3..00000000 --- a/nautobot_ssot/tests/servicenow/test_basic.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Basic tests that do not require Django.""" -import unittest -import os -import toml - -from nautobot_ssot_servicenow import __version__ as project_version - - -class TestVersion(unittest.TestCase): - """Test Version is the same.""" - - def test_version(self): - """Verify that pyproject.toml version is same as version specified in the package.""" - parent_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) - poetry_version = toml.load(os.path.join(parent_path, "pyproject.toml"))["tool"]["poetry"]["version"] - self.assertEqual(project_version, poetry_version) diff --git a/nautobot_ssot/tests/servicenow/test_jobs.py b/nautobot_ssot/tests/servicenow/test_jobs.py index 7d21f62a..c73d518c 100644 --- a/nautobot_ssot/tests/servicenow/test_jobs.py +++ b/nautobot_ssot/tests/servicenow/test_jobs.py @@ -9,8 +9,8 @@ from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices from nautobot.extras.models import Secret, SecretsGroup, SecretsGroupAssociation, Status -from nautobot_ssot_servicenow.jobs import ServiceNowDataTarget -from nautobot_ssot_servicenow.models import SSOTServiceNowConfig +from nautobot_ssot.integrations.servicenow.jobs import ServiceNowDataTarget +from nautobot_ssot.integrations.servicenow.models import SSOTServiceNowConfig class ServiceNowDataTargetJobTestCase(TestCase): diff --git a/poetry.lock b/poetry.lock index 74ad9e7b..3a30b1b2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1410,6 +1410,27 @@ files = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] +[[package]] +name = "ijson" +version = "2.6.1" +description = "Iterative JSON parser with a standard Python iterator interface" +optional = true +python-versions = "*" +files = [ + {file = "ijson-2.6.1-cp27-cp27m-macosx_10_6_x86_64.whl", hash = "sha256:60393946d73792d5adeeaa25e82ff2f5bf19b17f6617a468743a4db4a07298a0"}, + {file = "ijson-2.6.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d320dc1c1c9adbe404668b0fed6bfa0ac8693911159564f4655a5f2059746993"}, + {file = "ijson-2.6.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a4cd7f8ecf035d0e23db1cc6d6036e6c563f31abacbceae88904bb8b7f88b1f6"}, + {file = "ijson-2.6.1-cp34-cp34m-macosx_10_6_x86_64.whl", hash = "sha256:9904bf55bc1f170353c32144861d8295f0bdc41034e5e6ae58cbf30610023ca6"}, + {file = "ijson-2.6.1-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:7bac04b23691e6ab122d8f9ff06b26dbbb6df01babbf6bf8856ccad1c505278b"}, + {file = "ijson-2.6.1-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:ae9cc3ebbe8fa030b923b5dff912a61980edd03dc00b92f5c0223e44cbc51d9f"}, + {file = "ijson-2.6.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8ce67b7d3435c3fc831d5c06f60b2d20a853b599cdf885478e575a3416fbf655"}, + {file = "ijson-2.6.1-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:a8b486bdf24e389947e588f4021498f6cc56cafdfaec1c78e9952e0f338aef23"}, + {file = "ijson-2.6.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c0042bb768fb890c177af923c0ead157cdc70c6dfa64827765c1a3676a879190"}, + {file = "ijson-2.6.1-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:26978c02314233c87bddad8800b7b9a56a052334f495e2bce93b282397c6931d"}, + {file = "ijson-2.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:25d4d159405f75a7443c1fe83b6d7be5a7da017b4aa9cc1bb5cda3feb74aaf32"}, + {file = "ijson-2.6.1.tar.gz", hash = "sha256:75ebc60b23abfb1c97f475ab5d07a5ed725ad4bd1f58513d8b258c21f02703d0"}, +] + [[package]] name = "importlib-metadata" version = "4.13.0" @@ -2731,6 +2752,17 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-magic" +version = "0.4.27" +description = "File type identification using libmagic" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "python-magic-0.4.27.tar.gz", hash = "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b"}, + {file = "python_magic-0.4.27-py2.py3-none-any.whl", hash = "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"}, +] + [[package]] name = "python3-openid" version = "3.2.0" @@ -3632,10 +3664,12 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -all = ["dnspython"] +all = ["Jinja2", "PyYAML", "dnspython", "ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] infoblox = ["dnspython"] +pysnow = ["ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] +servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", "requests", "requests-oauthlib", "six"] [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "8fd2a2a0e9f571a46c5a224bf2ddc487f6dcb2545af1dbea61323c287ebd5481" +content-hash = "c7d3a8e4085754b32e13b58a06363e19744c3447a4035af2da5ee9537a5ee0b5" diff --git a/pyproject.toml b/pyproject.toml index 82d4dc2b..01cd09bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,13 +26,22 @@ packages = [ ] [tool.poetry.dependencies] +Jinja2 = { version = ">=2.11.3", optional = true } Markdown = "!=3.3.5" +PyYAML = { version = "^6", optional = true } diffsync = "^1.6.0" dnspython = { version = "^2.1.0", optional = true } +ijson = { version = "^2.5.1", optional = true } nautobot = "^1.4.0" +oauthlib = { version = "^3.1.0", optional = true } packaging = ">=21.3, <24" prometheus-client = "~0.14.1" python = "^3.7" +python-magic = { version = "^0.4.15", optional = true } +pytz = { version = ">=2019.3", optional = true } +requests = { version = "^2.21.0", optional = true } +requests-oauthlib = { version = "^1.3.0", optional = true } +six = { version = "^1.13.0", optional = true } [tool.poetry.dev-dependencies] bandit = "*" @@ -71,11 +80,45 @@ requests-mock = "^1.10.0" [tool.poetry.extras] all = [ + "Jinja2", + "PyYAML", "dnspython", + "ijson", + "oauthlib", + "python-magic", + "pytz", + "requests", + "requests-oauthlib", + "six", ] infoblox = [ "dnspython", ] +# pysnow = "^0.7.17" +# PySNow is currently pinned to an older version of pytz as a dependency, which blocks compatibility with newer +# versions of Nautobot. See https://github.com/rbw/pysnow/pull/186 +# For now, we have embedded a copy of PySNow under nautobot_ssot/integrations/servicenow/third_party/pysnow; +# here are its direct packaging dependencies: +pysnow = [ + "requests", + "oauthlib", + "python-magic", + "requests-oauthlib", + "six", + "ijson", + "pytz", +] +servicenow = [ + "Jinja2", + "PyYAML", + "ijson", + "oauthlib", + "python-magic", + "pytz", + "requests", + "requests-oauthlib", + "six", +] [tool.black] line-length = 120 @@ -94,6 +137,7 @@ exclude = ''' | buck-out | build | dist + | nautobot_ssot/integrations/servicenow/third_party )/ | settings.py # This is where you define files that should not be stylized by black # the root of the project From 0a490a0e0147bd6c0bf33087687faa075f8b23ba Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 14:05:13 +0000 Subject: [PATCH 03/11] fix: `nautobot_ssot_servicenow` leftover --- nautobot_ssot/integrations/servicenow/utils.py | 8 ++++---- nautobot_ssot/tests/servicenow/test_jobs.py | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nautobot_ssot/integrations/servicenow/utils.py b/nautobot_ssot/integrations/servicenow/utils.py index cd1c489e..3b50a767 100644 --- a/nautobot_ssot/integrations/servicenow/utils.py +++ b/nautobot_ssot/integrations/servicenow/utils.py @@ -14,11 +14,11 @@ def get_servicenow_parameters(): """Get a dictionary containing the instance, username, and password for connecting to ServiceNow.""" db_config = SSOTServiceNowConfig.load() - settings_config = settings.PLUGINS_CONFIG.get("nautobot_ssot_servicenow", {}) + settings_config = settings.PLUGINS_CONFIG.get("nautobot_ssot", {}) result = { - "instance": settings_config.get("instance", db_config.servicenow_instance), - "username": settings_config.get("username", ""), - "password": settings_config.get("password", ""), + "instance": settings_config.get("servicenow_instance", db_config.servicenow_instance), + "username": settings_config.get("servicenow_username", ""), + "password": settings_config.get("servicenow_password", ""), } if not result["username"]: try: diff --git a/nautobot_ssot/tests/servicenow/test_jobs.py b/nautobot_ssot/tests/servicenow/test_jobs.py index c73d518c..e03aa56c 100644 --- a/nautobot_ssot/tests/servicenow/test_jobs.py +++ b/nautobot_ssot/tests/servicenow/test_jobs.py @@ -59,7 +59,13 @@ def test_data_mappings(self): self.assertIsNone(mappings[5].target_url) @override_settings( - PLUGINS_CONFIG={"nautobot_ssot_servicenow": {"instance": "dev12345", "username": "admin", "password": ""}} + PLUGINS_CONFIG={ + "nautobot_ssot": { + "servicenow_instance": "dev12345", + "servicenow_username": "admin", + "servicenow_password": "", + } + } ) def test_config_information_settings(self): """Verify the config_information() API for configs provided in Django settings.""" From d2e53a6a9449cf578c4a05b704d4b5cead5f6f58 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 14:09:31 +0000 Subject: [PATCH 04/11] cleanup: Removed obsolete code --- .../tests/servicenow/test_adapter_nautobot.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py index 9ee3a286..eb54e832 100644 --- a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py +++ b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py @@ -1,9 +1,7 @@ """Unit tests for the Nautobot DiffSync adapter.""" -from unittest import mock import uuid -from django.conf import settings from django.contrib.contenttypes.models import ContentType from nautobot.dcim.models import Device, DeviceRole, DeviceType, Interface, Manufacturer, Region, Site @@ -14,11 +12,6 @@ from nautobot_ssot.integrations.servicenow.diffsync.adapter_nautobot import NautobotDiffSync -if "job_logs" in settings.DATABASES: - settings.DATABASES["job_logs"] = settings.DATABASES["job_logs"].copy() - settings.DATABASES["job_logs"]["TEST"] = {"MIRROR": "default"} - - class NautobotDiffSyncTestCase(TransactionTestCase): """Test the NautobotDiffSync adapter class.""" @@ -51,10 +44,6 @@ def setUp(self): Interface.objects.create(device=device_2, name="eth1") Interface.objects.create(device=device_2, name="eth2") - # Override the JOB_LOGS to None so that the Log Objects are created in the default database. - # This change is required as JOB_LOGS is a `fake` database pointed at the default. The django - # database cleanup will fail and cause tests to fail as this is not a real database. - @mock.patch("nautobot.extras.models.models.JOB_LOGS", None) def test_data_loading(self): """Test the load() function.""" job = ServiceNowDataTarget() From 51e8534a2c5d385e423f39d0eabdeee9a3aee1ee Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 14:46:39 +0000 Subject: [PATCH 05/11] fix: Better explain conflict between configs --- nautobot_ssot/templates/nautobot_ssot_servicenow/config.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html index d0637eb4..9227399b 100644 --- a/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html +++ b/nautobot_ssot/templates/nautobot_ssot_servicenow/config.html @@ -5,8 +5,9 @@ {% endif %} {% endblock header %} From 9fa4e0100e3d61f90094ad5dbd5342829b9496f5 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 14:57:12 +0000 Subject: [PATCH 06/11] fix: Create interfaces with status active in tests --- nautobot_ssot/tests/servicenow/test_adapter_nautobot.py | 8 ++++---- nautobot_ssot/tests/servicenow/test_jobs.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py index eb54e832..5d933dcc 100644 --- a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py +++ b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py @@ -39,10 +39,10 @@ def setUp(self): name="csr2", device_type=device_type, device_role=device_role, site=site_2, status=status_active ) - Interface.objects.create(device=device_1, name="eth1") - Interface.objects.create(device=device_1, name="eth2") - Interface.objects.create(device=device_2, name="eth1") - Interface.objects.create(device=device_2, name="eth2") + Interface.objects.create(device=device_1, name="eth1", status=status_active) + Interface.objects.create(device=device_1, name="eth2", status=status_active) + Interface.objects.create(device=device_2, name="eth1", status=status_active) + Interface.objects.create(device=device_2, name="eth2", status=status_active) def test_data_loading(self): """Test the load() function.""" diff --git a/nautobot_ssot/tests/servicenow/test_jobs.py b/nautobot_ssot/tests/servicenow/test_jobs.py index e03aa56c..6cb8c042 100644 --- a/nautobot_ssot/tests/servicenow/test_jobs.py +++ b/nautobot_ssot/tests/servicenow/test_jobs.py @@ -127,6 +127,7 @@ def test_config_information_db(self): def test_lookup_object(self): """Validate the lookup_object() API.""" + status_active = Status.objects.get(slug="active") region = Region.objects.create(name="My Region", slug="my-region") site = Site.objects.create(name="My Site", slug="my-site", status=Status.objects.get(slug="active")) manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") @@ -139,7 +140,7 @@ def test_lookup_object(self): site=site, status=Status.objects.get(slug="active"), ) - interface = Interface.objects.create(device=device, name="eth0") + interface = Interface.objects.create(device=device, name="eth0", status=status_active) job = ServiceNowDataTarget() From 26b5681b0b9e0013611a68aadd5224aa200d9643 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 15:03:58 +0000 Subject: [PATCH 07/11] doc: README for pysnow --- .../integrations/servicenow/third_party/pysnow/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 nautobot_ssot/integrations/servicenow/third_party/pysnow/README.md diff --git a/nautobot_ssot/integrations/servicenow/third_party/pysnow/README.md b/nautobot_ssot/integrations/servicenow/third_party/pysnow/README.md new file mode 100644 index 00000000..4eaca884 --- /dev/null +++ b/nautobot_ssot/integrations/servicenow/third_party/pysnow/README.md @@ -0,0 +1,8 @@ +# Copy of `pysnow` + +`pysnow` is currently pinned to an older version of `pytz` as a dependency, which blocks compatibility with newer +versions of Nautobot. [See](https://github.com/rbw/pysnow/pull/186). + +For now, we have embedded a copy of `pysnow` under `nautobot_ssot/integrations/servicenow/third_party/pysnow`. + +All `pysnow` dependencies are defined as an extra dependency in `pyproject.toml` From 74cb38f8e9cb753461c9f0fc5f0b8f5598017296 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 15:25:00 +0000 Subject: [PATCH 08/11] fix: Do not block future Nautobot dependencies updates --- pyproject.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 01cd09bb..8fb596f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,20 +28,20 @@ packages = [ [tool.poetry.dependencies] Jinja2 = { version = ">=2.11.3", optional = true } Markdown = "!=3.3.5" -PyYAML = { version = "^6", optional = true } +PyYAML = { version = ">=6", optional = true } diffsync = "^1.6.0" -dnspython = { version = "^2.1.0", optional = true } -ijson = { version = "^2.5.1", optional = true } +dnspython = { version = ">=2.1.0", optional = true } +ijson = { version = ">=2.5.1", optional = true } nautobot = "^1.4.0" -oauthlib = { version = "^3.1.0", optional = true } packaging = ">=21.3, <24" +oauthlib = { version = ">=3.1.0", optional = true } prometheus-client = "~0.14.1" python = "^3.7" -python-magic = { version = "^0.4.15", optional = true } +python-magic = { version = ">=0.4.15", optional = true } pytz = { version = ">=2019.3", optional = true } -requests = { version = "^2.21.0", optional = true } -requests-oauthlib = { version = "^1.3.0", optional = true } -six = { version = "^1.13.0", optional = true } +requests = { version = ">=2.21.0", optional = true } +requests-oauthlib = { version = ">=1.3.0", optional = true } +six = { version = ">=1.13.0", optional = true } [tool.poetry.dev-dependencies] bandit = "*" From 46ad0c3e074d9672b01438ebbc9a06b7c922fe59 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 15:25:45 +0000 Subject: [PATCH 09/11] cleanup: Unnecessary 1.2 dependency --- nautobot_ssot/jobs/base.py | 7 +------ pyproject.toml | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/nautobot_ssot/jobs/base.py b/nautobot_ssot/jobs/base.py index 4c0dbd1d..1b00b190 100644 --- a/nautobot_ssot/jobs/base.py +++ b/nautobot_ssot/jobs/base.py @@ -4,7 +4,6 @@ import traceback import tracemalloc from typing import Iterable -from packaging.version import Version from django.forms import HiddenInput from django.templatetags.static import static @@ -17,7 +16,6 @@ from diffsync.enum import DiffSyncFlags import structlog -from nautobot import __version__ as nautobot_version from nautobot.extras.jobs import BaseJob, BooleanVar from nautobot_ssot.choices import SyncLogEntryActionChoices @@ -273,10 +271,7 @@ def __init__(self): def as_form(self, data=None, files=None, initial=None, approval_view=False): """Render this instance as a Django form for user inputs, including a "Dry run" field.""" - if Version(nautobot_version) < Version("1.2"): - form = super().as_form(data=data, files=files, initial=initial) - else: - form = super().as_form(data=data, files=files, initial=initial, approval_view=approval_view) + form = super().as_form(data=data, files=files, initial=initial, approval_view=approval_view) # Set the "dry_run" widget's initial value based on our Meta attribute, if any form.fields["dry_run"].initial = getattr(self.Meta, "dry_run_default", True) # Hide the "commit" widget to reduce user confusion diff --git a/pyproject.toml b/pyproject.toml index 8fb596f0..6ed1a92f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ diffsync = "^1.6.0" dnspython = { version = ">=2.1.0", optional = true } ijson = { version = ">=2.5.1", optional = true } nautobot = "^1.4.0" -packaging = ">=21.3, <24" oauthlib = { version = ">=3.1.0", optional = true } prometheus-client = "~0.14.1" python = "^3.7" From 74b43c0555070ae71b0df7e608264c7cc826989c Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 21 Jun 2023 15:28:45 +0000 Subject: [PATCH 10/11] chore: Rebuild poetry.lock --- poetry.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3a30b1b2..3ef2c345 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3141,7 +3141,8 @@ files = [ {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win32.whl", hash = "sha256:763d65baa3b952479c4e972669f679fe490eee058d5aa85da483ebae2009d231"}, {file = "ruamel.yaml.clib-0.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:d000f258cf42fec2b1bbf2863c61d7b8918d31ffee905da62dede869254d3b8a"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:045e0626baf1c52e5527bd5db361bc83180faaba2ff586e763d3d5982a876a9e"}, - {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_12_6_arm64.whl", hash = "sha256:721bc4ba4525f53f6a611ec0967bdcee61b31df5a56801281027a3a6d1c2daf5"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:1a6391a7cabb7641c32517539ca42cf84b87b667bad38b78d4d42dd23e957c81"}, + {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:9c7617df90c1365638916b98cdd9be833d31d337dbcd722485597b43c4a215bf"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:41d0f1fa4c6830176eef5b276af04c89320ea616655d01327d5ce65e50575c94"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win32.whl", hash = "sha256:f6d3d39611ac2e4f62c3128a9eed45f19a6608670c5a2f4f07f24e8de3441d38"}, {file = "ruamel.yaml.clib-0.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:da538167284de58a52109a9b89b8f6a53ff8437dd6dc26d33b57bf6699153122"}, @@ -3672,4 +3673,4 @@ servicenow = ["Jinja2", "PyYAML", "ijson", "oauthlib", "python-magic", "pytz", " [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "c7d3a8e4085754b32e13b58a06363e19744c3447a4035af2da5ee9537a5ee0b5" +content-hash = "f453242c76e3eaf64ad7ca95dff8b832436eef5007a5841c3e05e537057c3fd6" From fd26de49203c2c0b706753332609e1e04893e2d1 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Tue, 11 Jul 2023 08:39:35 +0000 Subject: [PATCH 11/11] fix: Tests --- nautobot_ssot/tests/servicenow/test_adapter_nautobot.py | 4 ++-- nautobot_ssot/tests/servicenow/test_jobs.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py index 5d933dcc..982f42d3 100644 --- a/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py +++ b/nautobot_ssot/tests/servicenow/test_adapter_nautobot.py @@ -19,7 +19,7 @@ class NautobotDiffSyncTestCase(TransactionTestCase): def setUp(self): """Per-test-case data setup.""" - status_active = Status.objects.get(slug="active") + status_active, _ = Status.objects.get_or_create(name="Active", slug="active") region_1 = Region.objects.create(name="Region 1", slug="region-1") region_2 = Region.objects.create(name="Region 2", slug="region-2", parent=region_1) @@ -28,7 +28,7 @@ def setUp(self): site_1 = Site.objects.create(region=region_2, name="Site 1", slug="site-1", status=status_active) site_2 = Site.objects.create(region=region_3, name="Site/Region", slug="site-region", status=status_active) - manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") + manufacturer, _ = Manufacturer.objects.get_or_create(name="Cisco", slug="cisco") device_type = DeviceType.objects.create(manufacturer=manufacturer, model="CSR 1000v", slug="csr1000v") device_role = DeviceRole.objects.create(name="Router", slug="router") diff --git a/nautobot_ssot/tests/servicenow/test_jobs.py b/nautobot_ssot/tests/servicenow/test_jobs.py index 6cb8c042..f343171a 100644 --- a/nautobot_ssot/tests/servicenow/test_jobs.py +++ b/nautobot_ssot/tests/servicenow/test_jobs.py @@ -130,7 +130,7 @@ def test_lookup_object(self): status_active = Status.objects.get(slug="active") region = Region.objects.create(name="My Region", slug="my-region") site = Site.objects.create(name="My Site", slug="my-site", status=Status.objects.get(slug="active")) - manufacturer = Manufacturer.objects.create(name="Cisco", slug="cisco") + manufacturer, _ = Manufacturer.objects.get_or_create(name="Cisco", slug="cisco") device_type = DeviceType.objects.create(manufacturer=manufacturer, model="CSR 1000v", slug="csr1000v") device_role = DeviceRole.objects.create(name="Router", slug="router") device = Device.objects.create(