add option to auto-create catch-all netbox IPAM prefixes for private …

…IP space (idaholab#279)

Dockerfiles/netbox.Dockerfile

@@ -39,13 +39,15 @@ ARG NETBOX_DEVICETYPE_LIBRARY_PATH="/opt/netbox-devicetype-library"
 ARG NETBOX_DEFAULT_SITE=Malcolm
 ARG NETBOX_CRON=true
 ARG NETBOX_PRELOAD_PATH="/opt/netbox-preload"
+ARG NETBOX_PRELOAD_PREFIXES=false
 
 ENV NETBOX_PATH /opt/netbox
 ENV BASE_PATH netbox
 ENV NETBOX_DEVICETYPE_LIBRARY_PATH $NETBOX_DEVICETYPE_LIBRARY_PATH
 ENV NETBOX_DEFAULT_SITE $NETBOX_DEFAULT_SITE
 ENV NETBOX_CRON $NETBOX_CRON
 ENV NETBOX_PRELOAD_PATH $NETBOX_PRELOAD_PATH
+ENV NETBOX_PRELOAD_PREFIXES $NETBOX_PRELOAD_PREFIXES
 
 ADD netbox/patch/* /tmp/netbox-patches/
 

config/netbox-common.env.example

@@ -3,6 +3,8 @@
 # The name of the default "site" to be created upon NetBox initialization, and to be queried
 #   for enrichment (see LOGSTASH_NETBOX_ENRICHMENT)
 NETBOX_DEFAULT_SITE=Malcolm
+# Whether or not to create catch-all VRFs/IP Prefixes for private IP space
+NETBOX_PRELOAD_PREFIXES=false
 # Whether to disable Malcolm's NetBox instance ('true') or not ('false')
 NETBOX_DISABLED=true
 NETBOX_POSTGRES_DISABLED=true

docs/kubernetes.md

@@ -430,6 +430,8 @@ Should Malcolm automatically populate NetBox inventory based on observed network
 
 Specify default NetBox site name: Malcolm
 
+Should Malcolm create "catch-all" prefixes for private IP address space? (y / N): n
+
 Enable dark mode for OpenSearch Dashboards? (Y / n): y
 
 Malcolm has been installed to /home/user/Malcolm. See README.md for more information.

docs/malcolm-hedgehog-e2e-iso-install.md

@@ -255,6 +255,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
     - Answer **Y** to [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic. Autopopulation is **not** recommended: [manual inventory population](asset-interaction-analysis.md#NetBoxPopManual) is the preferred method to create an accurate representation of the intended network design.
 * **Specify default NetBox site name**
     - NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/); this default site name will be used as a query parameter for these enrichment lookups.
+* **Should Malcolm create "catch-all" prefixes for private IP address space?**
+    - Answer **Y** to automatically create "catch-all" NetBox prefixes for private IP address space (i.e., one each for `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`, respectively). This is not recommended for networks with more than one subnet.
 * **Should Malcolm capture live network traffic?**
     - Malcolm itself can perform [live analysis](live-analysis.md#LocalPCAP) of traffic it sees on another network interface (ideally not the same one used for its management). Answer **no** to this question in installations where Hedgehog Linux will be handling all network traffic capture. If users want Malcolm to observe and capture traffic instead of, or in addition to, a sensor running Hedgehog Linux, they should answer **yes** enable life traffic analysis using default settings, or select **customize** to proceed to answer the following related questions individually.
     - **Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?**

netbox/preload/prefixes_defaults.yml

@@ -0,0 +1,6 @@
+- prefix: 10.0.0.0/8
+  vrf: Private IP Space (10.0.0.0/8)
+- prefix: 172.16.0.0/12
+  vrf: Private IP Space (172.16.0.0/12)
+- prefix: 10.0.0.0/8
+  vrf: Private IP Space (192.168.0.0/16)

netbox/preload/vrfs_defaults.yml

@@ -0,0 +1,6 @@
+- enforce_unique: true
+  name: Private IP Space (10.0.0.0/8)
+- enforce_unique: true
+  name: Private IP Space (172.16.0.0/12)
+- enforce_unique: true
+  name: Private IP Space (192.168.0.0/16)

netbox/scripts/netbox_init.py

@@ -12,11 +12,14 @@
 import pynetbox
 import randomcolor
 import re
+import shutil
 import sys
+import tempfile
 import time
 import malcolm_utils
 
 from collections.abc import Iterable
+from distutils.dir_util import copy_tree
 from datetime import datetime
 from slugify import slugify
 from netbox_library_import import import_library
@@ -238,6 +241,16 @@ def main():
         required=False,
         help="Directory containing netbox-initializers files to preload",
     )
+    parser.add_argument(
+        '--preload-prefixes',
+        dest='preloadPrefixes',
+        type=malcolm_utils.str2bool,
+        metavar="true|false",
+        nargs='?',
+        const=True,
+        default=malcolm_utils.str2bool(os.getenv('NETBOX_PRELOAD_PREFIXES', default='False')),
+        help="Preload IPAM VRFs/IP Prefixes for private IP space",
+    )
     try:
         parser.error = parser.exit
         args = parser.parse_args()
@@ -642,20 +655,35 @@ def main():
     if os.path.isfile(netboxVenvPy) and os.path.isfile(manageScript) and os.path.isdir(args.preloadDir):
         try:
             with malcolm_utils.pushd(os.path.dirname(manageScript)):
-                retcode, output = malcolm_utils.run_process(
-                    [
-                        netboxVenvPy,
-                        os.path.basename(manageScript),
-                        "load_initializer_data",
-                        "--path",
-                        args.preloadDir,
-                    ],
-                    logger=logging,
-                )
-                if retcode == 0:
-                    logging.debug(f"netbox-initializers: {retcode} {output}")
-                else:
-                    logging.error(f"Error processing netbox-initializers: {retcode} {output}")
+                # make a local copy of the YMLs to preload
+                with tempfile.TemporaryDirectory() as tmpPreloadDir:
+                    copy_tree(args.preloadDir, tmpPreloadDir)
+
+                    # only preload catch-all VRFs and IP Prefixes if explicitly specified and they don't already exist
+                    if args.preloadPrefixes:
+                        for loadType in ('vrfs', 'prefixes'):
+                            defaultFileName = os.path.join(tmpPreloadDir, f'{loadType}_defaults.yml')
+                            loadFileName = os.path.join(tmpPreloadDir, f'{loadType}.yml')
+                            if os.path.isfile(defaultFileName) and (not os.path.isfile(loadFileName)):
+                                try:
+                                    shutil.copyfile(defaultFileName, loadFileName)
+                                except Exception:
+                                    pass
+
+                    retcode, output = malcolm_utils.run_process(
+                        [
+                            netboxVenvPy,
+                            os.path.basename(manageScript),
+                            "load_initializer_data",
+                            "--path",
+                            tmpPreloadDir,
+                        ],
+                        logger=logging,
+                    )
+                    if retcode == 0:
+                        logging.debug(f"netbox-initializers: {retcode} {output}")
+                    else:
+                        logging.error(f"Error processing netbox-initializers: {retcode} {output}")
 
         except Exception as e:
             logging.error(f"{type(e).__name__} processing netbox-initializers: {e}")

netbox/supervisord.conf

@@ -39,6 +39,8 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py
   --token "%(ENV_SUPERUSER_API_TOKEN)s"
   --net-map /usr/local/share/net-map.json
   --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_PATH)s"
+  --preload "%(ENV_NETBOX_PRELOAD_PATH)s"
+  --preload-prefixes %(ENV_NETBOX_PRELOAD_PREFIXES)s
 autostart=true
 autorestart=false
 startsecs=0

scripts/install.py

@@ -1263,6 +1263,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
         )
         if len(netboxSiteName) == 0:
             netboxSiteName = 'Malcolm'
+        netboxPreloadPrefixes = netboxEnabled and InstallerYesOrNo(
+            'Should Malcolm create "catch-all" prefixes for private IP address space?',
+            default=args.netboxPreloadPrefixes,
+        )
 
         # input packet capture parameters
         pcapNetSniff = False
@@ -1511,6 +1515,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 'NETBOX_DISABLED',
                 TrueOrFalseNoQuote(not netboxEnabled),
             ),
+            EnvValue(
+                os.path.join(args.configDir, 'netbox-common.env'),
+                'NETBOX_PRELOAD_PREFIXES',
+                TrueOrFalseNoQuote(netboxPreloadPrefixes),
+            ),
             # enable/disable netbox (postgres)
             EnvValue(
                 os.path.join(args.configDir, 'netbox-common.env'),
@@ -3671,6 +3680,16 @@ def main():
         default=False,
         help="Automatically populate NetBox inventory based on observed network traffic",
     )
+    netboxArgGroup.add_argument(
+        '--netbox-preload-prefixes',
+        dest='netboxPreloadPrefixes',
+        type=str2bool,
+        metavar="true|false",
+        nargs='?',
+        const=True,
+        default=False,
+        help="Preload NetBox IPAM VRFs/IP Prefixes for private IP space",
+    )
     netboxArgGroup.add_argument(
         '--netbox-site-name',
         dest='netboxSiteName',