Merge branch 'develop' into issue_2012_add_huge_tree_junos

.dockerignore

@@ -0,0 +1,19 @@
+.git*
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+.tox
+.venv
+.coverage
+.coverage.*
+.cache
+.mypy_cache
+.pytest_cache
+docs
+test
+static
+vagrant
+LICENSE
+report.json

.github/workflows/commit.yaml

@@ -14,10 +14,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -50,14 +50,14 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.11]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

Dockerfile

@@ -1,10 +1,16 @@
-FROM python:3.6-slim-stretch
+# syntax=docker/dockerfile:1.4
 
+FROM scratch AS installer
 COPY ./ /var/cache/napalm/
 
-RUN apt-get update \
- && apt-get install -y python-dev python-cffi libxslt1-dev libssl-dev libffi-dev \
- && apt-get autoremove \
- && rm -rf /var/lib/apt/lists/* \
- && pip --no-cache-dir install -U cffi cryptography /var/cache/napalm/ \
- && rm -rf /var/cache/napalm/
+FROM python:3.12-slim-bookworm
+
+RUN --mount=type=bind,from=installer,source=/var/cache/napalm,target=/var/cache/napalm,rw \
+    apt-get update && \
+    apt-get install -y \
+        python3-dev libxslt1-dev libssl-dev libffi-dev && \
+    apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip --no-cache-dir install -U cffi cryptography /var/cache/napalm/
+
+ ENTRYPOINT ["napalm"]

docs/requirements.txt

@@ -1,9 +1,8 @@
 urllib3==2.2.1 # https://github.com/readthedocs/readthedocs.org/issues/10290
-sphinx==1.8.6
-sphinx-rtd-theme==1.2.0
-sphinxcontrib-napoleon==0.7
+sphinx==7.2.6
+sphinx-rtd-theme==2.0.0
 invoke==2.2.0
-jinja2==2.11.3
-MarkupSafe==2.0.1
-pytest==7.2.2
-ansible==4.10.0
+jinja2==3.1.3
+MarkupSafe==2.1.5
+pytest==7.3.1
+ansible==9.4.0

napalm/base/base.py

@@ -44,7 +44,7 @@ def __init__(
         username: str,
         password: str,
         timeout: int = 60,
-        optional_args: Dict = None,
+        optional_args: Optional[Dict] = None,
     ) -> None:
         """
         This is the base class you have to inherit from when writing your own Network Driver to

napalm/junos/junos.py

@@ -1249,6 +1249,8 @@ def build_prefix_limit(**args):
         }
         _GROUP_FIELDS_DATATYPE_MAP_.update(_COMMON_FIELDS_DATATYPE_)
 
+        _UNWANTED_GROUP_FIELDS = ["multihop", "cluster"]
+
         _DATATYPE_DEFAULT_ = {str: "", int: 0, bool: False, list: []}
 
         bgp_config = {}
@@ -1305,6 +1307,8 @@ def build_prefix_limit(**args):
             is_nhs, boolean = is_nhs_list[0]
             nhs_policies[policy_name] = boolean if boolean is not None else False
 
+        unwanted_group_fields = dict()
+
         for bgp_group in bgp_items:
             bgp_group_name = bgp_group[0]
             bgp_group_details = bgp_group[1]
@@ -1317,6 +1321,9 @@ def build_prefix_limit(**args):
             # Always overwrite with the system local_as (this will either be
             # valid or will be zero i.e. the same as the default value).
             bgp_config[bgp_group_name]["local_as"] = system_bgp_asn
+            unwanted_group_fields[bgp_group_name] = dict(
+                {key: False for key in _UNWANTED_GROUP_FIELDS}
+            )
 
             for key, value in bgp_group_details:
                 if "_prefix_limit" in key or value is None:
@@ -1338,6 +1345,10 @@ def build_prefix_limit(**args):
                 if key == "neighbors":
                     bgp_group_peers = value
                     continue
+
+                if key in _UNWANTED_GROUP_FIELDS:
+                    unwanted_group_fields[bgp_group_name][key] = True
+                    continue
                 if datatype:
                     bgp_config[bgp_group_name].update(
                         {key: napalm.base.helpers.convert(datatype, value, default)}
@@ -1357,9 +1368,7 @@ def build_prefix_limit(**args):
             bgp_config[bgp_group_name]["prefix_limit"] = build_prefix_limit(
                 **prefix_limit_fields
             )
-            if "multihop" in bgp_config[bgp_group_name].keys():
-                # Delete 'multihop' key from the output
-                del bgp_config[bgp_group_name]["multihop"]
+            if unwanted_group_fields[bgp_group_name]["multihop"]:
                 if bgp_config[bgp_group_name]["multihop_ttl"] == 0:
                     # Set ttl to default value 64
                     bgp_config[bgp_group_name]["multihop_ttl"] = 64
@@ -1414,7 +1423,7 @@ def build_prefix_limit(**args):
                         # we do not want cluster in the output
                         del bgp_peer_details["cluster"]
 
-                if "cluster" in bgp_config[bgp_group_name].keys():
+                if unwanted_group_fields[bgp_group_name]["cluster"]:
                     bgp_peer_details["route_reflector_client"] = True
                 prefix_limit_fields = {}
                 for key, value in bgp_group_details:
@@ -1437,10 +1446,6 @@ def build_prefix_limit(**args):
                 if neighbor and bgp_peer_address == neighbor_ip:
                     break  # found the desired neighbor
 
-            if "cluster" in bgp_config[bgp_group_name].keys():
-                # we do not want cluster in the output
-                del bgp_config[bgp_group_name]["cluster"]
-
         return bgp_config
 
     def get_bgp_neighbors_detail(self, neighbor_address=""):

napalm/nxos/nxos.py

@@ -120,7 +120,6 @@ def __init__(
         self.timeout = timeout
         self.replace = True
         self.loaded = False
-        self.changed = False
         self.merge_candidate = ""
         self.candidate_cfg = "candidate_config.txt"
         self.rollback_cfg = "rollback_config.txt"
@@ -191,19 +190,40 @@ def _send_command(
     ) -> Dict[str, Union[str, Dict[str, Any]]]:
         raise NotImplementedError
 
+    def _check_file_exists(self, cfg_file: str) -> bool:
+        """
+        Check that the file exists on remote device using full path.
+
+        cfg_file can be a full path, e.g.: bootflash:rollback_config.txt
+        or just a filename, e.g.: rollback_config.txt
+
+        For example
+        # dir rollback_config.txt
+            71803    Sep 06 14:13:33 2023  rollback_config.txt
+
+        Usage for bootflash://sup-local
+        6211682304 bytes used
+        110314684416 bytes free
+        116526366720 bytes total
+        """
+        cmd = f"dir {cfg_file}"
+        output = self._send_command(command=cmd, raw_text=True)
+        if "No such file or directory" in output:
+            return False
+        else:
+            return True
+
     def _commit_merge(self) -> None:
         try:
             output = self._send_config(self.merge_candidate)
             if output and "Invalid command" in output:
                 raise MergeConfigException("Error while applying config!")
         except Exception as e:
-            self.changed = True
             self.rollback()
             err_header = "Configuration merge failed; automatic rollback attempted"
             merge_error = "{0}:\n{1}".format(err_header, repr(str(e)))
             raise MergeConfigException(merge_error)
 
-        self.changed = True
         # clear the merge buffer
         self.merge_candidate = ""
 
@@ -897,8 +917,6 @@ def _load_cfg_from_checkpoint(self) -> None:
         except ConnectionError:
             # requests will raise an error with verbose warning output (don't fail on this).
             return
-        finally:
-            self.changed = True
 
         # For nx-api a list is returned so extract the result associated with the
         # 'rollback' command.
@@ -918,10 +936,11 @@ def _load_cfg_from_checkpoint(self) -> None:
 
     def rollback(self) -> None:
         assert isinstance(self.device, NXOSDevice)
-        if self.changed:
-            self.device.rollback(self.rollback_cfg)
-            self._copy_run_start()
-            self.changed = False
+        if not self._check_file_exists(cfg_file=self.rollback_cfg):
+            msg = f"Rollback file '{self.rollback_cfg}' does not exist on device."
+            raise ReplaceConfigException(msg)
+        self.device.rollback(self.rollback_cfg)
+        self._copy_run_start()
 
     def get_facts(self) -> models.FactsDict:
         facts: models.FactsDict = {}  # type: ignore

napalm/nxos_ssh/nxos_ssh.py

@@ -542,32 +542,31 @@ def _load_cfg_from_checkpoint(self):
             "no terminal dont-ask",
         ]
 
-        try:
-            rollback_result = self._send_command_list(
-                commands, expect_string=r"[#>]", read_timeout=90
-            )
-        finally:
-            self.changed = True
+        rollback_result = self._send_command_list(
+            commands, expect_string=r"[#>]", read_timeout=90
+        )
         msg = rollback_result
         if "Rollback failed." in msg:
             raise ReplaceConfigException(msg)
 
     def rollback(self):
-        if self.changed:
-            commands = [
-                "terminal dont-ask",
-                "rollback running-config file {}".format(self.rollback_cfg),
-                "no terminal dont-ask",
-            ]
-            result = self._send_command_list(
-                commands, expect_string=r"[#>]", read_timeout=90
-            )
-            if "completed" not in result.lower():
-                raise ReplaceConfigException(result)
-            # If hostname changes ensure Netmiko state is updated properly
-            self._netmiko_device.set_base_prompt()
-            self._copy_run_start()
-            self.changed = False
+        if not self._check_file_exists(self.rollback_cfg):
+            msg = f"Rollback file '{self.rollback_cfg}' does not exist on device."
+            raise ReplaceConfigException(msg)
+
+        commands = [
+            "terminal dont-ask",
+            "rollback running-config file {}".format(self.rollback_cfg),
+            "no terminal dont-ask",
+        ]
+        result = self._send_command_list(
+            commands, expect_string=r"[#>]", read_timeout=90
+        )
+        if "completed" not in result.lower():
+            raise ReplaceConfigException(result)
+        # If hostname changes ensure Netmiko state is updated properly
+        self._netmiko_device.set_base_prompt()
+        self._copy_run_start()
 
     def _apply_key_map(self, key_map, table):
         new_dict = {}

requirements-dev.txt

@@ -1,17 +1,17 @@
 black==24.3.0
 coveralls==3.3.1
-ddt==1.6.0
+ddt==1.7.2
 flake8-import-order==0.18.2
 pytest==7.3.1
-pytest-cov==4.1.0
+pytest-cov==5.0.0
 pytest-json-report==1.5.0
-pyflakes==3.0.1
+pyflakes==3.2.0
 pylama==8.4.1
-mock==5.0.2
-mypy==0.982
-types-PyYAML==6.0.12.10
-types-requests==2.31.0.20240311
-types-six==1.16.21.8
-types-setuptools==67.8.0.0
-ttp==0.9.4
-ttp_templates==0.3.5
+mock==5.1.0
+mypy==1.9.0
+types-PyYAML==6.0.12.20240311
+types-requests==2.31.0.20240403
+types-six==1.16.21.20240311
+types-setuptools==69.2.0.20240317
+ttp==0.9.5
+ttp_templates==0.3.6

setup.py

@@ -13,7 +13,7 @@
 
 setup(
     name="napalm",
-    version="4.1.0",
+    version="5.0.0",
     packages=find_packages(exclude=("test*",)),
     test_suite="test_base",
     author="David Barroso, Kirk Byers, Mircea Ulinic",