Ianhelle/hotfix 2.5.2 2023 06 08 (#676)

* Changed Bokeh requirements to work with panel 1.x

Moved code from nbinit that checked CLI credentials to be run only in AML - avoiding non-core import.

* Adding back MicrosoftSentinel as mp attribute

Making the attribute failure more robust/informative.

* Adding documentation URL to ignored links
"https://username:password@proxy_host:port" appears in docstrings
and documentation and is being checked for URL validity.

* Checked in test version of readthedocs conf.py by mistake - reverting

conda/conda-reqs.txt

@@ -12,7 +12,7 @@ azure-mgmt-resource>=16.1.0
 azure-monitor-query>=1.0.0
 azure-storage-blob>=12.5.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <=2.4.3
+bokeh>=1.4.0, <4.0.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

msticpy/__init__.py

@@ -116,6 +116,7 @@
 """
 import importlib
 import os
+import traceback
 import warnings
 from typing import Any, Iterable, Union
 
@@ -125,7 +126,11 @@
 from ._version import VERSION
 from .common import pkg_config as settings
 from .common.check_version import check_version
-from .common.exceptions import MsticpyException
+from .common.exceptions import (
+    MsticpyException,
+    MsticpyImportExtraError,
+    MsticpyMissingDependencyError,
+)
 from .common.utility import search_name as search
 from .init.logging import set_logging_level, setup_logging
 
@@ -148,6 +153,7 @@
     "GeoLiteLookup": "msticpy.context.geoip",
     "init_notebook": "msticpy.init.nbinit",
     "reset_ipython_exception_handler": "msticpy.init.nbinit",
+    "MicrosoftSentinel": "msticpy.context.azure",
     "MpConfigEdit": "msticpy.config.mp_config_edit",
     "MpConfigFile": "msticpy.config.mp_config_file",
     "QueryProvider": "msticpy.data",
@@ -182,10 +188,22 @@ def __getattr__(attrib: str) -> Any:
     if attrib in _DEFAULT_IMPORTS:
         try:
             return getattr(importlib.import_module(_DEFAULT_IMPORTS[attrib]), attrib)
-        except (ImportError, MsticpyException):
-            warnings.warn("Unable to import msticpy.{attrib}", ImportWarning)
-            return None
-    raise AttributeError(f"msticpy has no attribute {attrib}")
+        except (MsticpyImportExtraError, MsticpyImportExtraError):
+            raise
+        except (ImportError, MsticpyException) as err:
+            warnings.warn(f"Unable to import module for 'msticpy.{attrib}'")
+            print(
+                f"WARNING. The msticpy attribute '{attrib}' is not loadable.",
+                "You may need to install one or more additional dependencies.\n",
+                "Please check the exception details below for more information.",
+                "\n".join(
+                    traceback.format_exception(
+                        type(err), value=err, tb=err.__traceback__
+                    )
+                ),
+            )
+            raise AttributeError(f"msticpy failed to load '{attrib}'") from err
+    raise AttributeError(f"msticpy has no attribute '{attrib}'")
 
 
 def __dir__():

msticpy/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "2.5.1"
+VERSION = "2.5.2"

msticpy/init/azure_ml_tools.py

@@ -22,7 +22,7 @@
 
 from .._version import VERSION
 from ..common.pkg_config import _HOME_PATH, refresh_config
-from ..common.utility import search_for_file
+from ..common.utility import search_for_file, unit_testing
 from ..config import MpConfigFile  # pylint: disable=no-name-in-module
 
 __version__ = VERSION
@@ -55,6 +55,17 @@
     Please restart the notebook kernel and re-run this cell - it should
     run without error.
     """
+_AZ_CLI_WIKI_URI = (
+    "https://github.com/Azure/Azure-Sentinel-Notebooks/wiki/"
+    "Caching-credentials-with-Azure-CLI"
+)
+_CLI_WIKI_MSSG_GEN = (
+    f"For more information see <a href='{_AZ_CLI_WIKI_URI}'>"
+    "Caching credentials with Azure CLI</>"
+)
+_CLI_WIKI_MSSG_SHORT = (
+    f"see <a href='{_AZ_CLI_WIKI_URI}'>Caching credentials with Azure CLI</>"
+)
 
 MIN_PYTHON_VER_DEF = "3.6"
 MSTICPY_REQ_VERSION = __version__
@@ -73,7 +84,7 @@ def is_in_aml():
     return os.environ.get("APPSETTING_WEBSITE_SITE_NAME") == "AMLComputeInstance"
 
 
-def check_versions(
+def check_aml_settings(
     min_py_ver: Union[str, Tuple] = MIN_PYTHON_VER_DEF,
     min_mp_ver: Union[str, Tuple] = MSTICPY_REQ_VERSION,
     extras: Optional[List[str]] = None,
@@ -104,7 +115,7 @@ def check_versions(
 
     """
     del kwargs
-    _disp_html("<h4>Starting notebook pre-checks...</h4>")
+    _disp_html("<h4>Starting AML notebook pre-checks...</h4>")
     if isinstance(min_py_ver, str):
         min_py_ver = _get_pkg_version(min_py_ver).release
     check_python_ver(min_py_ver=min_py_ver)
@@ -114,9 +125,14 @@ def check_versions(
     _set_kql_env_vars(extras)
     _run_user_settings()
     _set_mpconfig_var()
+    _check_azure_cli_status()
     _disp_html("<h4>Notebook pre-checks complete.</h4>")
 
 
+# retain previous name for backward compatibility
+check_versions = check_aml_settings
+
+
 def check_python_ver(min_py_ver: Union[str, Tuple] = MIN_PYTHON_VER_DEF):
     """
     Check the current version of the Python kernel.
@@ -447,3 +463,21 @@ def _check_kql_prereqs():
                 " Please run the following command:<br>"
                 "!conda install --yes -c conda-forge pygobject<br>"
             )
+
+
+def _check_azure_cli_status():
+    """Check for Azure CLI credentials."""
+    # import these only if we need them at runtime
+    # pylint: disable=import-outside-toplevel
+    from ..auth.azure_auth_core import AzureCliStatus, check_cli_credentials
+
+    if not unit_testing():
+        status, message = check_cli_credentials()
+        if status == AzureCliStatus.CLI_OK:
+            _disp_html(message)
+        elif status == AzureCliStatus.CLI_NOT_INSTALLED:
+            _disp_html(
+                "Azure CLI credentials not detected." f" ({_CLI_WIKI_MSSG_SHORT})"
+            )
+        elif message:
+            _disp_html("\n".join([message, _CLI_WIKI_MSSG_GEN]))

msticpy/init/nbinit.py

@@ -66,7 +66,6 @@
     sns = None
 
 from .._version import VERSION
-from ..auth.azure_auth_core import AzureCliStatus, check_cli_credentials
 from ..common.check_version import check_version
 from ..common.exceptions import MsticpyException, MsticpyUserError
 from ..common.pkg_config import _HOME_PATH
@@ -84,8 +83,7 @@
     search_for_file,
     unit_testing,
 )
-from .azure_ml_tools import check_versions as check_versions_aml
-from .azure_ml_tools import is_in_aml, populate_config_to_mp_config
+from .azure_ml_tools import check_aml_settings, is_in_aml, populate_config_to_mp_config
 from .azure_synapse_tools import init_synapse, is_in_synapse
 from .pivot import Pivot
 from .user_config import load_user_defaults
@@ -226,17 +224,6 @@ def _verbose(verbosity: Optional[int] = None) -> int:
     "Please run the <i>Getting Started Guide for Azure Sentinel "
     + "ML Notebooks</i> notebook."
 )
-_AZ_CLI_WIKI_URI = (
-    "https://github.com/Azure/Azure-Sentinel-Notebooks/wiki/"
-    "Caching-credentials-with-Azure-CLI"
-)
-_CLI_WIKI_MSSG_GEN = (
-    f"For more information see <a href='{_AZ_CLI_WIKI_URI}'>"
-    "Caching credentials with Azure CLI</>"
-)
-_CLI_WIKI_MSSG_SHORT = (
-    f"see <a href='{_AZ_CLI_WIKI_URI}'>Caching credentials with Azure CLI</>"
-)
 
 current_providers: Dict[str, Any] = {}  # pylint: disable=invalid-name
 
@@ -426,7 +413,7 @@ def init_notebook(
     logger.info("Starting Notebook initialization")
     # Check Azure ML environment
     if _detect_env("aml", **kwargs) and is_in_aml():
-        check_versions_aml(*_get_aml_globals(namespace))
+        check_aml_settings(*_get_aml_globals(namespace))
     else:
         # If not in AML check and print version status
         stdout_cap = io.StringIO()
@@ -459,7 +446,6 @@ def init_notebook(
     else:
         _pr_output("Checking configuration....")
         conf_ok = _get_or_create_config()
-        _check_azure_cli_status()
 
     # Notebook options
     _pr_output("Setting notebook options....")
@@ -897,17 +883,3 @@ def reset_ipython_exception_handler():
     """Remove MSTICPy custom exception handler."""
     if hasattr(InteractiveShell.showtraceback, "__wrapped__"):
         InteractiveShell.showtraceback = InteractiveShell.showtraceback.__wrapped__
-
-
-def _check_azure_cli_status():
-    """Check for Azure CLI credentials."""
-    if not unit_testing():
-        status, message = check_cli_credentials()
-        if status == AzureCliStatus.CLI_OK:
-            _pr_output(message)
-        elif status == AzureCliStatus.CLI_NOT_INSTALLED:
-            _pr_output(
-                "Azure CLI credentials not detected." f" ({_CLI_WIKI_MSSG_SHORT})"
-            )
-        elif message:
-            _pr_output("\n".join([message, _CLI_WIKI_MSSG_GEN]))

requirements-all.txt

@@ -15,7 +15,7 @@ azure-mgmt-subscription>=3.0.0
 azure-monitor-query>=1.0.0
 azure-storage-blob>=12.5.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <=2.4.3
+bokeh>=1.4.0, <4.0.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

requirements.txt

@@ -4,7 +4,7 @@ azure-core>=1.24.0
 azure-identity>=1.10.0
 azure-mgmt-subscription>=3.0.0
 beautifulsoup4>=4.0.0
-bokeh>=1.4.0, <=2.4.3
+bokeh>=1.4.0, <4.0.0
 cryptography>=3.1
 deprecated>=1.2.4
 dnspython>=2.0.0, <3.0.0

tests/ignored_uri_links.txt

@@ -8,4 +8,5 @@ https://api.xforce.ibmcloud.com/url/fkksjobnn43.org
 https://api.greynoise.io/v3/community/38.75.137.9
 https://management.azure.com/.default
 https://www.maxmind.com/en/accounts/current/license-key
-https://api.us2.sumologic.com/api
+https://api.us2.sumologic.com/api
+https://username:password@proxy_host:port

tests/init/test_azure_ml_tools.py

@@ -137,14 +137,14 @@ def test_check_versions(monkeypatch, aml_file_sys, check_vers):
     if check_vers.excep:
         with pytest.raises(check_vers.excep):
             with change_directory(str(user_dir)):
-                aml.check_versions(
+                aml.check_aml_settings(
                     min_py_ver=check_vers.py_req,
                     min_mp_ver=check_vers.mp_req,
                     extras=check_vers.extras,
                 )
     else:
         with change_directory(str(user_dir)):
-            aml.check_versions(
+            aml.check_aml_settings(
                 min_py_ver=check_vers.py_req,
                 min_mp_ver=check_vers.mp_req,
                 extras=check_vers.extras,
@@ -189,7 +189,7 @@ def test_check_versions_mpconfig(monkeypatch, aml_file_sys, test_case):
     _os.environ["APPSETTING_WEBSITE_SITE_NAME"] = "AMLComputeInstance"
 
     with change_directory(str(target_dir)):
-        aml.check_versions(min_py_ver=_MIN_PY_VER, min_mp_ver=_MIN_MP_VER)
+        aml.check_aml_settings(min_py_ver=_MIN_PY_VER, min_mp_ver=_MIN_MP_VER)
 
     if test_case.sub_dir and test_case.mpconf_exists:
         env = "MSTICPYCONFIG"
@@ -215,7 +215,7 @@ def test_check_versions_nbuser_settings(monkeypatch, aml_file_sys):
     nb_user_settings.write_text(_NBUSER_SETTINGS, encoding="utf-8")
 
     with change_directory(str(user_dir)):
-        aml.check_versions(min_py_ver=_MIN_PY_VER, min_mp_ver=_MIN_MP_VER)
+        aml.check_aml_settings(min_py_ver=_MIN_PY_VER, min_mp_ver=_MIN_MP_VER)
 
     check.is_in("nbuser_settings", sys.modules)
     nbus_import = sys.modules["nbuser_settings"]