From 1dc49e03a6d01a4807636519429efe600f8535e0 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Mon, 19 Jun 2023 08:52:49 -0700 Subject: [PATCH 1/4] Azure monitor endpoint URL has changed format in v1.2.0 Unfortunately, older versions break with new format - so need a version-specific code branch. --- msticpy/data/drivers/azure_monitor_driver.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/msticpy/data/drivers/azure_monitor_driver.py b/msticpy/data/drivers/azure_monitor_driver.py index 9d9b4d620..b54281b7b 100644 --- a/msticpy/data/drivers/azure_monitor_driver.py +++ b/msticpy/data/drivers/azure_monitor_driver.py @@ -22,6 +22,7 @@ import pandas as pd from azure.core.exceptions import HttpResponseError from azure.core.pipeline.policies import UserAgentPolicy +from pkg_resources import parse_version from ..._version import VERSION from ...auth.azure_auth import AzureCloudConfig, az_connect @@ -41,12 +42,14 @@ logger = logging.getLogger(__name__) +# pylint: disable=ungrouped-imports try: from azure.monitor.query import ( LogsQueryClient, LogsQueryPartialResult, LogsQueryResult, ) + from azure.monitor.query import __version__ as az_monitor_version except ImportError as imp_err: raise MsticpyMissingDependencyError( "Cannot use this feature without Azure monitor client installed", @@ -149,9 +152,13 @@ def __init__(self, connection_str: Optional[str] = None, **kwargs): @property def url_endpoint(self) -> str: """Return the current URL endpoint for Azure Monitor.""" - return _LOGANALYTICS_URL_BY_CLOUD.get( + base_url = _LOGANALYTICS_URL_BY_CLOUD.get( AzureCloudConfig().cloud, _LOGANALYTICS_URL_BY_CLOUD["global"] ) + # post v1.1.0 of azure-monitor-query, the API version requires a 'v1' suffix + if parse_version(az_monitor_version) > parse_version("1.1.0"): + return f"{base_url}v1" + return base_url def connect(self, connection_str: Optional[str] = None, **kwargs): """ From ef9efae034b41f41d31bbed6166cb288df6b7628 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Tue, 20 Jun 2023 11:22:43 -0700 Subject: [PATCH 2/4] Bug for missing attribute in kusto_driver - due to code change in kql_driver.py. Added documentation of need for additional packages to DataProv-Kusto-New.rst and DataProv-MSSentinel-New.rst --- docs/source/data_acquisition/DataProv-Kusto-New.rst | 4 +++- docs/source/data_acquisition/DataProv-MSSentinel-New.rst | 4 +++- msticpy/data/drivers/kql_driver.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-Kusto-New.rst b/docs/source/data_acquisition/DataProv-Kusto-New.rst index 7712368e2..727a8c162 100644 --- a/docs/source/data_acquisition/DataProv-Kusto-New.rst +++ b/docs/source/data_acquisition/DataProv-Kusto-New.rst @@ -10,7 +10,9 @@ QueryProvider using the .. warning:: This provider currently in beta and is available for testing. It is available alongside the existing Kusto provider for you - to compare old and new. + to compare old and new. To use it you will need the ``azure-data-kusto`` + package installed. You can install this with ``pip install azure-data-kusto`` + or ``pip install msticpy[azure_query]``. If you are using the existing implementation, see :doc:`./DataProv-Kusto` Changes from the previous implementation diff --git a/docs/source/data_acquisition/DataProv-MSSentinel-New.rst b/docs/source/data_acquisition/DataProv-MSSentinel-New.rst index c7db7a53e..c0417cb02 100644 --- a/docs/source/data_acquisition/DataProv-MSSentinel-New.rst +++ b/docs/source/data_acquisition/DataProv-MSSentinel-New.rst @@ -9,7 +9,9 @@ the .. note:: This provider currently in beta and is available for testing. It is available alongside the existing Sentinel provider for you - to compare old and new. + to compare old and new. To use it you will need the ``azure-monitor-query`` + package installed. You can install this with ``pip install azure-monitor-query`` + or ``pip install msticpy[azure_query]``. If you are using the existing implementation, see :doc:`./DataProv-MSSentinel` Changes from the previous implementation diff --git a/msticpy/data/drivers/kql_driver.py b/msticpy/data/drivers/kql_driver.py index f2dc09f9a..9362dd50f 100644 --- a/msticpy/data/drivers/kql_driver.py +++ b/msticpy/data/drivers/kql_driver.py @@ -117,7 +117,7 @@ def __init__(self, connection_str: str = None, **kwargs): self._ip = get_ipython() self._debug = kwargs.get("debug", False) super().__init__(**kwargs) - + self.workspace_id: Optional[str] = None self.set_driver_property( DriverProps.FORMATTERS, {"datetime": self._format_datetime, "list": self._format_list}, @@ -437,6 +437,8 @@ def _get_kql_current_connection(): """Get the current connection Workspace ID from KQLMagic.""" connections = kql_exec("--conn") current_connection = [conn for conn in connections if conn.startswith(" * ")] + if not current_connection: + return "" return current_connection[0].strip(" * ").split("@")[0] def _set_kql_cloud(self): From 77374585e7529fe5ed417c5c431395d8f419ea68 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Tue, 20 Jun 2023 16:15:54 -0700 Subject: [PATCH 3/4] Changing ipwidgets requirement to <9.0.0 --- conda/conda-reqs.txt | 2 +- requirements-all.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/conda-reqs.txt b/conda/conda-reqs.txt index 8335879db..3cd11cc53 100644 --- a/conda/conda-reqs.txt +++ b/conda/conda-reqs.txt @@ -21,7 +21,7 @@ geoip2>=2.9.0 html5lib httpx==0.24.0 ipython>=7.23.1 -ipywidgets>=7.4.2, <8.0.0 +ipywidgets>=7.4.2, <9.0.0 keyring>=13.2.1 lxml>=4.6.5 matplotlib>=3.0.0 diff --git a/requirements-all.txt b/requirements-all.txt index 0a99fe62f..079a84505 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -25,7 +25,7 @@ httpx==0.24.0 html5lib ipython >= 7.1.1; python_version < "3.8" ipython >= 7.23.1; python_version >= "3.8" -ipywidgets>=7.4.2, <8.0.0 +ipywidgets>=7.4.2, <9.0.0 keyring>=13.2.1 KqlmagicCustom[jupyter-basic,auth_code_clipboard]>=0.1.114.post22 KqlmagicCustom[jupyter-extended]>=0.1.114.post22 diff --git a/requirements.txt b/requirements.txt index df9066a3a..107732a5f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ httpx==0.24.0 html5lib ipython >= 7.1.1; python_version < "3.8" ipython >= 7.23.1; python_version >= "3.8" -ipywidgets>=7.4.2, <8.0.0 +ipywidgets>=7.4.2, <9.0.0 KqlmagicCustom[jupyter-basic,auth_code_clipboard]>=0.1.114.post22 lxml>=4.6.5 msal>=1.12.0 From d1755c872b8a295626befeb0cc40b92564a6d983 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Wed, 21 Jun 2023 14:21:32 -0700 Subject: [PATCH 4/4] Update DataProv-Kusto-New.rst Fixing name of `azure-kusto-data` --- docs/source/data_acquisition/DataProv-Kusto-New.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-Kusto-New.rst b/docs/source/data_acquisition/DataProv-Kusto-New.rst index 727a8c162..8974e82b7 100644 --- a/docs/source/data_acquisition/DataProv-Kusto-New.rst +++ b/docs/source/data_acquisition/DataProv-Kusto-New.rst @@ -3,15 +3,15 @@ Azure Data Explorer/Kusto Provider - New Implementation This is a new implementation of the Azure Data Explorer/Kusto QueryProvider using the -`azure-data-kusto SDK `__ +`azure-kusto-data SDK `__ (the earlier implementation used `Kqlmagic `__). .. warning:: This provider currently in beta and is available for testing. It is available alongside the existing Kusto provider for you - to compare old and new. To use it you will need the ``azure-data-kusto`` - package installed. You can install this with ``pip install azure-data-kusto`` + to compare old and new. To use it you will need the ``azure-kusto-data`` + package installed. You can install this with ``pip install azure-kusto-data`` or ``pip install msticpy[azure_query]``. If you are using the existing implementation, see :doc:`./DataProv-Kusto` @@ -487,4 +487,4 @@ For examples of using the Kusto provider, see the samples `Kusto Analysis Notebook `__ and `Kusto Ingest Notebook `__ -:py:mod:`Kusto driver API documentation` \ No newline at end of file +:py:mod:`Kusto driver API documentation`