Fixes for MSTICPy 2.9 compatibility

.github/ISSUE_TEMPLATE/workflows/python-package.yml

@@ -0,0 +1,186 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MSTICNB CI build and check
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main, release/*]
+  schedule:
+    - cron: "0 0 * * 0,2,4"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      # Print out details about the run
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Dump job context
+        env:
+          JOB_CONTEXT: ${{ toJSON(job) }}
+        run: echo "$JOB_CONTEXT"
+      # end print details
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+            ${{ runner.os }}-pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools
+          if [ -f requirements.txt ]; then
+            python -m pip install -r requirements.txt
+          fi
+          python -m pip install -e .
+      - name: Install test dependencies
+        run: |
+          if [ -f requirements-dev.txt ]; then
+            python -m pip install -r requirements-dev.txt
+          else
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
+            python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
+            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
+          fi
+          python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
+      - name: Prepare test dummy data
+        run: |
+          mkdir ~/.msticpy
+          cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
+          touch ~/.msticpy/GeoLite2-City.mmdb
+
+      - name: Pytest
+        env:
+          MAXMIND_AUTH: DUMMY_KEY
+          IPSTACK_AUTH: DUMMY_KEY
+          MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
+          MSTICPY_BUILD_SOURCE: fork
+        run: |
+          pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticnb --cov-report=xml
+        if: ${{ always() }}
+      - name: Upload pytest test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-results-${{ matrix.python-version }}
+          path: junit/test-${{ matrix.python-version }}-results.xml
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
+
+  lint:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }}
+            ${{ runner.os }}-pip-lint
+            ${{ runner.os }}-pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools
+          if [ -f requirements.txt ]; then
+            python -m pip install -r requirements.txt;
+          fi
+          python -m pip install -e .
+      - name: Install test dependencies
+        run: |
+          if [ -f requirements-dev.txt ]; then
+            python -m pip install -r requirements-dev.txt
+          else
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
+            python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
+          fi
+      - name: black
+        run: |
+          black --diff --check --exclude venv msticnb
+        if: ${{ always() }}
+      - name: flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 msticnb --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+        if: ${{ always() }}
+      - name: pylint
+        run: |
+          pylint msticnb --disable=duplicate-code --disable=E1135,E1101,E1133
+        if: ${{ always() }}
+      - name: Cache/restore MyPy data
+        id: cache-mypy
+        uses: actions/cache@v3
+        with:
+          # MyPy cache files are stored in `~/.mypy_cache`
+          path: .mypy_cache
+          key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
+            ${{ runner.os }}-build-mypy-${{ github.ref }}
+            ${{ runner.os }}-build-mypy
+      - name: mypy
+        run: |
+          mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticnb
+        if: ${{ always() }}
+      - name: Upload mypy test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: Mypy results ${{ matrix.python-version }}
+          path: junit/mypy-test-${{ matrix.python-version }}-results.xml
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
+      - name: flake8
+        run: |
+          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+        if: ${{ always() }}
+      - name: pydocstyle
+        run: |
+          pydocstyle --convention=numpy msticnb
+        if: ${{ always() }}
+      - name: pyroma
+        run: |
+          pyroma --min 10 .
+        if: ${{ always() }}
+  check_status:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    needs: [build, lint]
+    steps:
+      - name: File build fail issue
+        if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }}
+        uses: dacbd/create-issue-action@v1
+        with:
+          token: ${{ github.token }}
+          title: "Build failed for main branch"
+          body: The build failed on branch ${{ github.ref }}. Please investigate
+          labels: build_break, bug, high_severity

.github/ISSUE_TEMPLATE/workflows/python-publish.yml

@@ -0,0 +1,45 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package to PyPI Prod
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+    permissions: read-all
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install build
+    - name: Build package
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@v1.5.1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

azure-pipelines.yml

@@ -48,7 +48,7 @@ stages:
       condition: succeededOrFailed()
       displayName: pytest
       env:
-        MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/testdata/msticpyconfig-test.yaml
+        MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml
         MAXMIND_AUTH: $(maxmind_auth)
         IPSTACK_AUTH: $(ipstack_auth)
         MSTICPY_TEST_NOSKIP: 1

docs/notebooks/NotebookletsDemo.rst

@@ -4533,8 +4533,8 @@ Three sections:
     @set_text(docs=_CELL_DOCS, key="display_event_pivot")
     def _display_event_pivot(event_pivot):
         display(
-            event_pivot.style.applymap(lambda x: "color: white" if x == 0 else "")
-            .applymap(
+            event_pivot.style.map(lambda x: "color: white" if x == 0 else "")
+            .map(
                 lambda x: "background-color: lightblue"
                 if not isinstance(x, str) and x > 0
                 else ""
@@ -4656,8 +4656,8 @@ Three sections:
     @set_text(docs=_CELL_DOCS, key="display_acct_event_pivot")
     def _display_acct_event_pivot(event_pivot_df):
         display(
-            event_pivot_df.style.applymap(lambda x: "color: white" if x == 0 else "")
-            .applymap(
+            event_pivot_df.style.map(lambda x: "color: white" if x == 0 else "")
+            .map(
                 lambda x: "background-color: lightblue"
                 if not isinstance(x, str) and x > 0
                 else ""

msticnb/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "1.1.1"
+VERSION = "1.2.0"

msticnb/data_providers.py

@@ -11,15 +11,16 @@
 
 from msticpy.common.exceptions import MsticpyAzureConfigError
 from msticpy.common.wsconfig import WorkspaceConfig
-from msticpy.data import QueryProvider
 
 try:
     from msticpy.context import GeoLiteLookup, IPStackLookup, TILookup
     from msticpy.context.azure.azure_data import AzureData
+    from msticpy.data.core.data_providers import QueryProvider
     from msticpy.data.core.query_defns import DataEnvironment
 except ImportError:
     # Fall back to msticpy locations prior to v2.0.0
     from msticpy.data.azure.azure_data import AzureData
+    from msticpy.data import QueryProvider
     from msticpy.data.query_defns import DataEnvironment
     from msticpy.sectools import GeoLiteLookup, IPStackLookup, TILookup
 

msticnb/nb/azsent/account/account_summary.py

@@ -76,6 +76,8 @@ def parse(cls, name: str):
 
 # pylint: enable=invalid-name
 
+# pylint: disable=no-member
+
 
 # pylint: disable=too-few-public-methods, too-many-instance-attributes
 class AccountSummaryResult(NotebookletResult):

msticnb/nb/azsent/alert/ti_enrich.py

@@ -195,7 +195,7 @@ def run(
                     ["StartTimeUtc", "AlertName", "Severity", "TI Risk", "Description"]
                 ]
                 .sort_values(by=["StartTimeUtc"])
-                .style.applymap(_color_cells)
+                .style.map(_color_cells)
                 .hide_index()
             )
         if "details" in self.options:
@@ -241,7 +241,7 @@ def show_full_alert(selected_alert):
                         ["Ioc", "IocType", "Provider", "Result", "Severity", "Details"]
                     ]
                     .reset_index()
-                    .style.applymap(_color_cells)
+                    .style.map(_color_cells)
                     .hide_index()
                 )
                 ti_ips = ti_data[ti_data["IocType"] == "ipv4"]

msticnb/nb/azsent/host/win_host_events.py

@@ -6,6 +6,7 @@
 """Notebooklet for Windows Security Events."""
 import os
 import pkgutil
+from io import StringIO
 from typing import Any, Dict, Iterable, Optional, Union
 
 import numpy as np
@@ -281,8 +282,8 @@ def _get_win_security_events(qry_prov, host_name, timespan):
 @set_text(docs=_CELL_DOCS, key="display_event_pivot")
 def _display_event_pivot(event_pivot):
     display(
-        event_pivot.style.applymap(lambda x: "color: white" if x == 0 else "")
-        .applymap(
+        event_pivot.style.map(lambda x: "color: white" if x == 0 else "")
+        .map(
             lambda x: "background-color: lightblue"
             if not isinstance(x, str) and x > 0
             else ""
@@ -364,7 +365,7 @@ def _extract_acct_mgmt_events(event_data):
 
     w_evt = pkgutil.get_data("msticpy", f"resources{os.sep}WinSecurityEvent.json")
 
-    win_event_df = pd.read_json(w_evt.decode("utf-8"))
+    win_event_df = pd.read_json(StringIO(w_evt.decode("utf-8")))
 
     # Create criteria for events that we're interested in
     acct_sel = win_event_df["subcategory"] == "User Account Management"
@@ -407,8 +408,8 @@ def _create_acct_event_pivot(account_event_data):
 @set_text(docs=_CELL_DOCS, key="display_acct_event_pivot")
 def _display_acct_event_pivot(event_pivot_df):
     display(
-        event_pivot_df.style.applymap(lambda x: "color: white" if x == 0 else "")
-        .applymap(
+        event_pivot_df.style.map(lambda x: "color: white" if x == 0 else "")
+        .map(
             lambda x: "background-color: lightblue"
             if not isinstance(x, str) and x > 0
             else ""

msticnb/nb/azsent/url/url_summary.py

@@ -6,7 +6,7 @@
 """Notebooklet for URL Summary."""
 from collections import Counter
 from os.path import exists
-from typing import Any, Dict, Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional, Tuple, cast
 
 import dns.resolver
 import numpy as np
@@ -161,7 +161,7 @@ def run(  # noqa:MC0001
             self._last_result = result
 
         self.url = value.strip().lower()
-        _, domain, tld = tldextract.extract(self.url)
+        _, domain, tld = cast(Tuple[Any, str, str], tldextract.extract(self.url))  # type: ignore
         domain = f"{domain.lower()}.{tld.lower()}"
         domain_validator = DomainValidator()
         validated = domain_validator.validate_tld(domain)
@@ -283,7 +283,7 @@ def _display_domain_record(self):
         """Display Domain Record."""
         if self.check_valid_result_data("domain_record", silent=True):
             display(
-                self._last_result.domain_record.T.style.applymap(  # type: ignore
+                self._last_result.domain_record.T.style.map(  # type: ignore
                     color_domain_record_cells,
                     subset=pd.IndexSlice[["Page Rank", "Domain Name Entropy"], 0],
                 )
@@ -484,6 +484,7 @@ def _domain_whois_record(domain, ti_prov):
 
         # Remove duplicate Name Server records
         for server in whois_result["name_servers"]:
+            # pylint: disable=unpacking-non-sequence
             _, ns_domain, ns_tld = tldextract.extract(server)
             ns_dom = ns_domain.lower() + "." + ns_tld.lower()
             if domain not in ns_domains:

msticnb/nb/template/nb_template.py

@@ -294,7 +294,7 @@ def _get_all_events(qry_prov, host_name, timespan):
     del qry_prov, host_name, timespan
     return pd.DataFrame(
         {
-            "TimeGenerated": pd.date_range("2022-01-01", periods=5, tz="utc", freq="H"),
+            "TimeGenerated": pd.date_range("2022-01-01", periods=5, tz="utc", freq="h"),
             "EventID": [4688, 4688, 4625, 4624, 4624],
             "Computer": ["MyHost.dom"] * 5,
             "Account": [f"user{n}" for n in range(5)],