From cefc4ee5a22285d33e7abd91371c617fe42f8129 Mon Sep 17 00:00:00 2001 From: Robert Jaakke <11254908+rjaakke@users.noreply.github.com> Date: Thu, 16 Mar 2023 21:14:27 +0100 Subject: [PATCH 01/13] [Fix] Added MANIFEST.in to include YAML files (#38) Include YAML files in msticnb folder recursive Co-authored-by: Robert Jaakke --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bfa0744 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include msticnb *.yaml \ No newline at end of file From 01cd60b77979f0e4c62c56529b121f4179c765d3 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Fri, 9 Feb 2024 17:06:48 -0800 Subject: [PATCH 02/13] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2805abd..c75a7ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ ipython>=7.23.1 ipywidgets>=7.5.1 lxml>=4.4.2 Markdown>=3.2.1 -msticpy[azure]==2.3.1 +msticpy[azure]>=2.3.1 numpy>=1.17.3 pandas>=0.25.3 python-dateutil>=2.8.1 From 1ac1ca0089704d53aff4237eb959508afbebb148 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Fri, 9 Feb 2024 17:07:47 -0800 Subject: [PATCH 03/13] Update _version.py --- msticnb/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msticnb/_version.py b/msticnb/_version.py index 0c8d6f3..f1ab77c 100644 --- a/msticnb/_version.py +++ b/msticnb/_version.py @@ -1,2 +1,2 @@ """Version file.""" -VERSION = "1.1.0" +VERSION = "1.1.1" From d4de8c87b49ee0c1426bf1db436c5fb403871fc5 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sat, 10 Feb 2024 12:20:54 -0800 Subject: [PATCH 04/13] Create python-package.yml --- .github/workflows/python-package.yml | 186 +++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..ec90dfa --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,186 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MSTICNB CI build and check + +on: + push: + branches: [main] + pull_request: + branches: [main, release/*] + schedule: + - cron: "0 0 * * 0,2,4" + +jobs: + build: + runs-on: ubuntu-latest + permissions: read-all + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + steps: + # Print out details about the run + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJSON(job) }} + run: echo "$JOB_CONTEXT" + # end print details + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + ${{ runner.os }}-pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + if [ -f requirements.txt ]; then + python -m pip install -r requirements-all.txt + fi + python -m pip install -e . + - name: Install test dependencies + run: | + if [ -f requirements-dev.txt ]; then + python -m pip install -r requirements-dev.txt + else + echo "Missing requirements-dev.txt. Installing minimal requirements for testing." + python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions + python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml + fi + python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" + - name: Prepare test dummy data + run: | + mkdir ~/.msticpy + cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy + touch ~/.msticpy/GeoLite2-City.mmdb + + - name: Pytest + env: + MAXMIND_AUTH: DUMMY_KEY + IPSTACK_AUTH: DUMMY_KEY + MSTICPYCONFIG: ./tests/testdata/msticpyconfig-test.yaml + MSTICPY_BUILD_SOURCE: fork + run: | + pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticnb --cov-report=xml + if: ${{ always() }} + - name: Upload pytest test results + uses: actions/upload-artifact@v3 + with: + name: pytest-results-${{ matrix.python-version }} + path: junit/test-${{ matrix.python-version }}-results.xml + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} + + lint: + runs-on: ubuntu-latest + permissions: read-all + strategy: + matrix: + python-version: ["3.8"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }} + ${{ runner.os }}-pip-lint + ${{ runner.os }}-pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + if [ -f requirements.txt ]; then + python -m pip install -r requirements.txt; + fi + python -m pip install -e . + - name: Install test dependencies + run: | + if [ -f requirements-dev.txt ]; then + python -m pip install -r requirements-dev.txt + else + echo "Missing requirements-dev.txt. Installing minimal requirements for testing." + python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma + fi + - name: black + run: | + black --diff --check --exclude venv msticnb + if: ${{ always() }} + - name: flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 msticnb --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto + if: ${{ always() }} + - name: pylint + run: | + pylint msticnb --disable=duplicate-code --disable=E1135,E1101,E1133 + if: ${{ always() }} + - name: Cache/restore MyPy data + id: cache-mypy + uses: actions/cache@v3 + with: + # MyPy cache files are stored in `~/.mypy_cache` + path: .mypy_cache + key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} + ${{ runner.os }}-build-mypy-${{ github.ref }} + ${{ runner.os }}-build-mypy + - name: mypy + run: | + mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticnb + if: ${{ always() }} + - name: Upload mypy test results + uses: actions/upload-artifact@v3 + with: + name: Mypy results ${{ matrix.python-version }} + path: junit/mypy-test-${{ matrix.python-version }}-results.xml + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} + - name: flake8 + run: | + flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto + if: ${{ always() }} + - name: pydocstyle + run: | + pydocstyle --convention=numpy msticnb + if: ${{ always() }} + - name: pyroma + run: | + pyroma --min 10 . + if: ${{ always() }} + check_status: + runs-on: ubuntu-latest + permissions: read-all + needs: [build, lint] + steps: + - name: File build fail issue + if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }} + uses: dacbd/create-issue-action@v1 + with: + token: ${{ github.token }} + title: "Build failed for main branch" + body: The build failed on branch ${{ github.ref }}. Please investigate + labels: build_break, bug, high_severity From 1e7efecf174bb774c3d6cfa0b1624781a437582a Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sat, 10 Feb 2024 12:22:08 -0800 Subject: [PATCH 05/13] Add files via upload --- .github/workflows/python-publish.yml | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..7faf233 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,45 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package to PyPI Prod + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + permissions: read-all + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build package + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + - name: Publish package + uses: pypa/gh-action-pypi-publish@v1.5.1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From f3e700a4593e08299d3fe53740231923b30d0a4d Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sat, 10 Feb 2024 13:37:29 -0800 Subject: [PATCH 06/13] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ec90dfa..278c563 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -48,7 +48,7 @@ jobs: run: | python -m pip install --upgrade pip wheel setuptools if [ -f requirements.txt ]; then - python -m pip install -r requirements-all.txt + python -m pip install -r requirements.txt fi python -m pip install -e . - name: Install test dependencies From 91966c338684bc5e73094ed0f6c57094583e82e0 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sat, 10 Feb 2024 13:45:06 -0800 Subject: [PATCH 07/13] Fixes for MSTICPy 2.9 compatibility (#41) * Linting fixes for MSTICPy 2.9 compatibility * Moving test msticpyconfig to root of tests * Typo in python-package.yml --- .../workflows/python-package.yml | 186 ++++++++++++++++++ .../workflows/python-publish.yml | 45 +++++ azure-pipelines.yml | 2 +- docs/notebooks/NotebookletsDemo.rst | 8 +- msticnb/_version.py | 2 +- msticnb/data_providers.py | 3 +- msticnb/nb/azsent/account/account_summary.py | 2 + msticnb/nb/azsent/alert/ti_enrich.py | 4 +- msticnb/nb/azsent/host/win_host_events.py | 11 +- msticnb/nb/azsent/url/url_summary.py | 7 +- msticnb/nb/template/nb_template.py | 2 +- msticnb/nblib/azsent/host.py | 2 +- msticnb/read_modules.py | 9 +- requirements-dev.txt | 21 ++ tests/{testdata => }/msticpyconfig-test.yaml | 0 tests/nb/azsent/host/test_hostlogonsummary.py | 1 + .../azsent/host/test_logon_session_rarity.py | 6 + 17 files changed, 288 insertions(+), 23 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/workflows/python-package.yml create mode 100644 .github/ISSUE_TEMPLATE/workflows/python-publish.yml create mode 100644 requirements-dev.txt rename tests/{testdata => }/msticpyconfig-test.yaml (100%) diff --git a/.github/ISSUE_TEMPLATE/workflows/python-package.yml b/.github/ISSUE_TEMPLATE/workflows/python-package.yml new file mode 100644 index 0000000..67a9cec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/workflows/python-package.yml @@ -0,0 +1,186 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: MSTICNB CI build and check + +on: + push: + branches: [main] + pull_request: + branches: [main, release/*] + schedule: + - cron: "0 0 * * 0,2,4" + +jobs: + build: + runs-on: ubuntu-latest + permissions: read-all + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + steps: + # Print out details about the run + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJSON(job) }} + run: echo "$JOB_CONTEXT" + # end print details + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + ${{ runner.os }}-pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + if [ -f requirements.txt ]; then + python -m pip install -r requirements.txt + fi + python -m pip install -e . + - name: Install test dependencies + run: | + if [ -f requirements-dev.txt ]; then + python -m pip install -r requirements-dev.txt + else + echo "Missing requirements-dev.txt. Installing minimal requirements for testing." + python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions + python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml + fi + python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" + - name: Prepare test dummy data + run: | + mkdir ~/.msticpy + cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy + touch ~/.msticpy/GeoLite2-City.mmdb + + - name: Pytest + env: + MAXMIND_AUTH: DUMMY_KEY + IPSTACK_AUTH: DUMMY_KEY + MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml + MSTICPY_BUILD_SOURCE: fork + run: | + pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticnb --cov-report=xml + if: ${{ always() }} + - name: Upload pytest test results + uses: actions/upload-artifact@v3 + with: + name: pytest-results-${{ matrix.python-version }} + path: junit/test-${{ matrix.python-version }}-results.xml + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} + + lint: + runs-on: ubuntu-latest + permissions: read-all + strategy: + matrix: + python-version: ["3.8"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip + uses: actions/cache@v3 + with: + # This path is specific to Ubuntu + path: ~/.cache/pip + # Look to see if there is a cache hit for the corresponding requirements file + key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }} + ${{ runner.os }}-pip-lint + ${{ runner.os }}-pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + if [ -f requirements.txt ]; then + python -m pip install -r requirements.txt; + fi + python -m pip install -e . + - name: Install test dependencies + run: | + if [ -f requirements-dev.txt ]; then + python -m pip install -r requirements-dev.txt + else + echo "Missing requirements-dev.txt. Installing minimal requirements for testing." + python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma + fi + - name: black + run: | + black --diff --check --exclude venv msticnb + if: ${{ always() }} + - name: flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 msticnb --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto + if: ${{ always() }} + - name: pylint + run: | + pylint msticnb --disable=duplicate-code --disable=E1135,E1101,E1133 + if: ${{ always() }} + - name: Cache/restore MyPy data + id: cache-mypy + uses: actions/cache@v3 + with: + # MyPy cache files are stored in `~/.mypy_cache` + path: .mypy_cache + key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }} + ${{ runner.os }}-build-mypy-${{ github.ref }} + ${{ runner.os }}-build-mypy + - name: mypy + run: | + mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticnb + if: ${{ always() }} + - name: Upload mypy test results + uses: actions/upload-artifact@v3 + with: + name: Mypy results ${{ matrix.python-version }} + path: junit/mypy-test-${{ matrix.python-version }}-results.xml + # Use always() to always run this step to publish test results when there are test failures + if: ${{ always() }} + - name: flake8 + run: | + flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto + if: ${{ always() }} + - name: pydocstyle + run: | + pydocstyle --convention=numpy msticnb + if: ${{ always() }} + - name: pyroma + run: | + pyroma --min 10 . + if: ${{ always() }} + check_status: + runs-on: ubuntu-latest + permissions: read-all + needs: [build, lint] + steps: + - name: File build fail issue + if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }} + uses: dacbd/create-issue-action@v1 + with: + token: ${{ github.token }} + title: "Build failed for main branch" + body: The build failed on branch ${{ github.ref }}. Please investigate + labels: build_break, bug, high_severity diff --git a/.github/ISSUE_TEMPLATE/workflows/python-publish.yml b/.github/ISSUE_TEMPLATE/workflows/python-publish.yml new file mode 100644 index 0000000..91cdc9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/workflows/python-publish.yml @@ -0,0 +1,45 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package to PyPI Prod + +on: + release: + types: [published] + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + permissions: read-all + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.9' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build + - name: Build package + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + - name: Publish package + uses: pypa/gh-action-pypi-publish@v1.5.1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0b2d497..68a48e1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,7 +48,7 @@ stages: condition: succeededOrFailed() displayName: pytest env: - MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/testdata/msticpyconfig-test.yaml + MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml MAXMIND_AUTH: $(maxmind_auth) IPSTACK_AUTH: $(ipstack_auth) MSTICPY_TEST_NOSKIP: 1 diff --git a/docs/notebooks/NotebookletsDemo.rst b/docs/notebooks/NotebookletsDemo.rst index f27d9ec..757944a 100644 --- a/docs/notebooks/NotebookletsDemo.rst +++ b/docs/notebooks/NotebookletsDemo.rst @@ -4533,8 +4533,8 @@ Three sections: @set_text(docs=_CELL_DOCS, key="display_event_pivot") def _display_event_pivot(event_pivot): display( - event_pivot.style.applymap(lambda x: "color: white" if x == 0 else "") - .applymap( + event_pivot.style.map(lambda x: "color: white" if x == 0 else "") + .map( lambda x: "background-color: lightblue" if not isinstance(x, str) and x > 0 else "" @@ -4656,8 +4656,8 @@ Three sections: @set_text(docs=_CELL_DOCS, key="display_acct_event_pivot") def _display_acct_event_pivot(event_pivot_df): display( - event_pivot_df.style.applymap(lambda x: "color: white" if x == 0 else "") - .applymap( + event_pivot_df.style.map(lambda x: "color: white" if x == 0 else "") + .map( lambda x: "background-color: lightblue" if not isinstance(x, str) and x > 0 else "" diff --git a/msticnb/_version.py b/msticnb/_version.py index f1ab77c..dc393e8 100644 --- a/msticnb/_version.py +++ b/msticnb/_version.py @@ -1,2 +1,2 @@ """Version file.""" -VERSION = "1.1.1" +VERSION = "1.2.0" diff --git a/msticnb/data_providers.py b/msticnb/data_providers.py index ad51d9d..c2576f2 100644 --- a/msticnb/data_providers.py +++ b/msticnb/data_providers.py @@ -11,15 +11,16 @@ from msticpy.common.exceptions import MsticpyAzureConfigError from msticpy.common.wsconfig import WorkspaceConfig -from msticpy.data import QueryProvider try: from msticpy.context import GeoLiteLookup, IPStackLookup, TILookup from msticpy.context.azure.azure_data import AzureData + from msticpy.data.core.data_providers import QueryProvider from msticpy.data.core.query_defns import DataEnvironment except ImportError: # Fall back to msticpy locations prior to v2.0.0 from msticpy.data.azure.azure_data import AzureData + from msticpy.data import QueryProvider from msticpy.data.query_defns import DataEnvironment from msticpy.sectools import GeoLiteLookup, IPStackLookup, TILookup diff --git a/msticnb/nb/azsent/account/account_summary.py b/msticnb/nb/azsent/account/account_summary.py index 8033f21..5fcf6a6 100644 --- a/msticnb/nb/azsent/account/account_summary.py +++ b/msticnb/nb/azsent/account/account_summary.py @@ -76,6 +76,8 @@ def parse(cls, name: str): # pylint: enable=invalid-name +# pylint: disable=no-member + # pylint: disable=too-few-public-methods, too-many-instance-attributes class AccountSummaryResult(NotebookletResult): diff --git a/msticnb/nb/azsent/alert/ti_enrich.py b/msticnb/nb/azsent/alert/ti_enrich.py index 84aeda5..88af0fb 100644 --- a/msticnb/nb/azsent/alert/ti_enrich.py +++ b/msticnb/nb/azsent/alert/ti_enrich.py @@ -195,7 +195,7 @@ def run( ["StartTimeUtc", "AlertName", "Severity", "TI Risk", "Description"] ] .sort_values(by=["StartTimeUtc"]) - .style.applymap(_color_cells) + .style.map(_color_cells) .hide_index() ) if "details" in self.options: @@ -241,7 +241,7 @@ def show_full_alert(selected_alert): ["Ioc", "IocType", "Provider", "Result", "Severity", "Details"] ] .reset_index() - .style.applymap(_color_cells) + .style.map(_color_cells) .hide_index() ) ti_ips = ti_data[ti_data["IocType"] == "ipv4"] diff --git a/msticnb/nb/azsent/host/win_host_events.py b/msticnb/nb/azsent/host/win_host_events.py index 631e29f..c8b0b93 100644 --- a/msticnb/nb/azsent/host/win_host_events.py +++ b/msticnb/nb/azsent/host/win_host_events.py @@ -6,6 +6,7 @@ """Notebooklet for Windows Security Events.""" import os import pkgutil +from io import StringIO from typing import Any, Dict, Iterable, Optional, Union import numpy as np @@ -281,8 +282,8 @@ def _get_win_security_events(qry_prov, host_name, timespan): @set_text(docs=_CELL_DOCS, key="display_event_pivot") def _display_event_pivot(event_pivot): display( - event_pivot.style.applymap(lambda x: "color: white" if x == 0 else "") - .applymap( + event_pivot.style.map(lambda x: "color: white" if x == 0 else "") + .map( lambda x: "background-color: lightblue" if not isinstance(x, str) and x > 0 else "" @@ -364,7 +365,7 @@ def _extract_acct_mgmt_events(event_data): w_evt = pkgutil.get_data("msticpy", f"resources{os.sep}WinSecurityEvent.json") - win_event_df = pd.read_json(w_evt.decode("utf-8")) + win_event_df = pd.read_json(StringIO(w_evt.decode("utf-8"))) # Create criteria for events that we're interested in acct_sel = win_event_df["subcategory"] == "User Account Management" @@ -407,8 +408,8 @@ def _create_acct_event_pivot(account_event_data): @set_text(docs=_CELL_DOCS, key="display_acct_event_pivot") def _display_acct_event_pivot(event_pivot_df): display( - event_pivot_df.style.applymap(lambda x: "color: white" if x == 0 else "") - .applymap( + event_pivot_df.style.map(lambda x: "color: white" if x == 0 else "") + .map( lambda x: "background-color: lightblue" if not isinstance(x, str) and x > 0 else "" diff --git a/msticnb/nb/azsent/url/url_summary.py b/msticnb/nb/azsent/url/url_summary.py index 456c7fe..8ffcbd8 100644 --- a/msticnb/nb/azsent/url/url_summary.py +++ b/msticnb/nb/azsent/url/url_summary.py @@ -6,7 +6,7 @@ """Notebooklet for URL Summary.""" from collections import Counter from os.path import exists -from typing import Any, Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional, Tuple, cast import dns.resolver import numpy as np @@ -161,7 +161,7 @@ def run( # noqa:MC0001 self._last_result = result self.url = value.strip().lower() - _, domain, tld = tldextract.extract(self.url) + _, domain, tld = cast(Tuple[Any, str, str], tldextract.extract(self.url)) # type: ignore domain = f"{domain.lower()}.{tld.lower()}" domain_validator = DomainValidator() validated = domain_validator.validate_tld(domain) @@ -283,7 +283,7 @@ def _display_domain_record(self): """Display Domain Record.""" if self.check_valid_result_data("domain_record", silent=True): display( - self._last_result.domain_record.T.style.applymap( # type: ignore + self._last_result.domain_record.T.style.map( # type: ignore color_domain_record_cells, subset=pd.IndexSlice[["Page Rank", "Domain Name Entropy"], 0], ) @@ -484,6 +484,7 @@ def _domain_whois_record(domain, ti_prov): # Remove duplicate Name Server records for server in whois_result["name_servers"]: + # pylint: disable=unpacking-non-sequence _, ns_domain, ns_tld = tldextract.extract(server) ns_dom = ns_domain.lower() + "." + ns_tld.lower() if domain not in ns_domains: diff --git a/msticnb/nb/template/nb_template.py b/msticnb/nb/template/nb_template.py index 6e3fe71..6ccdb91 100644 --- a/msticnb/nb/template/nb_template.py +++ b/msticnb/nb/template/nb_template.py @@ -294,7 +294,7 @@ def _get_all_events(qry_prov, host_name, timespan): del qry_prov, host_name, timespan return pd.DataFrame( { - "TimeGenerated": pd.date_range("2022-01-01", periods=5, tz="utc", freq="H"), + "TimeGenerated": pd.date_range("2022-01-01", periods=5, tz="utc", freq="h"), "EventID": [4688, 4688, 4625, 4624, 4624], "Computer": ["MyHost.dom"] * 5, "Account": [f"user{n}" for n in range(5)], diff --git a/msticnb/nblib/azsent/host.py b/msticnb/nblib/azsent/host.py index 94ed44b..4e4a874 100644 --- a/msticnb/nblib/azsent/host.py +++ b/msticnb/nblib/azsent/host.py @@ -10,7 +10,7 @@ import pandas as pd from msticpy.common.timespan import TimeSpan -from msticpy.data import QueryProvider +from msticpy.data import QueryProvider # pylint: disable=no-name-in-module from msticpy.datamodel import entities from ..._version import VERSION diff --git a/msticnb/read_modules.py b/msticnb/read_modules.py index 199ca26..f548514 100644 --- a/msticnb/read_modules.py +++ b/msticnb/read_modules.py @@ -24,7 +24,8 @@ __author__ = "Ian Hellen" nblts: NBContainer = NBContainer() -nb_index: Dict[str, Notebooklet] = {} +# index of notebooklets classes by full path +nb_index: Dict[str, type] = {} def discover_modules(nb_path: Union[str, Iterable[str], None] = None) -> NBContainer: @@ -96,7 +97,7 @@ def _import_from_folder(nb_folder: Path, pkg_folder: Path): nb_index[cls_index] = nb_class -def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, Notebooklet]: +def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, type]: """ Import .py files in `folder` and return any Notebooklet classes found. @@ -109,7 +110,7 @@ def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, Notebooklet]: Returns ------- - Dict[str, Notebooklet] + Dict[str, type(Notebooklet)] Notebooklets classes (name, class) """ @@ -144,7 +145,7 @@ def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, Notebooklet]: # We need to store the path of the parent module in the class # - this makes it easier to retrieve when we need it for # reading metadata and generating the class docs. - mod_class.module_path = item + mod_class.module_path = str(item) # create a function (pointer) in the class that will # build and return our extended class documentation setattr( diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..3d041b6 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,21 @@ +black>=20.8b1, <24.0.0 +coverage>=5.5 +flake8>=3.8.4 +isort>=5.10.1 +mccabe>=0.6.1 +mypy>=0.812 +pep8-naming>=0.10.0 +pep8>=1.7.1 +pre-commit>=2.7.1 +pycodestyle>=2.6.0 +pydocstyle>=6.0.0 +pyflakes>=2.2.0 +pylint>=2.5.3 +pyroma>=3.1 +pytest-check>=1.0.1 +pytest-cov>=2.11.1 +pytest-xdist>=2.5.0 +pytest>=5.0.1 +responses>=0.13.2 +respx>=0.20.1 +types-attrs>=19.0.0 diff --git a/tests/testdata/msticpyconfig-test.yaml b/tests/msticpyconfig-test.yaml similarity index 100% rename from tests/testdata/msticpyconfig-test.yaml rename to tests/msticpyconfig-test.yaml diff --git a/tests/nb/azsent/host/test_hostlogonsummary.py b/tests/nb/azsent/host/test_hostlogonsummary.py index 9d89c95..df7b55c 100644 --- a/tests/nb/azsent/host/test_hostlogonsummary.py +++ b/tests/nb/azsent/host/test_hostlogonsummary.py @@ -61,6 +61,7 @@ def test_local_data(monkeypatch): """Test nblt output types and values using LocalData provider.""" test_data = str(Path.cwd().joinpath(TEST_DATA_PATH)) monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock) + discover_modules() data_providers.init( query_provider="LocalData", LocalData_data_paths=[test_data], diff --git a/tests/nb/azsent/host/test_logon_session_rarity.py b/tests/nb/azsent/host/test_logon_session_rarity.py index ff1fee8..652f31e 100644 --- a/tests/nb/azsent/host/test_logon_session_rarity.py +++ b/tests/nb/azsent/host/test_logon_session_rarity.py @@ -35,6 +35,12 @@ def init_notebooklets(monkeypatch): def test_logon_session_rarity_notebooklet(init_notebooklets): """Test basic run of notebooklet.""" + try: + # pylint: disable=import-outside-toplevel, unused-import + import sklearn + import matplotlib + except ImportError: + pytest.skip("sklearn and matplotlib required for this test") d_path = Path(TEST_DATA_PATH).joinpath("processes_on_host.pkl") raw_data = pd.read_pickle(d_path) filt_sess = raw_data[raw_data["Account"] == "MSTICAlertsWin1\\MSTICAdmin"] From 777f8008d55dbd92e268b298b5a51d984e4f95b4 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sun, 11 Feb 2024 15:07:17 -0800 Subject: [PATCH 08/13] Update python-package.yml Removing unneeded test data copy job from build action --- .github/workflows/python-package.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 278c563..c9501b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -61,12 +61,6 @@ jobs: python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml fi python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" - - name: Prepare test dummy data - run: | - mkdir ~/.msticpy - cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy - touch ~/.msticpy/GeoLite2-City.mmdb - - name: Pytest env: MAXMIND_AUTH: DUMMY_KEY From 30b1a0175f4daf91d6b23cb91c7e90e2a2fba378 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Sun, 11 Feb 2024 15:09:50 -0800 Subject: [PATCH 09/13] Ianhelle/fix build errors 2024 02 11 (#42) * Linting fixes for MSTICPy 2.9 compatibility * Moving test msticpyconfig to root of tests * Typo in python-package.yml * Additional fixes: - workaround for msticpy throwing errors when trying to import yaml file as query file (moving the custom notebook test out of the testdata folder) - getting rid of some code producing pandas deprecation warnings * Removing unneeded copy from build action --- .github/ISSUE_TEMPLATE/workflows/python-package.yml | 6 ------ msticnb/nb/azsent/host/win_host_events.py | 1 + msticnb/nblib/iptools.py | 2 +- tests/{testdata => }/custom_nb/__init__.py | 0 tests/{testdata => }/custom_nb/host/__init__.py | 0 tests/{testdata => }/custom_nb/host/host_test_nb.py | 0 tests/{testdata => }/custom_nb/host/host_test_nb.yaml | 0 tests/test_read_modules.py | 2 +- 8 files changed, 3 insertions(+), 8 deletions(-) rename tests/{testdata => }/custom_nb/__init__.py (100%) rename tests/{testdata => }/custom_nb/host/__init__.py (100%) rename tests/{testdata => }/custom_nb/host/host_test_nb.py (100%) rename tests/{testdata => }/custom_nb/host/host_test_nb.yaml (100%) diff --git a/.github/ISSUE_TEMPLATE/workflows/python-package.yml b/.github/ISSUE_TEMPLATE/workflows/python-package.yml index 67a9cec..74bc2ab 100644 --- a/.github/ISSUE_TEMPLATE/workflows/python-package.yml +++ b/.github/ISSUE_TEMPLATE/workflows/python-package.yml @@ -61,12 +61,6 @@ jobs: python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml fi python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0" - - name: Prepare test dummy data - run: | - mkdir ~/.msticpy - cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy - touch ~/.msticpy/GeoLite2-City.mmdb - - name: Pytest env: MAXMIND_AUTH: DUMMY_KEY diff --git a/msticnb/nb/azsent/host/win_host_events.py b/msticnb/nb/azsent/host/win_host_events.py index c8b0b93..43aa70f 100644 --- a/msticnb/nb/azsent/host/win_host_events.py +++ b/msticnb/nb/azsent/host/win_host_events.py @@ -333,6 +333,7 @@ def _expand_event_properties(input_df): right_index=True, ) .replace("", np.nan) # these 3 lines get rid of blank columns + .infer_objects(copy=False) .dropna(axis=1, how="all") .fillna("") ) diff --git a/msticnb/nblib/iptools.py b/msticnb/nblib/iptools.py index 01207e7..a299e21 100644 --- a/msticnb/nblib/iptools.py +++ b/msticnb/nblib/iptools.py @@ -55,7 +55,7 @@ def get_ip_ti( def _normalize_ip4(data, ip_col): ip4_rgx = r"((?:[0-9]{1,3}\.){3}[0-9]{1,3})" - ip4_match = data[ip_col].str.match(ip4_rgx).fillna(False) + ip4_match = data[ip_col].str.match(ip4_rgx).fillna(False).infer_objects(copy=False) ipv4_df = data[ip4_match] other_df = data[~ip4_match] diff --git a/tests/testdata/custom_nb/__init__.py b/tests/custom_nb/__init__.py similarity index 100% rename from tests/testdata/custom_nb/__init__.py rename to tests/custom_nb/__init__.py diff --git a/tests/testdata/custom_nb/host/__init__.py b/tests/custom_nb/host/__init__.py similarity index 100% rename from tests/testdata/custom_nb/host/__init__.py rename to tests/custom_nb/host/__init__.py diff --git a/tests/testdata/custom_nb/host/host_test_nb.py b/tests/custom_nb/host/host_test_nb.py similarity index 100% rename from tests/testdata/custom_nb/host/host_test_nb.py rename to tests/custom_nb/host/host_test_nb.py diff --git a/tests/testdata/custom_nb/host/host_test_nb.yaml b/tests/custom_nb/host/host_test_nb.yaml similarity index 100% rename from tests/testdata/custom_nb/host/host_test_nb.yaml rename to tests/custom_nb/host/host_test_nb.yaml diff --git a/tests/test_read_modules.py b/tests/test_read_modules.py index 8d80034..dbcb10c 100644 --- a/tests/test_read_modules.py +++ b/tests/test_read_modules.py @@ -35,7 +35,7 @@ def test_read_modules(): def test_read_custom_path(): """Test method.""" - cust_nb_path = Path(TEST_DATA_PATH) / "custom_nb" + cust_nb_path = Path(TEST_DATA_PATH).parent / "custom_nb" nbklts = discover_modules(nb_path=str(cust_nb_path)) check.greater_equal(len(list(nbklts.iter_classes())), 5) From 0f4e448d8ab08f4177a82cdc03bd2fbe96030331 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Mon, 12 Feb 2024 10:17:06 -0800 Subject: [PATCH 10/13] Adding patch to mock GeoIP provider in Folium map (#40) --- .../network/test_network_flow_summary.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/nb/azsent/network/test_network_flow_summary.py b/tests/nb/azsent/network/test_network_flow_summary.py index 148279d..fa63cf5 100644 --- a/tests/nb/azsent/network/test_network_flow_summary.py +++ b/tests/nb/azsent/network/test_network_flow_summary.py @@ -16,6 +16,7 @@ import respx from bokeh.models import LayoutDOM from msticpy.common.timespan import TimeSpan +from msticpy.vis import foliummap from msticnb import data_providers, discover_modules, nblts @@ -43,6 +44,7 @@ def init_notebooklets(monkeypatch): discover_modules() monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock) + monkeypatch.setattr(foliummap, "GeoLiteLookup", GeoIPLiteMock) monkeypatch.setattr(data_providers, "TILookup", TILookupMock) data_providers.init( query_provider="LocalData", @@ -73,19 +75,19 @@ def rdap_response(): @respx.mock @patch("msticpy.context.ip_utils._asn_whois_query") def test_network_flow_summary_notebooklet( - mock_whois, monkeypatch, init_notebooklets, rdap_response, whois_response + mock_whois, init_notebooklets, rdap_response, whois_response ): """Test basic run of notebooklet.""" - discover_modules() - test_data = str(Path(TEST_DATA_PATH).absolute()) + # discover_modules() + # test_data = str(Path(TEST_DATA_PATH).absolute()) mock_whois.return_value = whois_response["asn_response_1"] - monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock) - monkeypatch.setattr(data_providers, "TILookup", TILookupMock) - data_providers.init( - query_provider="LocalData", - LocalData_data_paths=[test_data], - LocalData_query_paths=[test_data], - ) + # monkeypatch.setattr(data_providers, "GeoLiteLookup", GeoIPLiteMock) + # monkeypatch.setattr(data_providers, "TILookup", TILookupMock) + # data_providers.init( + # query_provider="LocalData", + # LocalData_data_paths=[test_data], + # LocalData_query_paths=[test_data], + # ) respx.get(re.compile(r"http://rdap\.arin\.net/.*")).respond(200, json=rdap_response) test_nb = nblts.azsent.network.NetworkFlowSummary() From d1e39589a0ff8b3318511d8ff0281346596d7e40 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Mon, 4 Mar 2024 13:41:52 -0800 Subject: [PATCH 11/13] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 68a48e1..6252b5b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -119,7 +119,7 @@ stages: pool: vmImage: windows-latest variables: - python.version: '3.6' + python.version: '3.10' steps: - task: CredScan@2 displayName: 'Run CredScan' From a1bb25a2c7110feb789a501b15326980731b1bac Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Fri, 29 Mar 2024 10:35:02 -0700 Subject: [PATCH 12/13] Adding new azure pipelines to repo (#43) --- .azurepipelines/azure-pipelines-pr.yml | 42 +++++++++++++++++++++ .azurepipelines/azure-pipelines-release.yml | 41 ++++++++++++++++++++ pytest.ini | 2 + 3 files changed, 85 insertions(+) create mode 100644 .azurepipelines/azure-pipelines-pr.yml create mode 100644 .azurepipelines/azure-pipelines-release.yml diff --git a/.azurepipelines/azure-pipelines-pr.yml b/.azurepipelines/azure-pipelines-pr.yml new file mode 100644 index 0000000..dba1aeb --- /dev/null +++ b/.azurepipelines/azure-pipelines-pr.yml @@ -0,0 +1,42 @@ +# MSTICNB PR pipeline + +trigger: none +name: 1ES-MSTICNB-PR-$(date:yyyyMMdd)$(rev:.r) + +resources: + repositories: + - repository: self + type: git + ref: main + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: MSSecurity-1ES-Build-Agents-Pool + image: MSSecurity-1ES-Windows-2022 + os: windows + stages: + - stage: buildTasks + displayName: BuildTasks + jobs: + - job: additionalChecks + displayName: AdditionalChecks + steps: + - task: notice@0 + displayName: NOTICE File Generator + # This fails for external forks + condition: not(variables['System.PullRequest.IsFork']) + sdl: + apiScan: + enabled: false + policheck: + enabled: true + bandit: + enabled: true + + diff --git a/.azurepipelines/azure-pipelines-release.yml b/.azurepipelines/azure-pipelines-release.yml new file mode 100644 index 0000000..8ec6080 --- /dev/null +++ b/.azurepipelines/azure-pipelines-release.yml @@ -0,0 +1,41 @@ +# MSTICNB Release pipeline + +trigger: none +name: 1ES-MSTICNB-Rel-$(date:yyyyMMdd)$(rev:.r) + +resources: + repositories: + - repository: self + type: git + ref: main + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: MSSecurity-1ES-Build-Agents-Pool + image: MSSecurity-1ES-Windows-2022 + os: windows + stages: + - stage: buildTasks + displayName: BuildTasks + jobs: + - job: additionalChecks + displayName: AdditionalChecks + steps: + - task: notice@0 + displayName: NOTICE File Generator + # This fails for external forks + condition: not(variables['System.PullRequest.IsFork']) + sdl: + apiScan: + enabled: false + policheck: + enabled: true + bandit: + enabled: true + diff --git a/pytest.ini b/pytest.ini index 6b59f2a..c36d318 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,5 @@ markers = notebook: Test runs a notebook - slow junit_family=legacy +env = + MSTICPYCONFIG=tests/msticpyconfig-test.yaml From c2f2903baa0efa65959b950ad8c426ccb2f1ff11 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Tue, 28 May 2024 17:08:26 -0700 Subject: [PATCH 13/13] Misc build and linter fixes (#44) * Replaced reference to pywhois with msticpy whois - i url_summary.py Fixed creation of process tree - ensuring Rarity column remains numeric in logon_session_rarity.py Fixes in ti.py for potentially unitialized variables. Fixed missing respx mocked URLs in test_ip_summary.py * Updating requirements.txt to align with msticpy * Mypy and pylint fixes --- .pre-commit-config.yaml | 27 ++++-------- msticnb/nb/azsent/account/account_summary.py | 10 ++--- msticnb/nb/azsent/host/host_logons_summary.py | 8 ++-- .../nb/azsent/host/host_network_summary.py | 10 ++++- msticnb/nb/azsent/host/host_summary.py | 2 +- .../nb/azsent/host/logon_session_rarity.py | 15 ++++++- msticnb/nb/azsent/network/ip_summary.py | 2 +- .../nb/azsent/network/network_flow_summary.py | 2 +- msticnb/nb/azsent/url/url_summary.py | 6 ++- msticnb/nblib/azsent/host.py | 2 +- msticnb/nblib/ti.py | 42 +++++++++---------- msticnb/notebooklet.py | 5 ++- msticnb/read_modules.py | 7 ++-- pyproject.toml | 20 +++++++++ requirements.txt | 3 +- tests/nb/azsent/host/test_hostlogonsummary.py | 2 +- tests/nb/azsent/network/test_ip_summary.py | 3 ++ 17 files changed, 99 insertions(+), 67 deletions(-) create mode 100644 pyproject.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e1cd033..e970d69 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,33 +7,24 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/ambv/black - rev: 22.1.0 + rev: 24.4.2 hooks: - id: black language: python args: - -t - py36 - - repo: https://github.com/PyCQA/pylint - rev: v2.12.2 - hooks: - - id: pylint - args: - - --disable=E0401,W0511,duplicate-code - - --ignore-patterns=test_ - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 - args: - - --extend-ignore=E0401,E501,W503 - - --max-line-length=90 - - --exclude=tests,test*.py - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort name: isort (python) args: - --profile - - black \ No newline at end of file + - black + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.4.5 + hooks: + # Run the linter. + - id: ruff \ No newline at end of file diff --git a/msticnb/nb/azsent/account/account_summary.py b/msticnb/nb/azsent/account/account_summary.py index 5fcf6a6..3848b76 100644 --- a/msticnb/nb/azsent/account/account_summary.py +++ b/msticnb/nb/azsent/account/account_summary.py @@ -142,17 +142,17 @@ def __init__( self.description: str = "Account Activity Summary" self.account_entity: entities.Account = None self.account_activity: Optional[pd.DataFrame] = None - self.account_selector: nbwidgets.SelectItem = None + self.account_selector: Optional[nbwidgets.SelectItem] = None self.related_alerts: Optional[pd.DataFrame] = None - self.alert_timeline: LayoutDOM = None + self.alert_timeline: Optional[LayoutDOM] = None self.related_bookmarks: Optional[pd.DataFrame] = None self.host_logons: Optional[pd.DataFrame] = None self.host_logon_summary: Optional[pd.DataFrame] = None self.azure_activity: Optional[pd.DataFrame] = None self.azure_activity_summary: Optional[pd.DataFrame] = None - self.azure_timeline_by_provider: LayoutDOM = None - self.account_timeline_by_ip: LayoutDOM = None - self.azure_timeline_by_operation: LayoutDOM = None + self.azure_timeline_by_provider: Optional[LayoutDOM] = None + self.account_timeline_by_ip: Optional[LayoutDOM] = None + self.azure_timeline_by_operation: Optional[LayoutDOM] = None self.ip_summary: Optional[pd.DataFrame] = None self.ip_all_data: Optional[pd.DataFrame] = None diff --git a/msticnb/nb/azsent/host/host_logons_summary.py b/msticnb/nb/azsent/host/host_logons_summary.py index 33ea047..5e3a73d 100644 --- a/msticnb/nb/azsent/host/host_logons_summary.py +++ b/msticnb/nb/azsent/host/host_logons_summary.py @@ -107,9 +107,9 @@ class HostLogonsSummary(Notebooklet): # pylint: disable=too-few-public-methods metadata = _CLS_METADATA - @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001 + @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001, C901 # pylint: disable=too-many-locals, too-many-branches, too-many-statements - def run( # noqa:MC0001 + def run( # noqa:MC0001, C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, @@ -380,13 +380,13 @@ def _process_stack_bar(data: pd.DataFrame, silent: bool) -> figure: legend_label=results, ) - viz.y_range.start = 0 + viz.y_range.start = 0 # type: ignore[attr-defined] viz.x_range.range_padding = 0.1 # type: ignore[attr-defined] viz.xgrid.grid_line_color = None # type: ignore[attr-defined] viz.axis.minor_tick_line_color = None viz.yaxis.axis_label = "% of logons" viz.xaxis.axis_label = "Process name" # type: ignore[assignment] - viz.outline_line_color = None + viz.outline_line_color = None # type: ignore[assignment] viz.legend.location = "top_left" viz.legend.orientation = "horizontal" diff --git a/msticnb/nb/azsent/host/host_network_summary.py b/msticnb/nb/azsent/host/host_network_summary.py index 97719f5..e617b85 100644 --- a/msticnb/nb/azsent/host/host_network_summary.py +++ b/msticnb/nb/azsent/host/host_network_summary.py @@ -86,7 +86,7 @@ def __init__(self, *args, **kwargs): # pylint: disable=too-many-branches @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001 - def run( # noqa:MC0001 + def run( # noqa:MC0001, C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, @@ -163,6 +163,10 @@ def run( # noqa:MC0001 qry_prov=self.query_provider, timespan=self.timespan, ) + if result.flows is None: + nb_markdown("No network flow data found.") + self._last_result = result + return self._last_result remote_ip_col = "RemoteIP" local_ip_col = "LocalIP" @@ -239,7 +243,7 @@ def _display_results(self): @lru_cache() -def _get_host_flows(host_name, ip_addr, qry_prov, timespan) -> pd.DataFrame: +def _get_host_flows(host_name, ip_addr, qry_prov, timespan) -> Optional[pd.DataFrame]: if host_name: nb_data_wait("Host flow events") host_flows = qry_prov.MDE.host_connections(timespan, host_name=host_name) @@ -254,6 +258,8 @@ def _get_host_flows(host_name, ip_addr, qry_prov, timespan) -> pd.DataFrame: host_flows_csl = qry_prov.Network.ip_network_connections_csl( timespan, ip=ip_addr ) + else: + return None return pd.concat([host_flows, host_flows_csl], sort=False) diff --git a/msticnb/nb/azsent/host/host_summary.py b/msticnb/nb/azsent/host/host_summary.py index a1ef17a..5a8bc1d 100644 --- a/msticnb/nb/azsent/host/host_summary.py +++ b/msticnb/nb/azsent/host/host_summary.py @@ -124,7 +124,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # pylint: disable=too-many-branches, too-many-statements - def run( # noqa:MC0001 + def run( # noqa:MC0001, C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, diff --git a/msticnb/nb/azsent/host/logon_session_rarity.py b/msticnb/nb/azsent/host/logon_session_rarity.py index a6f9d57..f937cc8 100644 --- a/msticnb/nb/azsent/host/logon_session_rarity.py +++ b/msticnb/nb/azsent/host/logon_session_rarity.py @@ -10,6 +10,9 @@ from msticpy.analysis.eventcluster import char_ord_score, dbcluster_events, delim_count from msticpy.common.timespan import TimeSpan +# pylint: disable=unused-import +from msticpy.init import mp_pandas_accessors # noqa: F401 + try: from msticpy import nbwidgets @@ -252,13 +255,21 @@ def process_tree( acct_col = self.column_map.get(COL_ACCT) data = self._last_result.processes_with_cluster data = data[data[acct_col] == account] - data.mp_plot.process_tree(legend_col="Rarity") + proc_tree_data = data.mp.build_process_tree() + proc_tree_data["Rarity"] = pd.to_numeric( + proc_tree_data["Rarity"], errors="coerce" + ).fillna(0) + proc_tree_data.mp_plot.process_tree(legend_col="Rarity") return session = session or self._event_browser.value sess_col = self.column_map.get(COL_SESS) data = self._last_result.processes_with_cluster data = data[data[sess_col] == session] - data.mp_plot.process_tree(legend_col="Rarity") + proc_tree_data = data.mp.build_process_tree() + proc_tree_data["Rarity"] = pd.to_numeric( + proc_tree_data["Rarity"], errors="coerce" + ).fillna(0) + proc_tree_data.mp_plot.process_tree(legend_col="Rarity") def browse_events(self): """Browse the events by logon session.""" diff --git a/msticnb/nb/azsent/network/ip_summary.py b/msticnb/nb/azsent/network/ip_summary.py index af4fd53..aef9d43 100644 --- a/msticnb/nb/azsent/network/ip_summary.py +++ b/msticnb/nb/azsent/network/ip_summary.py @@ -209,7 +209,7 @@ class IpAddressSummary(Notebooklet): # pylint: disable=too-many-branches, too-many-statements @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001 - def run( # noqa: MC0001 + def run( # noqa: MC0001,C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, diff --git a/msticnb/nb/azsent/network/network_flow_summary.py b/msticnb/nb/azsent/network/network_flow_summary.py index fdd4a9d..bc2206b 100644 --- a/msticnb/nb/azsent/network/network_flow_summary.py +++ b/msticnb/nb/azsent/network/network_flow_summary.py @@ -166,7 +166,7 @@ def __init__(self, data_providers: Optional[DataProviders] = None, **kwargs): # pylint: disable=too-many-branches @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001 - def run( # noqa: MC0001 + def run( # noqa: MC0001, C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, diff --git a/msticnb/nb/azsent/url/url_summary.py b/msticnb/nb/azsent/url/url_summary.py index 8ffcbd8..d84972f 100644 --- a/msticnb/nb/azsent/url/url_summary.py +++ b/msticnb/nb/azsent/url/url_summary.py @@ -13,15 +13,16 @@ import pandas as pd import tldextract from IPython.display import Image, display -from whois import whois # type: ignore # pylint: disable=ungrouped-imports try: from msticpy import nbwidgets from msticpy.context.domain_utils import DomainValidator, screenshot + from msticpy.context.ip_utils import ip_whois as whois from msticpy.vis.timeline import display_timeline, display_timeline_values except ImportError: # Fall back to msticpy locations prior to v2.0.0 + from whois import whois # type: ignore from msticpy.sectools.domain_utils import DomainValidator, screenshot from msticpy.nbtools import nbwidgets from msticpy.nbtools.nbdisplay import display_timeline, display_timeline_values @@ -95,7 +96,7 @@ class URLSummary(Notebooklet): # pylint: disable=too-many-branches, too-many-locals, too-many-statements @set_text(docs=_CELL_DOCS, key="run") # noqa: MC0001 - def run( # noqa:MC0001 + def run( # noqa:MC0001, C901 self, value: Any = None, data: Optional[pd.DataFrame] = None, @@ -161,6 +162,7 @@ def run( # noqa:MC0001 self._last_result = result self.url = value.strip().lower() + _, domain, tld = cast(Tuple[Any, str, str], tldextract.extract(self.url)) # type: ignore domain = f"{domain.lower()}.{tld.lower()}" domain_validator = DomainValidator() diff --git a/msticnb/nblib/azsent/host.py b/msticnb/nblib/azsent/host.py index 4e4a874..5e1ec17 100644 --- a/msticnb/nblib/azsent/host.py +++ b/msticnb/nblib/azsent/host.py @@ -102,7 +102,7 @@ def get_aznet_topology( @lru_cache() # noqa:MC0001 -def verify_host_name( # noqa: MC0001 +def verify_host_name( # noqa: MC0001, C901 qry_prov: QueryProvider, host_name: str, timespan: TimeSpan = None, **kwargs ) -> HostNameVerif: """ diff --git a/msticnb/nblib/ti.py b/msticnb/nblib/ti.py index a1c1b53..990ca57 100644 --- a/msticnb/nblib/ti.py +++ b/msticnb/nblib/ti.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- """Threat Intelligence notebooklet feature support.""" -from typing import Any, Tuple, Optional +from typing import Any, Optional, Tuple import numpy as np import pandas as pd @@ -81,36 +81,36 @@ def extract_iocs( ) b64_iocs = b64_extracted.mp_ioc.extract(columns=["decoded_string"]) b64_iocs["SourceIndex"] = pd.to_numeric(b64_iocs["SourceIndex"]) - data_b64_iocs = pd.merge( + data = pd.merge( left=data, right=b64_iocs, how="outer", left_index=True, right_on="SourceIndex", ) - else: - data_b64_iocs = data - other_iocs = data_b64_iocs.mp_ioc.extract(columns=[col]) - all_data_w_iocs = pd.merge( - left=data_b64_iocs, - right=other_iocs, + + iocs = data.mp_ioc.extract(columns=[col]) + data = pd.merge( + left=data, + right=iocs, how="outer", left_index=True, right_on="SourceIndex", ) - if "Observable_x" in all_data_w_iocs.columns: - all_data_w_iocs["IoC"] = np.where( - all_data_w_iocs["Observable_x"].isna(), - all_data_w_iocs["Observable_y"], - all_data_w_iocs["Observable_x"], + + if "Observable_x" in data.columns: + data["IoC"] = np.where( + data["Observable_x"].isna(), + data["Observable_y"], + data["Observable_x"], ) - all_data_w_iocs["IoCType"] = np.where( - all_data_w_iocs["IoCType_x"].isna(), - all_data_w_iocs["IoCType_y"], - all_data_w_iocs["IoCType_x"], + data["IoCType"] = np.where( + data["IoCType_x"].isna(), + data["IoCType_y"], + data["IoCType_x"], ) - all_data_w_iocs["IoC"] = all_data_w_iocs["IoC"].astype("str") + data["IoC"] = data["IoC"].astype("str") else: - all_data_w_iocs["IoC"] = all_data_w_iocs["Observable"].astype("str") - all_data_w_iocs["IoCType"] = all_data_w_iocs["IoCType"].astype("str") - return all_data_w_iocs + data["IoC"] = data["Observable"].astype("str") + data["IoCType"] = data["IoCType"].astype("str") + return data diff --git a/msticnb/notebooklet.py b/msticnb/notebooklet.py index abe5ee2..78346a1 100644 --- a/msticnb/notebooklet.py +++ b/msticnb/notebooklet.py @@ -9,7 +9,8 @@ import warnings from abc import ABC, abstractmethod from functools import wraps -from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from pathlib import Path +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union import pandas as pd from IPython.core.getipython import get_ipython @@ -35,7 +36,7 @@ class Notebooklet(ABC): metadata: NBMetadata = NBMetadata( name="Notebooklet", description="Base class", default_options=[] ) - module_path = "" + module_path: Union[str, Path] = "" def __init__(self, data_providers: Optional[DataProviders] = None, **kwargs): """ diff --git a/msticnb/read_modules.py b/msticnb/read_modules.py index f548514..9ac439a 100644 --- a/msticnb/read_modules.py +++ b/msticnb/read_modules.py @@ -11,7 +11,7 @@ from functools import partial from operator import itemgetter from pathlib import Path -from typing import Dict, Iterable, List, Tuple, Union +from typing import Dict, Iterable, List, Tuple, Type, Union from warnings import warn from . import nb @@ -24,8 +24,7 @@ __author__ = "Ian Hellen" nblts: NBContainer = NBContainer() -# index of notebooklets classes by full path -nb_index: Dict[str, type] = {} +nb_index: Dict[str, Type[Notebooklet]] = {} def discover_modules(nb_path: Union[str, Iterable[str], None] = None) -> NBContainer: @@ -97,7 +96,7 @@ def _import_from_folder(nb_folder: Path, pkg_folder: Path): nb_index[cls_index] = nb_class -def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, type]: +def _find_cls_modules(folder: Path, pkg_folder: Path) -> Dict[str, Type[Notebooklet]]: """ Import .py files in `folder` and return any Notebooklet classes found. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0d554f7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" + +[tool.isort] +profile = "black" +src_paths = ["msticnb", "tests"] + +[tool.pydocstyle] +convention = "numpy" + +[tool.ruff.lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F", "W", "D", "C"] +ignore = ["D212", "D417", "D203"] diff --git a/requirements.txt b/requirements.txt index c75a7ec..aafdda5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -bokeh<3.0.0 +bokeh>=1.4.0, <3.4.0 defusedxml>=0.6.0 ipython>=7.23.1 ipywidgets>=7.5.1 @@ -9,5 +9,4 @@ numpy>=1.17.3 pandas>=0.25.3 python-dateutil>=2.8.1 tqdm>=4.41.1 -python-whois>=0.7.3 tldextract>=3.3.0 diff --git a/tests/nb/azsent/host/test_hostlogonsummary.py b/tests/nb/azsent/host/test_hostlogonsummary.py index df7b55c..5434674 100644 --- a/tests/nb/azsent/host/test_hostlogonsummary.py +++ b/tests/nb/azsent/host/test_hostlogonsummary.py @@ -18,7 +18,7 @@ from msticpy.vis.foliummap import FoliumMap except ImportError: # Fall back to msticpy locations prior to v2.0.0 - from msticpy.nbtools.foliummap import FoliumMap + from msticpy.nbtools.foliummap import FoliumMap # noqa: F401 from msticnb import data_providers, discover_modules, nblts diff --git a/tests/nb/azsent/network/test_ip_summary.py b/tests/nb/azsent/network/test_ip_summary.py index bf93b58..3285d71 100644 --- a/tests/nb/azsent/network/test_ip_summary.py +++ b/tests/nb/azsent/network/test_ip_summary.py @@ -135,6 +135,7 @@ def test_ip_summary_notebooklet( respx.get(re.compile(r".*SecOps-Institute/Tor-IP-Addresses.*")).respond( 200, content=b"12.34.56.78\n12.34.56.78\n12.34.56.78" ) + respx.get(re.compile(r"https://api\.greynoise\.io/.*")).respond(404) tspan = TimeSpan(period="1D") result = test_nb.run(value="11.1.2.3", timespan=tspan) @@ -233,6 +234,7 @@ def test_ip_summary_notebooklet_all( respx.get(re.compile(r".*SecOps-Institute/Tor-IP-Addresses.*")).respond( 200, content=b"12.34.56.78\n12.34.56.78\n12.34.56.78" ) + respx.get(re.compile(r"https://api\.greynoise\.io/.*")).respond(404) tspan = TimeSpan(period="1D") result = test_nb.run(value="40.76.43.124", timespan=tspan, options=opts) @@ -297,6 +299,7 @@ def test_ip_summary_mde_data( respx.get(re.compile(r".*SecOps-Institute/Tor-IP-Addresses.*")).respond( 200, content=b"12.34.56.78\n12.34.56.78\n12.34.56.78" ) + respx.get(re.compile(r"https://api\.greynoise\.io/.*")).respond(404) tspan = TimeSpan(period="1D") result = test_nb.run(value="40.76.43.124", timespan=tspan, options=opts)