Merge branch 'main' into ianhelle/tldextract-fix-2024-07-22

.azurepipelines/azure-pipelines-pr.yml

@@ -0,0 +1,42 @@
+# MSTICNB PR pipeline
+
+trigger: none
+name: 1ES-MSTICNB-PR-$(date:yyyyMMdd)$(rev:.r)
+
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: main
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool:
+      name: MSSecurity-1ES-Build-Agents-Pool
+      image: MSSecurity-1ES-Windows-2022
+      os: windows
+    stages:
+    - stage: buildTasks
+      displayName: BuildTasks
+      jobs:
+      - job: additionalChecks
+        displayName: AdditionalChecks
+        steps:
+        - task: notice@0
+          displayName: NOTICE File Generator
+          # This fails for external forks
+          condition: not(variables['System.PullRequest.IsFork'])
+    sdl:
+      apiScan:
+        enabled: false
+      policheck:
+        enabled: true
+      bandit:
+        enabled: true
+
+

.azurepipelines/azure-pipelines-release.yml

@@ -0,0 +1,41 @@
+# MSTICNB Release pipeline
+
+trigger: none
+name: 1ES-MSTICNB-Rel-$(date:yyyyMMdd)$(rev:.r)
+
+resources:
+  repositories:
+  - repository: self
+    type: git
+    ref: main
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates
+  parameters:
+    pool:
+      name: MSSecurity-1ES-Build-Agents-Pool
+      image: MSSecurity-1ES-Windows-2022
+      os: windows
+    stages:
+    - stage: buildTasks
+      displayName: BuildTasks
+      jobs:
+      - job: additionalChecks
+        displayName: AdditionalChecks
+        steps:
+        - task: notice@0
+          displayName: NOTICE File Generator
+          # This fails for external forks
+          condition: not(variables['System.PullRequest.IsFork'])
+    sdl:
+      apiScan:
+        enabled: false
+      policheck:
+        enabled: true
+      bandit:
+        enabled: true
+

.github/ISSUE_TEMPLATE/workflows/python-package.yml

@@ -0,0 +1,180 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MSTICNB CI build and check
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main, release/*]
+  schedule:
+    - cron: "0 0 * * 0,2,4"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      # Print out details about the run
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Dump job context
+        env:
+          JOB_CONTEXT: ${{ toJSON(job) }}
+        run: echo "$JOB_CONTEXT"
+      # end print details
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+            ${{ runner.os }}-pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools
+          if [ -f requirements.txt ]; then
+            python -m pip install -r requirements.txt
+          fi
+          python -m pip install -e .
+      - name: Install test dependencies
+        run: |
+          if [ -f requirements-dev.txt ]; then
+            python -m pip install -r requirements-dev.txt
+          else
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
+            python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
+            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
+          fi
+          python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
+      - name: Pytest
+        env:
+          MAXMIND_AUTH: DUMMY_KEY
+          IPSTACK_AUTH: DUMMY_KEY
+          MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
+          MSTICPY_BUILD_SOURCE: fork
+        run: |
+          pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticnb --cov-report=xml
+        if: ${{ always() }}
+      - name: Upload pytest test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-results-${{ matrix.python-version }}
+          path: junit/test-${{ matrix.python-version }}-results.xml
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
+
+  lint:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    strategy:
+      matrix:
+        python-version: ["3.8"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-lint-${{ hashFiles('requirements.txt') }}
+            ${{ runner.os }}-pip-lint
+            ${{ runner.os }}-pip
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools
+          if [ -f requirements.txt ]; then
+            python -m pip install -r requirements.txt;
+          fi
+          python -m pip install -e .
+      - name: Install test dependencies
+        run: |
+          if [ -f requirements-dev.txt ]; then
+            python -m pip install -r requirements-dev.txt
+          else
+            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
+            python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
+          fi
+      - name: black
+        run: |
+          black --diff --check --exclude venv msticnb
+        if: ${{ always() }}
+      - name: flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 msticnb --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+        if: ${{ always() }}
+      - name: pylint
+        run: |
+          pylint msticnb --disable=duplicate-code --disable=E1135,E1101,E1133
+        if: ${{ always() }}
+      - name: Cache/restore MyPy data
+        id: cache-mypy
+        uses: actions/cache@v3
+        with:
+          # MyPy cache files are stored in `~/.mypy_cache`
+          path: .mypy_cache
+          key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
+            ${{ runner.os }}-build-mypy-${{ github.ref }}
+            ${{ runner.os }}-build-mypy
+      - name: mypy
+        run: |
+          mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticnb
+        if: ${{ always() }}
+      - name: Upload mypy test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: Mypy results ${{ matrix.python-version }}
+          path: junit/mypy-test-${{ matrix.python-version }}-results.xml
+        # Use always() to always run this step to publish test results when there are test failures
+        if: ${{ always() }}
+      - name: flake8
+        run: |
+          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
+        if: ${{ always() }}
+      - name: pydocstyle
+        run: |
+          pydocstyle --convention=numpy msticnb
+        if: ${{ always() }}
+      - name: pyroma
+        run: |
+          pyroma --min 10 .
+        if: ${{ always() }}
+  check_status:
+    runs-on: ubuntu-latest
+    permissions: read-all
+    needs: [build, lint]
+    steps:
+      - name: File build fail issue
+        if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }}
+        uses: dacbd/create-issue-action@v1
+        with:
+          token: ${{ github.token }}
+          title: "Build failed for main branch"
+          body: The build failed on branch ${{ github.ref }}. Please investigate
+          labels: build_break, bug, high_severity

.github/ISSUE_TEMPLATE/workflows/python-publish.yml

@@ -0,0 +1,45 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package to PyPI Prod
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+    permissions: read-all
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install build
+    - name: Build package
+      run: >-
+        python -m
+        build
+        --sdist
+        --wheel
+        --outdir dist/
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@v1.5.1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}