Skip to content

MSTICPy CI build and check #33

MSTICPy CI build and check

MSTICPy CI build and check #33

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: MSTICPy CI build and check
on:
push:
branches: [main]
pull_request:
branches: [main, release/*]
schedule:
- cron: "0 0 * * 0,2,4"
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
steps:
# Print out details about the run
- name: Dump GitHub context
env:
GITHUB_CONTEXT: ${{ toJSON(github) }}
run: echo "$GITHUB_CONTEXT"
- name: Dump job context
env:
JOB_CONTEXT: ${{ toJSON(job) }}
run: echo "$JOB_CONTEXT"
# end print details
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements file
key: ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
restore-keys: |
${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
${{ runner.os }}-pip
- name: Install dependencies
run: |
python -m pip install --upgrade pip wheel setuptools
if [ -f requirements-all.txt ]; then
python -m pip install -r requirements-all.txt
elif [ -f requirements.txt ]; then
python -m pip install -r requirements.txt;
fi
python -m pip install -e .
- name: Install test dependencies
run: |
if [ -f requirements-dev.txt ]; then
python -m pip install -r requirements-dev.txt
else
echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
fi
python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
- name: Prepare test dummy data
run: |
mkdir ~/.msticpy
mkdir ~/.msticpy/mordor
cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
touch ~/.msticpy/GeoLite2-City.mmdb
cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor
touch ~/.msticpy/mordor/mitre_tact_cache.pkl
touch ~/.msticpy/mordor/mitre_tech_cache.pkl
touch ~/.msticpy/mordor/mordor_cache.pkl
- name: Pytest
env:
MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }}
IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }}
MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
MSTICPY_BUILD_SOURCE: fork
run: |
pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml
if: ${{ always() }}
- name: Upload pytest test results
uses: actions/upload-artifact@v3
with:
name: pytest-results-${{ matrix.python-version }}
path: junit/test-${{ matrix.python-version }}-results.xml
# Use always() to always run this step to publish test results when there are test failures
if: ${{ always() }}
lint:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.8"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements file
key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
restore-keys: |
${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
${{ runner.os }}-pip-lint
${{ runner.os }}-pip
- name: Install dependencies
run: |
python -m pip install --upgrade pip wheel setuptools
if [ -f requirements-all.txt ]; then
python -m pip install -r requirements-all.txt
elif [ -f requirements.txt ]; then
python -m pip install -r requirements.txt;
fi
python -m pip install -e .
- name: Install test dependencies
run: |
if [ -f requirements-dev.txt ]; then
python -m pip install -r requirements-dev.txt
else
echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
fi
- name: black
run: |
black --diff --check --exclude venv msticpy
if: ${{ always() }}
- name: flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 msticpy --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
if: ${{ always() }}
- name: pylint
run: |
pylint msticpy --disable=duplicate-code --disable=E1135,E1101,E1133
if: ${{ always() }}
- name: Cache/restore MyPy data
id: cache-mypy
uses: actions/cache@v3
with:
# MyPy cache files are stored in `~/.mypy_cache`
path: .mypy_cache
key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
${{ runner.os }}-build-mypy-${{ github.ref }}
${{ runner.os }}-build-mypy
- name: mypy
run: |
mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy
if: ${{ always() }}
- name: Upload mypy test results
uses: actions/upload-artifact@v3
with:
name: Mypy results ${{ matrix.python-version }}
path: junit/mypy-test-${{ matrix.python-version }}-results.xml
# Use always() to always run this step to publish test results when there are test failures
if: ${{ always() }}
- name: bandit
run: |
bandit -x tests -r -s B303,B404,B603,B607,B608 msticpy
if: ${{ always() }}
- name: flake8
run: |
flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
if: ${{ always() }}
- name: pydocstyle
run: |
pydocstyle --convention=numpy msticpy
if: ${{ always() }}
- name: pyroma
run: |
pyroma --min 10 .
if: ${{ always() }}
check_status:
runs-on: ubuntu-latest
needs: [build, lint]
steps:
- name: File build fail issue
if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }}
uses: dacbd/create-issue-action@v1
with:
token: ${{ github.token }}
title: "Build failed for main branch"
body: The build failed on branch ${{ github.ref }}. Please investigate
labels: build_break, bug, high_severity