OX-4086 | Make sarif.py support function calls as sources/sinks #22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The goals of this workflow are to check that: | |
| # - we can build semgrep-core and semgrep | |
| # - we can build a Docker image as well as Linux and MacOS binaries | |
| # - all our tests (the one in semgrep-core and the one in semgrep-cli) are passing | |
| # - we don't have any perf regressions in our benchmarks | |
| name: tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| paths-ignore: | |
| - "**.md" | |
| push: | |
| branches: | |
| - develop | |
| paths-ignore: | |
| - "**.md" | |
| jobs: | |
| #TODO: do we need this job now that we have build-test-core-x86.yaml? | |
| test-core: | |
| name: test semgrep-core | |
| runs-on: ubuntu-22.04 | |
| container: returntocorp/ocaml:alpine-2023-06-16 | |
| env: | |
| HOME: /root | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| persist-credentials: false | |
| - name: Build semgrep-core | |
| run: | | |
| eval $(opam env) | |
| make install-deps-ALPINE-for-semgrep-core | |
| make install-deps-for-semgrep-core | |
| make core | |
| # needed for the e2e test below | |
| make core-install | |
| - name: Test semgrep-core | |
| run: | | |
| eval $(opam env) | |
| START=`date +%s` | |
| make core-test | |
| make core-e2etest | |
| END=`date +%s` | |
| TEST_RUN_TIME=$((END-START)) | |
| curl --fail -L -X POST "https://dashboard.semgrep.dev/api/metric/semgrep.core.test-run-time-seconds.num" -d "$TEST_RUN_TIME" | |
| - name: Report Number of Tests Stats | |
| if: github.ref == 'refs/heads/develop' | |
| run: ./scripts/report_test_metrics.sh | |
| # TODO: move this to a stable host for more reliable results. | |
| # | |
| # It's not clear how to push the stats only when "on the main | |
| # branch". The GitHub Actions documentation is unhelpful. So we | |
| # keep things simple and publish the results every time. | |
| # | |
| - name: Publish match performance | |
| run: | | |
| # This runs a short test suite to track the match performance | |
| # of semgrep-core over time. The results are pushed to the | |
| # dashboard at https://dashboard.semgrep.dev/ | |
| # | |
| opam exec -- make report-perf-matching | |
| #TODO: merge with the previous job (which should be merged in build-test-core-x86.yaml) | |
| test-osemgrep: | |
| name: test osemgrep | |
| runs-on: ubuntu-22.04 | |
| container: returntocorp/ocaml:alpine-2023-06-16 | |
| env: | |
| HOME: /root | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| persist-credentials: false | |
| - name: Build semgrep-core | |
| run: | | |
| eval $(opam env) | |
| make install-deps-ALPINE-for-semgrep-core | |
| make install-deps-for-semgrep-core | |
| make core | |
| - name: Install osemgrep | |
| run: | | |
| eval $(opam env) | |
| make core-install | |
| # needed for pipenv install to work below | |
| cp bin/semgrep-core /usr/bin/ | |
| - name: Install Python dependencies | |
| run: | | |
| make install-deps-ALPINE-for-pysemgrep | |
| (cd cli; pipenv install --dev) | |
| - name: Run pytest for osemgrep known passing tests | |
| working-directory: cli | |
| run: | | |
| make osempass | |
| test-cli: | |
| name: test semgrep-cli | |
| runs-on: ubuntu-22.04 | |
| needs: [build-test-core-x86] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| strategy: | |
| matrix: | |
| python: ["3.7", "3.8", "3.9", "3.10", "3.11"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| #TODO? just use submodule:true above instead of this? | |
| - name: Fetch semgrep-cli submodules | |
| run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/semgrep_interfaces | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| cache: pipenv | |
| - run: pip install pipenv==2022.6.7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ocaml-build-artifacts-release | |
| - name: Install artifacts | |
| run: | | |
| tar xf ocaml-build-artifacts.tgz | |
| sudo cp ocaml-build-artifacts/bin/* /usr/bin | |
| - name: Install Python dependencies | |
| working-directory: cli | |
| run: pipenv install --dev | |
| - name: Run pytest | |
| working-directory: cli | |
| run: | | |
| # tests should simulate CI environment iff they need one | |
| unset CI | |
| unset "${!GITHUB_@}" | |
| pipenv run pytest -n auto -vv --snapshot-update --allow-snapshot-deletion | |
| # because of the fail-fast setting, we expect only the fastest failing job to get to the steps below | |
| - name: Prepare repo for snapshot commit | |
| if: failure() | |
| run: | | |
| # the commit step that follows will fail to fetch the pfff submodule | |
| # (perhaps because of the github token's permissions) | |
| # so we disable recursive fetching | |
| git config fetch.recurseSubmodules false | |
| # Because we are not persisting creds in the checkout step, we must create the creds file | |
| # when it's needed to push up snapshot changes. | |
| # Note that this is only done after tests are run, so that tests do not have access to the token | |
| - name: Configure git creds for push | |
| id: configure-creds | |
| if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository)) | |
| run: | | |
| echo "machine github.com" >> ~/.netrc | |
| echo "login ${{ github.repository }}" >> ~/.netrc | |
| echo "password ${{ secrets.GITHUB_TOKEN }}" >> ~/.netrc | |
| - name: Commit snapshot updates | |
| id: snapshot-commit | |
| if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository)) | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| add: cli/tests/e2e/snapshots | |
| default_author: github_actions | |
| message: "Update pytest snapshots" | |
| new_branch: snapshot-updates-${{ github.run_id }}-${{ github.run_attempt }} | |
| - name: Remove Credentials | |
| id: remove-creds | |
| if: failure() && github.event_name == 'pull_request' && (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository)) | |
| run: rm ~/.netrc | |
| - name: Comment about any snapshot updates | |
| if: failure() && steps.snapshot-commit.outputs.pushed == 'true' | |
| run: | | |
| echo ":camera_flash: The pytest shapshots changed in your PR." >> /tmp/message.txt | |
| echo "Please carefully review these changes and make sure they are intended:" >> /tmp/message.txt | |
| echo >> /tmp/message.txt | |
| echo "1. Review the changes at https://github.com/returntocorp/semgrep/commit/${{ steps.snapshot-commit.outputs.commit_long_sha }}" >> /tmp/message.txt | |
| echo "2. Accept the new snapshots with" >> /tmp/message.txt | |
| echo >> /tmp/message.txt | |
| echo " git fetch origin && git cherry-pick ${{ steps.snapshot-commit.outputs.commit_sha }} && git push" >> /tmp/message.txt | |
| gh pr comment ${{ github.event.pull_request.number }} --body-file /tmp/message.txt | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # These tests aren't run by default by pytest. | |
| # To reproduce errors locally, use: | |
| # $ cd cli/tests | |
| # $ make qa | |
| # | |
| # TODO: if you know this, please explain what the code below is meant | |
| # to achieve and how to make sure it works. | |
| # | |
| test-qa: | |
| name: quality assurance on semgrep | |
| runs-on: ubuntu-22.04 | |
| needs: [build-test-core-x86] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| split: [1, 2, 3, 4] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Fetch semgrep-cli submodules | |
| run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/semgrep_interfaces tests/semgrep-rules | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| cache: pipenv | |
| - run: pip install pipenv==2022.6.7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ocaml-build-artifacts-release | |
| - name: Install artifacts | |
| run: | | |
| tar xf ocaml-build-artifacts.tgz | |
| sudo cp ocaml-build-artifacts/bin/* /usr/bin | |
| - name: Install semgrep | |
| working-directory: cli | |
| run: | | |
| export PATH=/github/home/.local/bin:$PATH | |
| pipenv install --dev | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/qa-public-repos | |
| key: qa-public-repos-${{ hashFiles('semgrep/tests/qa/*public_repos*') }}-${{ matrix.split }} | |
| - run: | | |
| mkdir -p ~/.cache/qa-public-repos | |
| touch ~/.cache/qa-public-repos/ok | |
| - name: Test semgrep | |
| working-directory: cli | |
| run: | | |
| export PATH=/github/home/.local/bin:$PATH | |
| pipenv run pytest -n auto -vv --tb=short --splits 4 --group ${{ matrix.split }} tests/qa | |
| env: | |
| QA_TESTS_CACHE_PATH: ~/.cache/qa-public-repos | |
| # Run abbreviated version of benchmarks to check that they work | |
| benchmarks-lite: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-test-core-x86] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Fetch semgrep-cli submodules | |
| run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/semgrep_interfaces | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.7" | |
| cache: pipenv | |
| - run: pip install pipenv==2022.6.7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ocaml-build-artifacts-release | |
| - name: Install artifacts | |
| run: | | |
| tar xf ocaml-build-artifacts.tgz | |
| sudo cp ocaml-build-artifacts/bin/* /usr/bin | |
| - name: Install cli dependencies | |
| working-directory: cli | |
| run: pipenv install --dev | |
| - name: Test dummy benchmarks on latest | |
| working-directory: cli | |
| run: | | |
| pipenv run semgrep --version | |
| pipenv run python -m semgrep --version | |
| pipenv run semgrep-core -version | |
| pipenv run python3 ../perf/run-benchmarks --dummy | |
| # Run each benchmark twice to decrease effect of natural variance | |
| benchmarks-full: | |
| runs-on: ubuntu-22.04 | |
| needs: [build-test-core-x86] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| - name: Fetch semgrep-cli submodules | |
| run: git submodule update --init --recursive --recommend-shallow cli/src/semgrep/semgrep_interfaces | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.7" | |
| cache: pipenv | |
| - run: pip install pipenv==2022.6.7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ocaml-build-artifacts-release | |
| - name: Install artifacts | |
| run: | | |
| tar xf ocaml-build-artifacts.tgz | |
| sudo cp ocaml-build-artifacts/bin/* /usr/bin | |
| - name: Install cli dependencies | |
| working-directory: cli | |
| run: pipenv install --dev | |
| - name: Run perf benchmark | |
| run: scripts/run-benchmarks.sh ${{ secrets.GITHUB_TOKEN }} ${{ github.event.number }} | |
| - name: Run python performance tests | |
| working-directory: cli | |
| run: pipenv run pytest tests/performance | |
| build-test-docker: | |
| uses: ./.github/workflows/build-test-docker.yaml | |
| secrets: inherit | |
| with: | |
| docker-tags: | | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=ref,event=pr | |
| type=ref,event=branch | |
| type=sha,event=branch | |
| type=edge | |
| artifact-name: image-test | |
| repository-name: ${{ github.repository }} | |
| file: Dockerfile | |
| enable-tests: true | |
| push-docker: | |
| needs: [build-test-docker] | |
| uses: ./.github/workflows/push-docker.yaml | |
| if: github.ref == 'refs/heads/develop' || (github.actor != 'dependabot[bot]' && !(github.event.pull_request.head.repo.full_name != github.repository)) | |
| secrets: inherit | |
| with: | |
| artifact-name: image-test | |
| repository-name: ${{ github.repository }} | |
| dry-run: false | |
| test-semgrep-pro: | |
| needs: [build-test-docker, push-docker] | |
| uses: ./.github/workflows/test-semgrep-pro.yaml | |
| if: github.ref == 'refs/heads/develop' || github.event.pull_request.head.repo.full_name == github.repository # only returntocorp has the necessary credentials to access semgrep pro | |
| secrets: inherit | |
| with: | |
| artifact-name: image-test | |
| repository-name: ${{ github.repository }} | |
| build-test-core-x86: | |
| uses: ./.github/workflows/build-test-core-x86.yaml | |
| secrets: inherit | |
| build-test-manylinux-x86: | |
| needs: [build-test-core-x86] | |
| uses: ./.github/workflows/build-test-manylinux-x86.yaml | |
| secrets: inherit | |
| build-test-manylinux-aarch64: | |
| needs: [build-test-docker] | |
| uses: ./.github/workflows/build-test-manylinux-aarch64.yaml | |
| secrets: inherit | |
| build-test-osx-x86: | |
| uses: ./.github/workflows/build-test-osx-x86.yaml | |
| secrets: inherit | |
| build-test-osx-arm64: | |
| uses: ./.github/workflows/build-test-osx-arm64.yaml | |
| secrets: inherit | |
| build-test-javascript: | |
| uses: ./.github/workflows/build-test-javascript.yaml | |
| secrets: inherit |