Skip to content

Commit

Permalink
airbyte-ci: dynamic test step tree according to metadata (#38281)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alafanechere
alafanechere authored May 21, 2024
1 parent e5a390d commit e669832
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 6 deletions.
5 changes: 3 additions & 2 deletions airbyte-ci/connectors/pipelines/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,14 +298,14 @@ flowchart TD
#### Options

| Option | Multiple | Default value | Description |
| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--skip-step/-x` | True | | Skip steps by id e.g. `-x unit -x acceptance` |
| `--only-step/-k` | True | | Only run specific steps by id e.g. `-k unit -k acceptance` |
| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. |
| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. |
| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. |
| `--<step-id>.<extra-parameter>=<extra-parameter-value>` | True | | You can pass extra parameters for specific test steps. More details in the extra parameters section below |
| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. |
| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use.

Note:

Expand Down Expand Up @@ -745,6 +745,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:

| Version | PR | Description |
|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
| 4.14.0 | [#38281](https://github.com/airbytehq/airbyte/pull/38281) | Conditionally run test suites according to `connectorTestSuitesOptions` in metadata files. |
| 4.13.3 | [#38221](https://github.com/airbytehq/airbyte/pull/38221) | Add dagster cloud dev deployment pipeline opitions |
| 4.13.2 | [#38246](https://github.com/airbytehq/airbyte/pull/38246) | Remove invalid connector test step options. |
| 4.13.1 | [#38020](https://github.com/airbytehq/airbyte/pull/38020) | Add `auto_merge` as an internal package to test. |
Expand Down
36 changes: 35 additions & 1 deletion airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from __future__ import annotations

from copy import deepcopy
from datetime import datetime
from pathlib import Path
from types import TracebackType
Expand All @@ -15,6 +16,7 @@
from asyncer import asyncify
from dagger import Directory, Platform, Secret
from github import PullRequest
from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID
from pipelines.airbyte_ci.connectors.reports import ConnectorReport
from pipelines.consts import BUILD_PLATFORMS
from pipelines.dagger.actions import secrets
Expand All @@ -29,6 +31,13 @@
from pathlib import Path as NativePath
from typing import Dict, FrozenSet, List, Optional, Sequence

# These test suite names are declared in metadata.yaml files
TEST_SUITE_NAME_TO_STEP_ID = {
"unitTests": CONNECTOR_TEST_STEP_ID.UNIT,
"integrationTests": CONNECTOR_TEST_STEP_ID.INTEGRATION,
"acceptanceTests": CONNECTOR_TEST_STEP_ID.ACCEPTANCE,
}


class ConnectorContext(PipelineContext):
"""The connector context is used to store configuration for a specific connector pipeline run."""
Expand Down Expand Up @@ -140,7 +149,7 @@ def __init__(
ci_gcs_credentials=ci_gcs_credentials,
ci_git_user=ci_git_user,
ci_github_access_token=ci_github_access_token,
run_step_options=run_step_options,
run_step_options=self._skip_metadata_disabled_test_suites(run_step_options),
enable_report_auto_open=enable_report_auto_open,
)

Expand Down Expand Up @@ -286,3 +295,28 @@ async def __aexit__(

def create_slack_message(self) -> str:
raise NotImplementedError

def _get_step_id_to_skip_according_to_metadata(self) -> List[CONNECTOR_TEST_STEP_ID]:
"""The connector metadata have a connectorTestSuitesOptions field.
It allows connector developers to declare the test suites that are enabled for a connector.
This function retrieved enabled test suites according to this field value and returns the test suites steps that are skipped (because they're not declared in this field.)
The skippable test suites steps are declared in TEST_SUITE_NAME_TO_STEP_ID.
Returns:
List[CONNECTOR_TEST_STEP_ID]: List of step ids that should be skipped according to connector metadata.
"""
enabled_test_suites = [option["suite"] for option in self.metadata.get("connectorTestSuitesOptions", [])]
return [step_id for test_suite_name, step_id in TEST_SUITE_NAME_TO_STEP_ID.items() if test_suite_name not in enabled_test_suites]

def _skip_metadata_disabled_test_suites(self, run_step_options: RunStepOptions) -> RunStepOptions:
"""Updated the original run_step_options to skip the disabled test suites according to connector metadata.
Args:
run_step_options (RunStepOptions): Original run step options.
Returns:
RunStepOptions: Updated run step options.
"""
run_step_options = deepcopy(run_step_options)
run_step_options.skip_steps += self._get_step_id_to_skip_according_to_metadata()
return run_step_options
2 changes: 1 addition & 1 deletion airbyte-ci/connectors/pipelines/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "pipelines"
version = "4.13.3"
version = "4.14.0"
description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
authors = ["Airbyte <contact@airbyte.io>"]

Expand Down
4 changes: 2 additions & 2 deletions airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

import asyncclick as click
import pytest
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.dagger.actions.python import common
from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles

pytestmark = [
pytest.mark.anyio,
Expand All @@ -16,7 +16,7 @@
def connector_context(dagger_client):
context = ConnectorContext(
pipeline_name="test",
connector="source-faker",
connector=ConnectorWithModifiedFiles("source-faker", modified_files={}),
git_branch="test",
git_revision="test",
diffed_branch="test",
Expand Down

0 comments on commit e669832

Please sign in to comment.