Skip to content

Commit

Permalink
CLI sync S3 end to end integration test (#3001)
Browse files Browse the repository at this point in the history
Co-authored-by: Nick Z <2420177+nickzelei@users.noreply.github.com>
  • Loading branch information
alishakawaguchi and nickzelei authored Dec 5, 2024
1 parent 570ecd8 commit 7157f84
Show file tree
Hide file tree
Showing 30 changed files with 2,431 additions and 147 deletions.
15 changes: 15 additions & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ jobs:
integration-tests:
name: Integration Tests
runs-on: ubuntu-latest
permissions:
id-token: write # required for requesting JWT for use with retrieving AWS creds
contents: read # required for actions/checkout
env:
AWS_REGION: "us-west-2"
steps:
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -90,11 +95,21 @@ jobs:
go-version-file: go.mod
cache-dependency-path: go.sum

- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: ${{ env.AWS_REGION }}
role-to-assume: ${{ vars.INTEGRATION_TEST_AWS_ROLE_ARN }}
role-session-name: NeosyncCiIntegrationTests

- name: Run Integration Tests
run: |
go test -race -timeout 1800s -coverprofile=integration-coverage.out -covermode=atomic -run TestIntegrationTestSuite ./...
env:
INTEGRATION_TESTS_ENABLED: 1
S3_INTEGRATION_TESTS_ENABLED: 1
TEST_S3_REGION: ${{ env.AWS_REGION }}
TEST_S3_BUCKET: ${{ vars.INTEGRATION_TEST_BUCKET_NAME }}

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
Expand Down
30 changes: 30 additions & 0 deletions backend/pkg/integration-test/integration-test-util.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,36 @@ func CreateMysqlConnection(
return resp.Msg.GetConnection()
}

func CreateS3Connection(
ctx context.Context,
t *testing.T,
connclient mgmtv1alpha1connect.ConnectionServiceClient,
accountId, name string,
bucket string,
region *string,
) *mgmtv1alpha1.Connection {
resp, err := connclient.CreateConnection(
ctx,
connect.NewRequest(&mgmtv1alpha1.CreateConnectionRequest{
AccountId: accountId,
Name: name,
ConnectionConfig: &mgmtv1alpha1.ConnectionConfig{
Config: &mgmtv1alpha1.ConnectionConfig_AwsS3Config{
AwsS3Config: &mgmtv1alpha1.AwsS3ConnectionConfig{
Bucket: bucket,
PathPrefix: nil,
Region: region,
Endpoint: nil,
Credentials: nil,
},
},
},
}),
)
RequireNoErrResp(t, resp, err)
return resp.Msg.GetConnection()
}

func SetUser(ctx context.Context, t *testing.T, client mgmtv1alpha1connect.UserAccountServiceClient) string {
resp, err := client.SetUser(ctx, connect.NewRequest(&mgmtv1alpha1.SetUserRequest{}))
RequireNoErrResp(t, resp, err)
Expand Down
22 changes: 22 additions & 0 deletions backend/pkg/integration-test/integration-test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
promapiv1mock "github.com/nucleuscloud/neosync/internal/mocks/github.com/prometheus/client_golang/api/prometheus/v1"
"github.com/nucleuscloud/neosync/internal/testutil"
tcpostgres "github.com/nucleuscloud/neosync/internal/testutil/testcontainers/postgres"
"github.com/stretchr/testify/mock"
)

var (
Expand Down Expand Up @@ -399,6 +400,27 @@ func (s *NeosyncApiTestClient) Setup(ctx context.Context, t testing.TB) error {
return nil
}

func (s *NeosyncApiTestClient) MockTemporalForCreateJob(returnId string) {
s.Mocks.TemporalClientManager.
On(
"DoesAccountHaveNamespace", mock.Anything, mock.Anything, mock.Anything,
).
Return(true, nil).
Once()
s.Mocks.TemporalClientManager.
On(
"GetSyncJobTaskQueue", mock.Anything, mock.Anything, mock.Anything,
).
Return("sync-job", nil).
Once()
s.Mocks.TemporalClientManager.
On(
"CreateSchedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
).
Return(returnId, nil).
Once()
}

func (s *NeosyncApiTestClient) InitializeTest(ctx context.Context, t testing.TB) error {
err := neomigrate.Up(ctx, s.Pgcontainer.URL, s.migrationsDir, testutil.GetTestLogger(t))
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/google/uuid"
mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1"
"github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

Expand All @@ -29,7 +28,7 @@ func (s *IntegrationTestSuite) Test_CreateJob_Ok() {
srcconn := s.createPostgresConnection(s.UnauthdClients.Connections, accountId, "source", "test")
destconn := s.createPostgresConnection(s.UnauthdClients.Connections, accountId, "dest", "test2")

s.mockTemporalForCreateJob("test-id")
s.MockTemporalForCreateJob("test-id")

resp, err := s.UnauthdClients.Jobs.CreateJob(s.ctx, connect.NewRequest(&mgmtv1alpha1.CreateJobRequest{
AccountId: accountId,
Expand Down Expand Up @@ -59,27 +58,6 @@ func (s *IntegrationTestSuite) Test_CreateJob_Ok() {
require.NotNil(s.T(), resp.Msg.GetJob())
}

func (s *IntegrationTestSuite) mockTemporalForCreateJob(returnId string) {
s.Mocks.TemporalClientManager.
On(
"DoesAccountHaveNamespace", mock.Anything, mock.Anything, mock.Anything,
).
Return(true, nil).
Once()
s.Mocks.TemporalClientManager.
On(
"GetSyncJobTaskQueue", mock.Anything, mock.Anything, mock.Anything,
).
Return("sync-job", nil).
Once()
s.Mocks.TemporalClientManager.
On(
"CreateSchedule", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
).
Return(returnId, nil).
Once()
}

func (s *IntegrationTestSuite) Test_JobService_JobHooks() {
t := s.T()
ctx := s.ctx
Expand Down Expand Up @@ -131,7 +109,7 @@ func (s *IntegrationTestSuite) Test_JobService_JobHooks() {
srcconn := s.createPostgresConnection(s.NeosyncCloudClients.GetConnectionClient(testAuthUserId), accountId, "source", "test")
destconn := s.createPostgresConnection(s.NeosyncCloudClients.GetConnectionClient(testAuthUserId), accountId, "dest", "test2")

s.mockTemporalForCreateJob("test-id")
s.MockTemporalForCreateJob("test-id")
jobResp, err := client.CreateJob(ctx, connect.NewRequest(&mgmtv1alpha1.CreateJobRequest{
JobName: "cloud-testjob-1",
AccountId: accountId,
Expand Down
Loading

0 comments on commit 7157f84

Please sign in to comment.