diff --git a/backend/src/apiserver/server/api_converter.go b/backend/src/apiserver/server/api_converter.go index 451013ea050e..426ea05b807a 100644 --- a/backend/src/apiserver/server/api_converter.go +++ b/backend/src/apiserver/server/api_converter.go @@ -302,7 +302,7 @@ func toApiTrigger(trigger model.Trigger) *api.Trigger { return &api.Trigger{Trigger: &api.Trigger_CronSchedule{CronSchedule: &cronSchedule}} } - if trigger.IntervalSecond != nil { + if trigger.IntervalSecond != nil && *trigger.IntervalSecond != 0 { var periodicSchedule api.PeriodicSchedule periodicSchedule.IntervalSecond = *trigger.IntervalSecond if trigger.PeriodicScheduleStartTimeInSec != nil { diff --git a/backend/test/integration/BUILD.bazel b/backend/test/integration/BUILD.bazel index 30f148d529ce..2c0c7f07215e 100644 --- a/backend/test/integration/BUILD.bazel +++ b/backend/test/integration/BUILD.bazel @@ -8,6 +8,7 @@ go_test( "pipeline_api_test.go", "pipeline_version_api_test.go", "run_api_test.go", + "upgrade_test.go", "visualization_api_test.go", ], embed = [":go_default_library"], @@ -32,6 +33,7 @@ go_test( "@com_github_go_openapi_strfmt//:go_default_library", "@com_github_golang_glog//:go_default_library", "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", "@com_github_stretchr_testify//suite:go_default_library", "@io_k8s_apimachinery//pkg/util/yaml:go_default_library", "@org_golang_google_grpc//:go_default_library", diff --git a/backend/test/integration/flags.go b/backend/test/integration/flags.go index dcd9dabc5c73..6e688bf3ee78 100644 --- a/backend/test/integration/flags.go +++ b/backend/test/integration/flags.go @@ -8,6 +8,7 @@ import ( var namespace = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to") var initializeTimeout = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization") var runIntegrationTests = flag.Bool("runIntegrationTests", false, "Whether to also run integration tests that call the service") +var runUpgradeTests = flag.Bool("runUpgradeTests", false, "Whether to run upgrade tests") /** * Differences in dev mode: diff --git a/backend/test/integration/upgrade_test.go b/backend/test/integration/upgrade_test.go new file mode 100644 index 000000000000..d634c273a2d8 --- /dev/null +++ b/backend/test/integration/upgrade_test.go @@ -0,0 +1,432 @@ +package integration + +import ( + "io/ioutil" + "testing" + "time" + + "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1" + "github.com/ghodss/yaml" + "github.com/golang/glog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + experimentParams "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_client/experiment_service" + "github.com/kubeflow/pipelines/backend/api/go_http_client/experiment_model" + jobparams "github.com/kubeflow/pipelines/backend/api/go_http_client/job_client/job_service" + "github.com/kubeflow/pipelines/backend/api/go_http_client/job_model" + pipelineParams "github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_client/pipeline_service" + "github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_model" + uploadParams "github.com/kubeflow/pipelines/backend/api/go_http_client/pipeline_upload_client/pipeline_upload_service" + runParams "github.com/kubeflow/pipelines/backend/api/go_http_client/run_client/run_service" + "github.com/kubeflow/pipelines/backend/api/go_http_client/run_model" + "github.com/kubeflow/pipelines/backend/src/common/client/api_server" + "github.com/kubeflow/pipelines/backend/src/common/util" + "github.com/kubeflow/pipelines/backend/test" +) + +// Methods are organized into two types: "prepare" and "verify". +// "prepare" tests setup resources before upgrade +// "verify" tests verifies resources are expected after upgrade +type UpgradeTests struct { + suite.Suite + namespace string + experimentClient *api_server.ExperimentClient + pipelineClient *api_server.PipelineClient + pipelineUploadClient *api_server.PipelineUploadClient + runClient *api_server.RunClient + jobClient *api_server.JobClient +} + +func TestUpgrade(t *testing.T) { + suite.Run(t, new(UpgradeTests)) +} + +func (s *UpgradeTests) TestPrepare() { + t := s.T() + + test.DeleteAllJobs(s.jobClient, t) + test.DeleteAllRuns(s.runClient, t) + test.DeleteAllPipelines(s.pipelineClient, t) + test.DeleteAllExperiments(s.experimentClient, t) + + s.PrepareExperiments() + s.PreparePipelines() + s.PrepareRuns() + s.PrepareJobs() +} + +func (s *UpgradeTests) TestVerify() { + s.VerifyExperiments() + s.VerifyPipelines() + s.VerifyRuns() + s.VerifyJobs() +} + +// Check the namespace have ML job installed and ready +func (s *UpgradeTests) SetupSuite() { + // Integration tests also run these tests to first ensure they work, so that + // when integration tests pass and upgrade tests fail, we know for sure + // upgrade process went wrong somehow. + if !(*runIntegrationTests || *runUpgradeTests) { + s.T().SkipNow() + return + } + + err := test.WaitForReady(*namespace, *initializeTimeout) + if err != nil { + glog.Exitf("Failed to initialize test. Error: %v", err) + } + s.namespace = *namespace + clientConfig := test.GetClientConfig(*namespace) + s.experimentClient, err = api_server.NewExperimentClient(clientConfig, false) + if err != nil { + glog.Exitf("Failed to get experiment client. Error: %v", err) + } + s.pipelineUploadClient, err = api_server.NewPipelineUploadClient(clientConfig, false) + if err != nil { + glog.Exitf("Failed to get pipeline upload client. Error: %s", err.Error()) + } + s.pipelineClient, err = api_server.NewPipelineClient(clientConfig, false) + if err != nil { + glog.Exitf("Failed to get pipeline client. Error: %s", err.Error()) + } + s.runClient, err = api_server.NewRunClient(clientConfig, false) + if err != nil { + glog.Exitf("Failed to get run client. Error: %s", err.Error()) + } + s.jobClient, err = api_server.NewJobClient(clientConfig, false) + if err != nil { + glog.Exitf("Failed to get job client. Error: %s", err.Error()) + } +} + +func (s *UpgradeTests) TearDownSuite() { + if *runIntegrationTests { + t := s.T() + + // Clean up after the suite to unblock other tests. (Not needed for upgrade + // tests because it needs changes in prepare tests to persist and verified + // later.) + test.DeleteAllExperiments(s.experimentClient, t) + test.DeleteAllPipelines(s.pipelineClient, t) + test.DeleteAllRuns(s.runClient, t) + test.DeleteAllJobs(s.jobClient, t) + } +} + +func (s *UpgradeTests) PrepareExperiments() { + t := s.T() + + /* ---------- Create a new experiment ---------- */ + experiment := &experiment_model.APIExperiment{Name: "training", Description: "my first experiment"} + _, err := s.experimentClient.Create(&experimentParams.CreateExperimentParams{ + Body: experiment, + }) + require.Nil(t, err) + + /* ---------- Create a few more new experiment ---------- */ + // This ensures they can be sorted by create time in expected order. + time.Sleep(1 * time.Second) + experiment = &experiment_model.APIExperiment{Name: "prediction", Description: "my second experiment"} + _, err = s.experimentClient.Create(&experimentParams.CreateExperimentParams{ + Body: experiment, + }) + require.Nil(t, err) + + time.Sleep(1 * time.Second) + experiment = &experiment_model.APIExperiment{Name: "moonshot", Description: "my third experiment"} + _, err = s.experimentClient.Create(&experimentParams.CreateExperimentParams{ + Body: experiment, + }) + require.Nil(t, err) +} + +func (s *UpgradeTests) VerifyExperiments() { + t := s.T() + + /* ---------- Verify list experiments sorted by creation time ---------- */ + experiments, _, _, err := s.experimentClient.List( + &experimentParams.ListExperimentParams{SortBy: util.StringPointer("created_at")}) + require.Nil(t, err) + // after upgrade, default experiment may be inserted, but the oldest 3 + // experiments should be the ones created in this test + require.True(t, len(experiments) >= 3) + + assert.Equal(t, "training", experiments[0].Name) + assert.Equal(t, "my first experiment", experiments[0].Description) + assert.NotEmpty(t, experiments[0].ID) + assert.NotEmpty(t, experiments[0].CreatedAt) + + assert.Equal(t, "prediction", experiments[1].Name) + assert.Equal(t, "my second experiment", experiments[1].Description) + assert.NotEmpty(t, experiments[1].ID) + assert.NotEmpty(t, experiments[1].CreatedAt) + + assert.Equal(t, "moonshot", experiments[2].Name) + assert.Equal(t, "my third experiment", experiments[2].Description) + assert.NotEmpty(t, experiments[2].ID) + assert.NotEmpty(t, experiments[2].CreatedAt) +} + +func (s *UpgradeTests) PreparePipelines() { + t := s.T() + + test.DeleteAllPipelines(s.pipelineClient, t) + + /* ---------- Upload pipelines YAML ---------- */ + argumentYAMLPipeline, err := s.pipelineUploadClient.UploadFile("../resources/arguments-parameters.yaml", uploadParams.NewUploadPipelineParams()) + require.Nil(t, err) + assert.Equal(t, "arguments-parameters.yaml", argumentYAMLPipeline.Name) + + /* ---------- Import pipeline YAML by URL ---------- */ + time.Sleep(1 * time.Second) + sequentialPipeline, err := s.pipelineClient.Create(&pipelineParams.CreatePipelineParams{ + Body: &pipeline_model.APIPipeline{Name: "sequential", URL: &pipeline_model.APIURL{ + PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/sequential.yaml"}}}) + require.Nil(t, err) + assert.Equal(t, "sequential", sequentialPipeline.Name) + + /* ---------- Upload pipelines zip ---------- */ + time.Sleep(1 * time.Second) + argumentUploadPipeline, err := s.pipelineUploadClient.UploadFile( + "../resources/arguments.pipeline.zip", &uploadParams.UploadPipelineParams{Name: util.StringPointer("zip-arguments-parameters")}) + require.Nil(t, err) + assert.Equal(t, "zip-arguments-parameters", argumentUploadPipeline.Name) + + /* ---------- Import pipeline tarball by URL ---------- */ + time.Sleep(1 * time.Second) + argumentUrlPipeline, err := s.pipelineClient.Create(&pipelineParams.CreatePipelineParams{ + Body: &pipeline_model.APIPipeline{URL: &pipeline_model.APIURL{ + PipelineURL: "https://storage.googleapis.com/ml-pipeline-dataset/arguments.pipeline.zip"}}}) + require.Nil(t, err) + assert.Equal(t, "arguments.pipeline.zip", argumentUrlPipeline.Name) + + time.Sleep(1 * time.Second) +} + +func (s *UpgradeTests) VerifyPipelines() { + t := s.T() + + /* ---------- Verify list pipeline sorted by creation time ---------- */ + pipelines, _, _, err := s.pipelineClient.List( + &pipelineParams.ListPipelinesParams{SortBy: util.StringPointer("created_at")}) + require.Nil(t, err) + // During upgrade, default pipelines may be installed, so we only verify the + // 4 oldest pipelines here. + assert.True(t, len(pipelines) >= 4) + assert.Equal(t, "arguments-parameters.yaml", pipelines[0].Name) + assert.Equal(t, "sequential", pipelines[1].Name) + assert.Equal(t, "zip-arguments-parameters", pipelines[2].Name) + assert.Equal(t, "arguments.pipeline.zip", pipelines[3].Name) + + verifyPipeline(t, pipelines[0]) + + /* ---------- Verify get template works ---------- */ + template, err := s.pipelineClient.GetTemplate(&pipelineParams.GetTemplateParams{ID: pipelines[0].ID}) + require.Nil(t, err) + expected, err := ioutil.ReadFile("../resources/arguments-parameters.yaml") + require.Nil(t, err) + var expectedWorkflow v1alpha1.Workflow + err = yaml.Unmarshal(expected, &expectedWorkflow) + assert.Equal(t, expectedWorkflow, *template) +} + +func (s *UpgradeTests) PrepareRuns() { + t := s.T() + + helloWorldPipeline := s.getHelloWorldPipeline(true) + helloWorldExperiment := s.getHelloWorldExperiment(true) + if helloWorldExperiment == nil { + helloWorldExperiment = s.createHelloWorldExperiment() + } + + hello2 := s.getHelloWorldExperiment(true) + require.Equal(t, hello2, helloWorldExperiment) + + /* ---------- Create a new hello world run by specifying pipeline ID ---------- */ + createRunRequest := &runParams.CreateRunParams{Body: &run_model.APIRun{ + Name: "hello world", + Description: "this is hello world", + PipelineSpec: &run_model.APIPipelineSpec{ + PipelineID: helloWorldPipeline.ID, + }, + ResourceReferences: []*run_model.APIResourceReference{ + {Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: helloWorldExperiment.ID}, + Name: helloWorldExperiment.Name, Relationship: run_model.APIRelationshipOWNER}, + }, + }} + _, _, err := s.runClient.Create(createRunRequest) + require.Nil(t, err) +} + +func (s *UpgradeTests) VerifyRuns() { + t := s.T() + + /* ---------- List the runs, sorted by creation time ---------- */ + runs, _, _, err := s.runClient.List( + &runParams.ListRunsParams{SortBy: util.StringPointer("created_at")}) + require.Nil(t, err) + require.True(t, len(runs) >= 1) + require.Equal(t, "hello world", runs[0].Name) + + /* ---------- Get hello world run ---------- */ + helloWorldRunDetail, _, err := s.runClient.Get(&runParams.GetRunParams{RunID: runs[0].ID}) + require.Nil(t, err) + checkHelloWorldRunDetail(t, helloWorldRunDetail) +} + +func (s *UpgradeTests) PrepareJobs() { + t := s.T() + + pipeline := s.getHelloWorldPipeline(true) + experiment := s.getHelloWorldExperiment(true) + + /* ---------- Create a new hello world job by specifying pipeline ID ---------- */ + createJobRequest := &jobparams.CreateJobParams{Body: &job_model.APIJob{ + Name: "hello world", + Description: "this is hello world", + PipelineSpec: &job_model.APIPipelineSpec{ + PipelineID: pipeline.ID, + }, + ResourceReferences: []*job_model.APIResourceReference{ + {Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypeEXPERIMENT, ID: experiment.ID}, + Relationship: job_model.APIRelationshipOWNER}, + }, + MaxConcurrency: 10, + Enabled: true, + NoCatchup: true, + }} + _, err := s.jobClient.Create(createJobRequest) + require.Nil(t, err) +} + +func (s *UpgradeTests) VerifyJobs() { + t := s.T() + + pipeline := s.getHelloWorldPipeline(false) + experiment := s.getHelloWorldExperiment(false) + + /* ---------- Get hello world job ---------- */ + jobs, _, _, err := s.jobClient.List(&jobparams.ListJobsParams{}) + require.Nil(t, err) + require.Len(t, jobs, 1) + job := jobs[0] + + // Check workflow manifest is not empty + assert.Contains(t, job.PipelineSpec.WorkflowManifest, "whalesay") + expectedJob := &job_model.APIJob{ + ID: job.ID, + Name: "hello world", + Description: "this is hello world", + PipelineSpec: &job_model.APIPipelineSpec{ + PipelineID: pipeline.ID, + PipelineName: "hello-world.yaml", + WorkflowManifest: job.PipelineSpec.WorkflowManifest, + }, + ResourceReferences: []*job_model.APIResourceReference{ + {Key: &job_model.APIResourceKey{Type: job_model.APIResourceTypeEXPERIMENT, ID: experiment.ID}, + Name: experiment.Name, Relationship: job_model.APIRelationshipOWNER, + }, + }, + MaxConcurrency: 10, + NoCatchup: true, + Enabled: true, + CreatedAt: job.CreatedAt, + UpdatedAt: job.UpdatedAt, + Status: job.Status, + Trigger: &job_model.APITrigger{}, + } + + assert.Equal(t, expectedJob, job) +} + +func checkHelloWorldRunDetail(t *testing.T, runDetail *run_model.APIRunDetail) { + // Check workflow manifest is not empty + assert.Contains(t, runDetail.Run.PipelineSpec.WorkflowManifest, "whalesay") + // Check runtime workflow manifest is not empty + assert.Contains(t, runDetail.PipelineRuntime.WorkflowManifest, "whalesay") + + expectedRun := &run_model.APIRun{ + ID: runDetail.Run.ID, + Name: "hello world", + Description: "this is hello world", + Status: runDetail.Run.Status, + PipelineSpec: &run_model.APIPipelineSpec{ + PipelineID: runDetail.Run.PipelineSpec.PipelineID, + PipelineName: "hello-world.yaml", + WorkflowManifest: runDetail.Run.PipelineSpec.WorkflowManifest, + }, + ResourceReferences: []*run_model.APIResourceReference{ + {Key: &run_model.APIResourceKey{Type: run_model.APIResourceTypeEXPERIMENT, ID: runDetail.Run.ResourceReferences[0].Key.ID}, + Name: "hello world experiment", Relationship: run_model.APIRelationshipOWNER, + }, + }, + CreatedAt: runDetail.Run.CreatedAt, + ScheduledAt: runDetail.Run.ScheduledAt, + FinishedAt: runDetail.Run.FinishedAt, + } + assert.Equal(t, expectedRun, runDetail.Run) +} + +func (s *UpgradeTests) createHelloWorldExperiment() *experiment_model.APIExperiment { + t := s.T() + + experiment := &experiment_model.APIExperiment{Name: "hello world experiment"} + helloWorldExperiment, err := s.experimentClient.Create(&experimentParams.CreateExperimentParams{Body: experiment}) + require.Nil(t, err) + + return helloWorldExperiment +} + +func (s *UpgradeTests) getHelloWorldExperiment(createIfNotExist bool) *experiment_model.APIExperiment { + t := s.T() + + experiments, err := s.experimentClient.ListAll(&experimentParams.ListExperimentParams{}, 1000) + require.Nil(t, err) + var helloWorldExperiment *experiment_model.APIExperiment + for _, experiment := range experiments { + if experiment.Name == "hello world experiment" { + helloWorldExperiment = experiment + } + } + + if helloWorldExperiment == nil && createIfNotExist { + return s.createHelloWorldExperiment() + } + + return helloWorldExperiment +} + +func (s *UpgradeTests) getHelloWorldPipeline(createIfNotExist bool) *pipeline_model.APIPipeline { + t := s.T() + + pipelines, err := s.pipelineClient.ListAll(&pipelineParams.ListPipelinesParams{}, 1000) + require.Nil(t, err) + var helloWorldPipeline *pipeline_model.APIPipeline + for _, pipeline := range pipelines { + if pipeline.Name == "hello-world.yaml" { + helloWorldPipeline = pipeline + } + } + + if helloWorldPipeline == nil && createIfNotExist { + return s.createHelloWorldPipeline() + } + + return helloWorldPipeline +} + +func (s *UpgradeTests) createHelloWorldPipeline() *pipeline_model.APIPipeline { + t := s.T() + + /* ---------- Upload pipelines YAML ---------- */ + uploadedPipeline, err := s.pipelineUploadClient.UploadFile("../resources/hello-world.yaml", uploadParams.NewUploadPipelineParams()) + require.Nil(t, err) + + helloWorldPipeline, err := s.pipelineClient.Get(&pipelineParams.GetPipelineParams{ID: uploadedPipeline.ID}) + require.Nil(t, err) + + return helloWorldPipeline +} diff --git a/test/api-integration-test/run_test.sh b/test/api-integration-test/run_test.sh index fc4b4298a4c2..b8a7a9b496fc 100755 --- a/test/api-integration-test/run_test.sh +++ b/test/api-integration-test/run_test.sh @@ -23,8 +23,10 @@ NAMESPACE=kubeflow usage() { echo "usage: run_test.sh - --results-gcs-dir GCS directory for the test results. Usually gs:////backend_unit_test + [--results-gcs-dir GCS directory for the test results. Usually gs:////backend_unit_test] [--namespace k8s namespace where ml-pipelines is deployed. The tests run against the instance in this namespace] + [--run_upgrade_tests_preparation run preparation step of upgrade tests instead] + [--run_upgrade_tests_verification run verification step of upgrade tests instead] [-h help]" } @@ -36,6 +38,12 @@ while [ "$1" != "" ]; do --namespace ) shift NAMESPACE=$1 ;; + --run_upgrade_tests_preparation ) + UPGRADE_TESTS_PREPARATION=true + ;; + --run_upgrade_tests_verification ) + UPGRADE_TESTS_VERIFICATION=true + ;; -h | --help ) usage exit ;; @@ -68,7 +76,13 @@ echo "Run integration test..." LOG_FILE=$(mktemp) # Note, "set -o pipefail" at top of file is required to catch exit code of the pipe. TEST_EXIT_CODE=0 # reference for how to save exit code: https://stackoverflow.com/a/18622662 -go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? +if [ -n "$UPGRADE_TESTS_PREPARATION" ]; then + go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Prepare |& tee $LOG_FILE || TEST_EXIT_CODE=$? +elif [ -n "$UPGRADE_TESTS_VERIFICATION" ]; then + go test -v ./... -namespace ${NAMESPACE} -args -runUpgradeTests=true -testify.m=Verify |& tee $LOG_FILE || TEST_EXIT_CODE=$? +else + go test -v ./... -namespace ${NAMESPACE} -args -runIntegrationTests=true |& tee $LOG_FILE || TEST_EXIT_CODE=$? +fi # Convert test result to junit.xml < "$LOG_FILE" go-junit-report > "${JUNIT_TEST_RESULT}" diff --git a/test/deploy-cluster.sh b/test/deploy-cluster.sh index ce2876b606c5..dd62105849c2 100755 --- a/test/deploy-cluster.sh +++ b/test/deploy-cluster.sh @@ -18,8 +18,10 @@ set -ex # Env inputs: # * COMMIT_SHA - decides TEST_CLUSTER's name +# * TEST_CLUSTER_PREFIX - decides default TEST_CLUSTER naming # * TEST_CLUSTER - [optional] specify to reuse existing TEST_CLUSTER -TEST_CLUSTER_PREFIX=${WORKFLOW_FILE%.*} +DEFAULT_TEST_CLUSTER_PREFIX=${WORKFLOW_FILE%.*} +TEST_CLUSTER_PREFIX=${TEST_CLUSTER_PREFIX:-${DEFAULT_TEST_CLUSTER_PREFIX}} TEST_CLUSTER_DEFAULT=$(echo $TEST_CLUSTER_PREFIX | cut -d _ -f 1)-${COMMIT_SHA:0:7}-${RANDOM} TEST_CLUSTER=${TEST_CLUSTER:-${TEST_CLUSTER_DEFAULT}} ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} diff --git a/test/deploy-pipeline-lite.sh b/test/deploy-pipeline-lite.sh index e9ce4aa9b221..dd7476062f2e 100755 --- a/test/deploy-pipeline-lite.sh +++ b/test/deploy-pipeline-lite.sh @@ -19,10 +19,14 @@ set -ex # Env inputs: # * $GCR_IMAGE_BASE_DIR # * $GCR_IMAGE_TAG -# * ENABLE_WORKLOAD_IDENTITY +# * $KFP_DEPLOY_RELEASE +# * $ENABLE_WORKLOAD_IDENTITY GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest} ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false} +KFP_MANIFEST_DIR="${DIR}/manifests" +pushd ${KFP_MANIFEST_DIR} + if ! which kustomize; then # Download kustomize cli tool TOOL_DIR=${DIR}/bin @@ -33,39 +37,67 @@ if ! which kustomize; then PATH=${PATH}:${TOOL_DIR} fi -# delete argo first because KFP comes with argo too -kubectl delete namespace argo --wait || echo "No argo installed" - -KFP_MANIFEST_DIR=${DIR}/manifests - -pushd ${KFP_MANIFEST_DIR}/crd -kustomize build . | kubectl apply -f - -popd - -kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io - -pushd ${KFP_MANIFEST_DIR}/dev - -# This is the recommended approach to do this. -# reference: https://github.com/kubernetes-sigs/kustomize/blob/master/docs/eschewedFeatures.md#build-time-side-effects-from-cli-args-or-env-variables -kustomize edit set image gcr.io/ml-pipeline/api-server=${GCR_IMAGE_BASE_DIR}/api-server:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/persistenceagent=${GCR_IMAGE_BASE_DIR}/persistenceagent:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/scheduledworkflow=${GCR_IMAGE_BASE_DIR}/scheduledworkflow:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/frontend=${GCR_IMAGE_BASE_DIR}/frontend:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/viewer-crd-controller=${GCR_IMAGE_BASE_DIR}/viewer-crd-controller:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/visualization-server=${GCR_IMAGE_BASE_DIR}/visualization-server:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/inverse-proxy-agent=${GCR_IMAGE_BASE_DIR}/inverse-proxy-agent:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline/metadata-writer=${GCR_IMAGE_BASE_DIR}/metadata-writer:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline-test/cache-server=${GCR_IMAGE_BASE_DIR}/cache-server:${GCR_IMAGE_TAG} -kustomize edit set image gcr.io/ml-pipeline-test/cache-deployer=${GCR_IMAGE_BASE_DIR}/cache-deployer:${GCR_IMAGE_TAG} -cat kustomization.yaml - -kustomize build . | kubectl apply -f - +if [ -z "$KFP_DEPLOY_RELEASE" ]; then + echo "Deploying KFP in working directory..." + KFP_MANIFEST_DIR=${DIR}/manifests + + pushd ${KFP_MANIFEST_DIR}/crd + kustomize build . | kubectl apply -f - + kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io + popd + + pushd ${KFP_MANIFEST_DIR}/dev + + # This is the recommended approach to do this. + # reference: https://github.com/kubernetes-sigs/kustomize/blob/master/docs/eschewedFeatures.md#build-time-side-effects-from-cli-args-or-env-variables + kustomize edit set image gcr.io/ml-pipeline/api-server=${GCR_IMAGE_BASE_DIR}/api-server:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/persistenceagent=${GCR_IMAGE_BASE_DIR}/persistenceagent:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/scheduledworkflow=${GCR_IMAGE_BASE_DIR}/scheduledworkflow:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/frontend=${GCR_IMAGE_BASE_DIR}/frontend:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/viewer-crd-controller=${GCR_IMAGE_BASE_DIR}/viewer-crd-controller:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/visualization-server=${GCR_IMAGE_BASE_DIR}/visualization-server:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/inverse-proxy-agent=${GCR_IMAGE_BASE_DIR}/inverse-proxy-agent:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline/metadata-writer=${GCR_IMAGE_BASE_DIR}/metadata-writer:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline-test/cache-server=${GCR_IMAGE_BASE_DIR}/cache-server:${GCR_IMAGE_TAG} + kustomize edit set image gcr.io/ml-pipeline-test/cache-deployer=${GCR_IMAGE_BASE_DIR}/cache-deployer:${GCR_IMAGE_TAG} + cat kustomization.yaml + + kustomize build . | kubectl apply -f - + popd +else + KFP_LATEST_RELEASE=$(git tag --sort=v:refname | tail -1) + echo "Deploying KFP release $KFP_LATEST_RELEASE" + + # temporarily checkout last release tag + git checkout $KFP_LATEST_RELEASE + + pushd ${KFP_MANIFEST_DIR}/crd + kustomize build . | kubectl apply -f - + kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io + popd + + pushd ${KFP_MANIFEST_DIR}/dev + kustomize build . | kubectl apply -f - + popd + + # go back to previous commit + git checkout - +fi # show current info echo "Status of pods after kubectl apply" kubectl get pods -n ${NAMESPACE} +# wait for all deployments to be successful +# note, after we introduce statefulsets or daemonsets, we need to wait their rollout status here too +for deployment in $(kubectl get deployments -n ${NAMESPACE} -o name) +do + kubectl rollout status $deployment -n ${NAMESPACE} +done + +echo "Status of pods after rollouts are successful" +kubectl get pods -n ${NAMESPACE} + if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then # Use static GSAs for testing, so we don't need to GC them. export SYSTEM_GSA="test-kfp-system" diff --git a/test/e2e_test_gke_v2.yaml b/test/e2e_test_gke_v2.yaml index 3cd328140ae3..60a7743e2080 100644 --- a/test/e2e_test_gke_v2.yaml +++ b/test/e2e_test_gke_v2.yaml @@ -134,6 +134,48 @@ spec: - recursion - volume_ops + - name: upgrade-test-preparation + inputs: + parameters: + - name: target-image-prefix + - name: test-results-gcs-dir + - name: api-integration-test-image-suffix + steps: + - - name: build-api-integration-test-image + template: build-image + arguments: + parameters: + - name: docker-path + value: . + - name: docker-file + value: test/api-integration-test/Dockerfile + - name: image-name + value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.api-integration-test-image-suffix}}" + - - name: run-upgrade-tests-preparation + template: run-upgrade-tests-preparation + arguments: + parameters: + - name: test-results-gcs-dir + value: "{{inputs.parameters.test-results-gcs-dir}}" + - name: api-integration-test-image + value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.api-integration-test-image-suffix}}" + + - name: upgrade-test-verification + inputs: + parameters: + - name: target-image-prefix + - name: test-results-gcs-dir + - name: api-integration-test-image-suffix + steps: + - - name: run-upgrade-tests-verification + template: run-upgrade-tests-verification + arguments: + parameters: + - name: test-results-gcs-dir + value: "{{inputs.parameters.test-results-gcs-dir}}" + - name: api-integration-test-image + value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.api-integration-test-image-suffix}}" + # Build and push image - name: build-image retryStrategy: @@ -221,3 +263,27 @@ spec: "--namespace", "{{inputs.parameters.namespace}}", "--test-name", "{{inputs.parameters.test-name}}", ] + + - name: run-upgrade-tests-preparation + inputs: + parameters: + - name: test-results-gcs-dir + - name: api-integration-test-image + container: + image: "{{inputs.parameters.api-integration-test-image}}" + args: [ + "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", + "--run_upgrade_tests_preparation", + ] + + - name: run-upgrade-tests-verification + inputs: + parameters: + - name: test-results-gcs-dir + - name: api-integration-test-image + container: + image: "{{inputs.parameters.api-integration-test-image}}" + args: [ + "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", + "--run_upgrade_tests_verification", + ] diff --git a/test/postsubmit-tests-with-pipeline-deployment.sh b/test/postsubmit-tests-with-pipeline-deployment.sh index c7579321a09f..966e53860ac1 100755 --- a/test/postsubmit-tests-with-pipeline-deployment.sh +++ b/test/postsubmit-tests-with-pipeline-deployment.sh @@ -150,7 +150,6 @@ ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ -p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \ -p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \ -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ --p cluster-type="${CLUSTER_TYPE}" \ -n ${NAMESPACE} \ --serviceaccount test-runner \ -o name diff --git a/test/presubmit-tests-with-pipeline-deployment.sh b/test/presubmit-tests-with-pipeline-deployment.sh index df78e11b5675..17793ab6833e 100755 --- a/test/presubmit-tests-with-pipeline-deployment.sh +++ b/test/presubmit-tests-with-pipeline-deployment.sh @@ -92,15 +92,15 @@ echo "KFP images cloudbuild jobs submitted" time source "${DIR}/deploy-cluster.sh" echo "cluster deployed" -time source "${DIR}/check-build-image-status.sh" -echo "KFP images built" - # Install Argo CLI and test-runner service account time source "${DIR}/install-argo.sh" echo "argo installed" +time source "${DIR}/check-build-image-status.sh" +echo "KFP images built" + time source "${DIR}/deploy-pipeline-lite.sh" -echo "KFP lite deployed" +echo "KFP standalone deployed" echo "submitting argo workflow to run tests for commit ${COMMIT_SHA}..." ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ @@ -108,7 +108,6 @@ ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ ${IMAGE_BUILDER_ARG} \ -p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \ -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ --p cluster-type="${CLUSTER_TYPE}" \ -n ${NAMESPACE} \ --serviceaccount test-runner \ -o name diff --git a/test/upgrade-tests.sh b/test/upgrade-tests.sh new file mode 100755 index 000000000000..221983d55496 --- /dev/null +++ b/test/upgrade-tests.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +usage() +{ + echo "usage: upgrade-tests.sh + [--platform the deployment platform. Valid values are: [gcp, minikube]. Default is gcp.] + [--project the gcp project. Default is ml-pipeline-test. Only used when platform is gcp.] + [--test_result_bucket the gcs bucket that argo workflow store the result to. Default is ml-pipeline-test + [--test_result_folder the gcs folder that argo workflow store the result to. Always a relative directory to gs:///[PULL_SHA]] + [--timeout timeout of the tests in seconds. Default is 1800 seconds. ] + [-h help]" +} + +PLATFORM=gcp +PROJECT=ml-pipeline-test +TEST_RESULT_BUCKET=ml-pipeline-test +TIMEOUT_SECONDS=2700 # 45 minutes +NAMESPACE=kubeflow +WORKFLOW_FILE=e2e_test_gke_v2.yaml + +while [ "$1" != "" ]; do + case $1 in + --platform ) shift + PLATFORM=$1 + ;; + --project ) shift + PROJECT=$1 + ;; + --test_result_bucket ) shift + TEST_RESULT_BUCKET=$1 + ;; + --test_result_folder ) shift + TEST_RESULT_FOLDER=$1 + ;; + --timeout ) shift + TIMEOUT_SECONDS=$1 + ;; + -h | --help ) usage + exit + ;; + * ) usage + exit 1 + esac + shift +done + +# This is merged commit's SHA. +COMMIT_SHA="$(git rev-parse HEAD)" + +# Paths are using commit sha, instead of pull sha, because tests may be rerun with the same PR +# commit, but merged on a different master version. When this happens, we cannot reuse cached +# results on the previous test run. +GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/${COMMIT_SHA} +TEST_RESULTS_GCS_DIR=gs://${TEST_RESULT_BUCKET}/${COMMIT_SHA}/${TEST_RESULT_FOLDER} +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)" +LATEST_RELEASED_TAG=$(git tag --sort=v:refname | tail -1) + +# Configure `time` command output format. +TIMEFORMAT="[test-timing] It took %lR." + +echo "upgrade test starts" +if [ -n "$PULL_PULL_SHA" ]; then + echo "PR commit is ${PULL_PULL_SHA}" +fi +time source "${DIR}/test-prep.sh" +echo "test env prepared" + +time source "${DIR}/build-images.sh" +echo "KFP images cloudbuild jobs submitted" + +time TEST_CLUSTER_PREFIX="upgrade" \ + source "${DIR}/deploy-cluster.sh" +echo "cluster deployed" + +# Install Argo CLI and test-runner service account +time source "${DIR}/install-argo.sh" +echo "argo installed" + +time KFP_DEPLOY_RELEASE=true source "${DIR}/deploy-pipeline-lite.sh" +echo "KFP standalone of latest release deployed" + +echo "submitting argo workflow to setup test env before upgrade..." +ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ +--entrypoint upgrade-test-preparation \ +-p image-build-context-gcs-uri="$remote_code_archive_uri" \ +${IMAGE_BUILDER_ARG} \ +-p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \ +-p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ +-n ${NAMESPACE} \ +--serviceaccount test-runner \ +-o name +` +time source "${DIR}/check-argo-status.sh" +echo "upgrade test preparation workflow completed" + +time source "${DIR}/check-build-image-status.sh" +echo "KFP images built" + +time source "${DIR}/deploy-pipeline-lite.sh" +echo "KFP standalone of commit ${COMMIT_SHA} deployed" + +echo "submitting argo workflow to verify test env after upgrade..." +ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \ +--entrypoint upgrade-test-verification \ +-p image-build-context-gcs-uri="$remote_code_archive_uri" \ +${IMAGE_BUILDER_ARG} \ +-p target-image-prefix="${GCR_IMAGE_BASE_DIR}/" \ +-p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \ +-n ${NAMESPACE} \ +--serviceaccount test-runner \ +-o name +` +time source "${DIR}/check-argo-status.sh" +echo "upgrade test verification workflow completed" diff --git a/test/upgrade_test_setup.yaml b/test/upgrade_test_setup.yaml new file mode 100644 index 000000000000..d4618dfce435 --- /dev/null +++ b/test/upgrade_test_setup.yaml @@ -0,0 +1,126 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: upgrade-test-setup- +spec: + entrypoint: integration-test + volumes: + - name: gcp-credentials + secret: + secretName: user-gcp-sa + arguments: + parameters: + - name: image-build-context-gcs-uri + - name: image-builder-image + value: gcr.io/ml-pipeline-test/image-builder:v20200208-0.1.25-771-g4c571961 + - name: target-image-prefix + - name: test-results-gcs-dir + - name: initialization-test-image-suffix + value: initialization_test + - name: cluster-type + value: gke + - name: namespace + value: kubeflow + templates: + - name: integration-test + inputs: + parameters: + - name: target-image-prefix + - name: test-results-gcs-dir + - name: initialization-test-image-suffix + - name: api-integration-test-image-suffix + - name: frontend-integration-tests-image-suffix + - name: basic-e2e-tests-image-suffix + - name: namespace + steps: + - - name: build-initialization-test-image + template: build-image + arguments: + parameters: + - name: docker-path + value: . + - name: docker-file + value: test/initialization-test/Dockerfile + - name: image-name + value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.initialization-test-image-suffix}}" + - - name: run-initialization-tests + template: run-initialization-tests + arguments: + parameters: + - name: test-results-gcs-dir + value: "{{inputs.parameters.test-results-gcs-dir}}" + - name: initialization-test-image + value: "{{inputs.parameters.target-image-prefix}}{{inputs.parameters.initialization-test-image-suffix}}" + + # Build and push image + - name: build-image + inputs: + parameters: + # GCS URI prefix pointing to a .tar.gz archive of Docker build context + - name: image-build-context-gcs-uri + value: "{{workflow.parameters.image-build-context-gcs-uri}}" + # The relative code path to the Dockerfile + - name: docker-path + # Name of the Docker file to use. "Dockerfile" by default + - name: docker-file + value: Dockerfile + - name: image-name + outputs: + parameters: + - name: strict-image-name + valueFrom: + path: /outputs/strict-image-name/file + container: + image: "{{workflow.parameters.image-builder-image}}" + imagePullPolicy: 'Always' + args: [ + "--image-build-context-gcs-uri", "{{inputs.parameters.image-build-context-gcs-uri}}", + "--docker_path", "{{inputs.parameters.docker-path}}", + "--docker_file", "{{inputs.parameters.docker-file}}", + "--image_name", "{{inputs.parameters.image-name}}", + ] + env: + - name: DOCKER_HOST + value: 127.0.0.1 + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/gcp-credentials/user-gcp-sa.json + volumeMounts: + - name: gcp-credentials + mountPath: /secret/gcp-credentials + sidecars: + - name: dind + image: docker:17.10-dind + securityContext: + privileged: true + mirrorVolumeMounts: true + + - name: run-initialization-tests + inputs: + parameters: + - name: test-results-gcs-dir + - name: initialization-test-image + container: + image: "{{inputs.parameters.initialization-test-image}}" + args: [ + "--results-gcs-dir", "{{inputs.parameters.test-results-gcs-dir}}", + ] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/gcp-credentials/user-gcp-sa.json + volumeMounts: + - name: gcp-credentials + mountPath: /secret/gcp-credentials