diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource index ba577d4d3..6b92f4b36 100644 --- a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource +++ b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource @@ -321,6 +321,7 @@ Query Model Multiple Times ... ${runtime}=caikit-tgis-runtime ... ${isvc_name}=${model_name} ... ${inference_type}=all-tokens + ... ${status_code}=200 ... ${n_times}=10 ... ${query_idx}=0 ... ${validate_response}=${TRUE} @@ -356,7 +357,7 @@ Query Model Multiple Times ... inference_type=${inference_type} model_name=${model_name} body_params=${body_params} ... query_text=${EXP_RESPONSES}[queries][${query_idx}][query_text] IF "${token}" != "${None}" - ${header}= Set Variable "Authorization: Bearer ${token}" -H ${header} + Set To Dictionary ${header} Authorization Bearer ${token} END ${runtime_details}= Set Variable ${RUNTIME_FORMATS}[${runtime}][endpoints][${inference_type}][${protocol}] ${endpoint}= Set Variable ${runtime_details}[endpoint] @@ -381,7 +382,7 @@ Query Model Multiple Times Log Dictionary ${args} ${res}= Run Keyword And Continue On Failure Perform Request request_type=POST ... skip_res_json=${skip_json_load_response} &{args} - Run Keyword And Continue On Failure Status Should Be 200 + Run Keyword And Continue On Failure Status Should Be ${status_code} END Log ${res} IF ${validate_response} == ${TRUE} diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index c7fddd02b..06b8d57b3 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -22,6 +22,8 @@ ${SERVING_ACCELERATOR_MINUS_BUTTON_XPATH}= xpath=${SERVING_ACCELERATOR_INPUT_ ${SERVING_ACCELERATOR_PLUS_BUTTON_XPATH}= xpath=${SERVING_ACCELERATOR_INPUT_XPATH}/following-sibling::button ${SERVING_MODEL_SERVERS_SIDE_MENU}= xpath=//span[text()='Models and model servers'] ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] +${ADD_SERVICE_ACCOUNT_BUTTON}= xpath://button[text()='Add a service account'] +${SERVICE_ACCOUNT_INPUT}= xpath://input[@data-testid='service-account-form-name'] ${REPLICAS_COUNT_XP}= xpath://section[@class='pf-v5-c-form__section']//span[@class='pf-v5-c-form-control']/input @@ -176,22 +178,56 @@ Enable External Serving Route Enable Token Authentication [Documentation] Enables Token authentication to serving route + [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} - # TODO: change service account name + IF "${service_account_name}" != "${NONE}" + Input Service Account Name ${service_account_name} + END Disable Token Authentication - [Documentation] Enables Token authentication to serving route - SeleniumLibrary.Unselect Checkbox ${TOKEN_AUTH_CHECKBOX_XP} - # TODO: change service account name + [Documentation] Disable Token authentication to serving route + ... If a Service Account is given, just disable the token for the specific one. + [Arguments] ${service_account_name}=${NONE} + IF "${service_account_name}" != "${NONE}" + Click Button xpath://input[@value="${service_account_name}"]/parent::span/parent::div/following::div/button[@aria-label="Remove service account"] + ELSE + SeleniumLibrary.Unselect Checkbox ${TOKEN_AUTH_CHECKBOX_XP} + END + +Add Another Service Account + [Documentation] Add another Service Account once the Token Authorization is enabled + [Arguments] ${service_account_name}=default-name2 + Click Button ${ADD_SERVICE_ACCOUNT_BUTTON} + Input Service Account Name ${service_account_name} + +Input Service Account Name + [Documentation] Input the name of the last Added Service Account. + [Arguments] ${service_account_name}=default-name + ${service_accounts}= Get WebElements ${SERVICE_ACCOUNT_INPUT} + ${last_service_account}= Set Variable ${service_accounts}[-1] + Clear Element And Input Text ${last_service_account} ${service_account_name} Get Model Serving Access Token via UI [Documentation] Returns the token used for authentication to the serving route ... TODO: There can be multiple tokens defined for each model server, handle this case as well - [Arguments] ${service_account_name}=default-name - Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - Click Element xpath://td[@data-label="Tokens"]/button + [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} ${model_name}=${NONE} + IF ${single_model} + # Expand the model + SeleniumLibrary.Click Button xpath: //div[text()='${model_name}']/parent::td[@data-label='Name']/preceding-sibling::td/button + ${token}= Get Single Model Token ${service_account_name} + ELSE + Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button + Click Element xpath://td[@data-label="Tokens"]/button + ${token}= Get Element Attribute + ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value + END + RETURN ${token} + +Get Single Model Token + [Documentation] Gets the (Single Model) Token of an specific Service Account + [Arguments] ${service_account_name} ${token}= Get Element Attribute - ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value + ... xpath://div[text()='${service_account_name}']/parent::td[@data-label='Token Name']/following-sibling::td[@data-label='Token Secret']//input value RETURN ${token} Click Plus Button @@ -274,6 +310,7 @@ Switch Model Serving Project IF "${project_name}" == "${NONE}" ${project_name}= Set Variable All projects END + Wait Until Element Is Visible //main[contains(@id, 'dashboard-page-main')]//button[contains(@id, 'dropdown-toggle')] Click Element //main[contains(@id, 'dashboard-page-main')]//button[contains(@id, 'dropdown-toggle')] Wait Until Element Is Visible //ul//a[contains(., '${project_name}')] Click Element //ul//a[contains(., '${project_name}')] diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 345ee7675..2137b243c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -200,7 +200,8 @@ Open Model Edit Modal [Documentation] Opens the modal used to edit the details of an already deployed model. [Arguments] ${model_name} ODHDashboard.Click Action From Actions Menu item_title=${model_name} action=Edit - Wait For Dashboard Page Title Deploy model + # Wait for modal header + Wait Until Page Contains Element xpath://span[contains(@class, "modal") and text()="Deploy model"] Get Model Route Via UI [Documentation] Grabs the serving route (URL) of an already deployed model from the Model Serving page. @@ -238,12 +239,13 @@ Open ${section} Options Menu SeleniumLibrary.Click Element xpath://span[.="${section}"]/../../..//button # Remove attribute which is not present anymore END -Get Access Token via UI +Get Access Token Via UI [Documentation] Returns the access token for models deployed in a specific project ... by using the UI of DSP - [Arguments] ${project_name} ${service_account_name}=default-name + [Arguments] ${project_name} ${service_account_name}=default-name ${single_model}=${FALSE} + ... ${model_name}=${NONE} Open Data Science Project Details Page ${project_name} tab_id=model-server - ${token}= Get Model Serving Access Token via UI ${service_account_name} + ${token}= Get Model Serving Access Token via UI ${service_account_name} ${single_model} ${model_name} RETURN ${token} Get Model Project @@ -269,7 +271,7 @@ Get Model Inference IF "${project_title}" == "${NONE}" ${project_title}= Get Model Project ${model_name} END - ${token}= Get Access Token via UI ${project_title} + ${token}= Get Access Token Via UI ${project_title} ${curl_cmd}= Catenate ${curl_cmd} -H "Authorization: Bearer ${token}" END IF ${kserve} @@ -388,6 +390,7 @@ Deploy Kserve Model Via UI #robocop: disable [Documentation] Deploys a model using the kserve/caikit runtime using the Data Science Projects UI [Arguments] ${model_name} ${serving_runtime} ${data_connection} ${path} ... ${model_framework}=caikit ${replicas}=1 ${size}=Small ${no_gpus}=${0} + ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 Move To Tab Models Click Button ${DEPLOY_MODEL_BTN} Wait Until Page Contains Element xpath=${KSERVE_MODAL_HEADER} @@ -396,6 +399,12 @@ Deploy Kserve Model Via UI #robocop: disable Select Framework ${model_framework} Set Replicas Number With Buttons ${replicas} Set Server Size ${size} + IF ${token} + Enable Token Authentication + IF ${multi_token} + Add Another Service Account ${multi_service_account_name} + END + END Select Existing Data Connection ${data_connection} Set Folder Path ${path} IF ${no_gpus} > ${0} diff --git a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_UI.robot b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_UI.robot index 690411f01..4b427fbac 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_UI.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_UI.robot @@ -195,6 +195,134 @@ Verify Non Admin Can Serve And Query A Model Using The UI # robocop: disable ... namespace=${test_namespace} protocol=http validate_response=$FALSE [Teardown] Run Keywords Clean Up DSP Page AND Close Browser AND Switch Browser 1 +Verify User Can Serve And Query A Token Protected Model Using The UI + [Documentation] Deploying and querying a Token Protected LLM model + ... using Kserve and Caikit runtime + [Tags] Tier1 RHOAIENG-4603 + [Setup] Set Up Project namespace=${TEST_NS} + ${test_namespace}= Set Variable ${TEST_NS} + ${flan_model_name}= Set Variable flan-t5-small-caikit + Deploy Kserve Model Via UI ${flan_model_name} serving_runtime=Caikit TGIS data_connection=kserve-connection + ... path=flan-t5-small/${flan_model_name} token=${TRUE} + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + ${model_token}= Get Access Token Via UI ${test_namespace} single_model=${TRUE} + ... model_name=${flan_model_name} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + ... token=${model_token} + Delete Model Via UI ${flan_model_name} + [Teardown] Clean Up DSP Page + +Verify User Can Serve But Can't Query A Token Protected Model Without The Token + [Documentation] Deploying and querying a Token Protected LLM model + ... using Kserve and Caikit runtime + [Tags] Tier1 RHOAIENG-4603 + [Setup] Set Up Project namespace=${TEST_NS} + ${test_namespace}= Set Variable ${TEST_NS} + ${flan_model_name}= Set Variable flan-t5-small-caikit + Deploy Kserve Model Via UI ${flan_model_name} serving_runtime=Caikit TGIS data_connection=kserve-connection + ... path=flan-t5-small/${flan_model_name} token=${TRUE} + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens status_code=401 n_times=1 validate_response=${FALSE} + ... namespace=${test_namespace} protocol=http + Delete Model Via UI ${flan_model_name} + [Teardown] Clean Up DSP Page + +Verify User Can Serve And Query A Model Using The UI Protected With Multiple Tokens + [Documentation] Deploying and querying a Token Protected LLM model + ... using Kserve and Caikit runtime, using multiple tokens + [Tags] Tier1 RHOAIENG-4603 + [Setup] Set Up Project namespace=${TEST_NS} + ${test_namespace}= Set Variable ${TEST_NS} + ${flan_model_name}= Set Variable flan-t5-small-caikit + Deploy Kserve Model Via UI ${flan_model_name} serving_runtime=Caikit TGIS data_connection=kserve-connection + ... path=flan-t5-small/${flan_model_name} token=${TRUE} multi_token=${TRUE} + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + ${model_token_1}= Get Access Token Via UI ${test_namespace} service_account_name=default-name + ... single_model=${TRUE} model_name=${flan_model_name} + ${model_token_2}= Get Access Token Via UI ${test_namespace} service_account_name=default-name2 + ... single_model=${TRUE} model_name=${flan_model_name} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + ... token=${model_token_1} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + ... token=${model_token_2} + Delete Model Via UI ${flan_model_name} + [Teardown] Clean Up DSP Page + +Verify User Can not Query A Token Protected Model With A Disabled Token Using The UI + [Documentation] Deploy an LLM (using Kserve and Caikit runtime) protected by different Tokens + ... and try to query it using a disabled token + [Tags] Tier1 RHOAIENG-4603 + [Setup] Set Up Project namespace=${TEST_NS} + ${test_namespace}= Set Variable ${TEST_NS} + ${flan_model_name}= Set Variable flan-t5-small-caikit + Deploy Kserve Model Via UI ${flan_model_name} serving_runtime=Caikit TGIS data_connection=kserve-connection + ... path=flan-t5-small/${flan_model_name} token=${TRUE} multi_token=${TRUE} + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + ${model_token_1}= Get Access Token Via UI ${test_namespace} service_account_name=default-name + ... single_model=${TRUE} model_name=${flan_model_name} + Open Model Edit Modal ${flan_model_name} + Disable Token Authentication service_account_name=default-name + Click Button Deploy + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + Run Keyword And Expect Error *Expected status: 401 != 200* + ... Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + ... token=${model_token_1} + Delete Model Via UI ${flan_model_name} + [Teardown] Clean Up DSP Page + +Verify User Can Query A Token Protected Model Using The UI Without Token After Disabling Token Protection + [Documentation] Deploying and querying a Token Protected LLM model using Kserve and Caikit runtime. + ... Verify the token Works and without Token it Does not Work. + ... Disable the token authentication and verify that the model can be queried without token. + [Tags] Sanity Tier1 RHOAIENG-4603 + [Setup] Set Up Project namespace=${TEST_NS} + ${test_namespace}= Set Variable ${TEST_NS} + ${flan_model_name}= Set Variable flan-t5-small-caikit + Deploy Kserve Model Via UI ${flan_model_name} serving_runtime=Caikit TGIS data_connection=kserve-connection + ... path=flan-t5-small/${flan_model_name} token=${TRUE} + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + ${model_token}= Get Access Token Via UI ${test_namespace} single_model=${TRUE} + ... model_name=${flan_model_name} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + ... token=${model_token} + Run Keyword And Expect Error *Expected status: 401 != 200* + ... Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + Open Model Edit Modal ${flan_model_name} + Disable Token Authentication + Click Button Deploy + # The verification will fail due to a temporary duplication of the replicas while the model is restarted + Sleep 60s msg=Wait for the model pod replicas to scale down + Wait For Pods Numbers 1 + ... namespace=${test_namespace} + ... label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... timeout=1200 + Wait For Model KServe Deployment To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} runtime=${CAIKIT_TGIS_RUNTIME_NAME} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} protocol=http + Delete Model Via UI ${flan_model_name} + [Teardown] Clean Up DSP Page + Verify User Can Serve And Query Flan-t5 Grammar Syntax Corrector Using The UI # robocop: disable [Documentation] Deploys and queries flan-t5-large-grammar-synthesis model [Tags] Tier2 ODS-2553 diff --git a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot index 3bf109155..2f936d22f 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1007__model_serving_llm/1007__model_serving_llm_models.robot @@ -19,7 +19,7 @@ ${KSERVE_MODE}= RawDeployment #Serverless ${MODEL_FORMAT}= pytorch #vLLM ${PROTOCOL}= grpc #http ${OVERLAY}= ${EMPTY} #vllm - +${GPU_TYPE}= NVIDIA *** Test Cases *** Verify User Can Serve And Query A bigscience/mt0-xxl Model @@ -845,7 +845,14 @@ Setup Test Variables Set Test Variable ${storage_uri} s3://${S3.BUCKET_3.NAME}/${model_path} END IF ${use_gpu} - ${limits}= Create Dictionary nvidia.com/gpu=1 + ${supported_gpu_type}= Convert To Lowercase ${GPU_TYPE} + IF "${supported_gpu_type}" == "nvidia" + ${limits}= Create Dictionary nvidia.com/gpu=1 + ELSE IF "${supported_gpu_type}" == "amd" + ${limits}= Create Dictionary amd.com/gpu=1 + ELSE + FAIL msg=Provided GPU type is not yet supported. Only nvidia and amd gpu type are supported + END Set Test Variable ${limits} ELSE Set Test Variable ${limits} &{EMPTY} diff --git a/ods_ci/tests/Tests/100__deploy/100__installation/112__model_registry.robot b/ods_ci/tests/Tests/100__deploy/100__installation/112__model_registry.robot index f92f5f738..1c51c8240 100644 --- a/ods_ci/tests/Tests/100__deploy/100__installation/112__model_registry.robot +++ b/ods_ci/tests/Tests/100__deploy/100__installation/112__model_registry.robot @@ -18,6 +18,8 @@ Verify Model Registry Operator Installation ... deployed in the ${APPLICATIONS_NAMESPACE} namespace in ODS [Tags] Smoke ... Tier1 + ... Operator + ... ModelRegistry ... OpenDataHub robot:recursive-continue-on-failure Wait Until Keyword Succeeds 1 min 10 sec Verify Model Registry Operator Deployment Wait Until Keyword Succeeds 10 times 5s Verify Model Registry ReplicaSets Info @@ -52,7 +54,6 @@ Registry Suite Setup IF "${REGISTRY}" == "false" Enable Component modelregistry Set Suite Variable ${REGISTRY} - Teardown [Documentation] Disable Registry if Enabled SeleniumLibrary.Close Browser diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/142__dsc_components.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/142__dsc_components.robot index 3f2768a66..8b3f5e51d 100644 --- a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/142__dsc_components.robot +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/142__dsc_components.robot @@ -23,6 +23,12 @@ ${TRAINING_LABEL_SELECTOR} app.kubernetes.io/name=training-operator ${TRAINING_DEPLOYMENT_NAME} kubeflow-training-operator ${DATASCIENCEPIPELINES_LABEL_SELECTOR} app.kubernetes.io/name=data-science-pipelines-operator ${DATASCIENCEPIPELINES_DEPLOYMENT_NAME} data-science-pipelines-operator-controller-manager +${MODELMESH_CONTROLLER_LABEL_SELECTOR} app.kubernetes.io/instance=modelmesh-controller +${MODELMESH_CONTROLLER_DEPLOYMENT_NAME} modelmesh-controller +${ETCD_LABEL_SELECTOR} component=model-mesh-etcd +${ETCD_DEPLOYMENT_NAME} etcd +${ODH_MODEL_CONTROLLER_LABEL_SELECTOR} app=odh-model-controller +${ODH_MODEL_CONTROLLER_DEPLOYMENT_NAME} odh-model-controller ${IS_PRESENT} 0 ${IS_NOT_PRESENT} 1 &{SAVED_MANAGEMENT_STATES} @@ -32,6 +38,8 @@ ${IS_NOT_PRESENT} 1 ... TRAINING=${EMPTY} ... DASHBOARD=${EMPTY} ... DATASCIENCEPIPELINES=${EMPTY} +... MODELMESHERVING=${EMPTY} + @{CONTROLLERS_LIST} kserve-controller-manager odh-model-controller modelmesh-controller @@ -138,6 +146,38 @@ Validate Datasciencepipelines Removed State [Teardown] Restore DSC Component State datasciencepipelines ${DATASCIENCEPIPELINES_DEPLOYMENT_NAME} ${DATASCIENCEPIPELINES_LABEL_SELECTOR} ${SAVED_MANAGEMENT_STATES.DATASCIENCEPIPELINES} +Validate Modelmeshserving Managed State + [Documentation] Validate that the DSC Modelmeshserving component Managed state creates the expected resources, + ... check that Modelmeshserving deployment is created and pods are in Ready state + [Tags] Operator Tier1 RHOAIENG-8546 modelmeshserving-managed + + Set DSC Component Managed State And Wait For Completion modelmeshserving ${MODELMESH_CONTROLLER_DEPLOYMENT_NAME} ${MODELMESH_CONTROLLER_LABEL_SELECTOR} + + # Check that ETC resources are ready + Wait For Resources To Be Available ${ETCD_DEPLOYMENT_NAME} ${ETCD_LABEL_SELECTOR} + + # Check that ODH Model Controller resources are ready + Wait For Resources To Be Available ${ODH_MODEL_CONTROLLER_DEPLOYMENT_NAME} ${ODH_MODEL_CONTROLLER_LABEL_SELECTOR} + + [Teardown] Restore DSC Component State modelmeshserving ${MODELMESH_CONTROLLER_DEPLOYMENT_NAME} ${MODELMESH_CONTROLLER_LABEL_SELECTOR} ${SAVED_MANAGEMENT_STATES.MODELMESHERVING} + +Validate Modelmeshserving Removed State + [Documentation] Validate that Modelmeshserving management state Removed does remove relevant resources. + [Tags] Operator Tier1 RHOAIENG-8546 modelmeshserving-removed + + Set DSC Component Removed State And Wait For Completion modelmeshserving ${MODELMESH_CONTROLLER_DEPLOYMENT_NAME} ${MODELMESH_CONTROLLER_LABEL_SELECTOR} + + # Check that ETC resources are removed + Wait For Resources To Be Removed ${ETCD_DEPLOYMENT_NAME} ${ETCD_LABEL_SELECTOR} + + # Check that ODH Model Controller resources are removed, if KServe managementState is Removed + ${state}= Get DSC Component State ${DSC_NAME} kserve ${OPERATOR_NS} + IF "${state}" == "Removed" + Wait For Resources To Be Removed ${ODH_MODEL_CONTROLLER_DEPLOYMENT_NAME} ${ODH_MODEL_CONTROLLER_LABEL_SELECTOR} + END + + [Teardown] Restore DSC Component State modelmeshserving ${MODELMESH_CONTROLLER_DEPLOYMENT_NAME} ${MODELMESH_CONTROLLER_LABEL_SELECTOR} ${SAVED_MANAGEMENT_STATES.MODELMESHERVING} + Validate Support For Configuration Of Controller Resources [Documentation] Validate support for configuration of controller resources in component deployments [Tags] Operator Tier1 ODS-2664 @@ -167,6 +207,8 @@ Validate Support For Configuration Of Controller Resources Suite Setup [Documentation] Suite Setup RHOSi Setup + ${DSC_SPEC}= Get DataScienceCluster Spec ${DSC_NAME} + Log To Console DSC Spec: ${DSC_SPEC} Wait For DSC Conditions Reconciled ${OPERATOR_NS} ${DSC_NAME} ${SAVED_MANAGEMENT_STATES.RAY}= Get DSC Component State ${DSC_NAME} ray ${OPERATOR_NS} ${SAVED_MANAGEMENT_STATES.KUEUE}= Get DSC Component State ${DSC_NAME} kueue ${OPERATOR_NS} @@ -174,6 +216,7 @@ Suite Setup ${SAVED_MANAGEMENT_STATES.TRAINING}= Get DSC Component State ${DSC_NAME} trainingoperator ${OPERATOR_NS} ${SAVED_MANAGEMENT_STATES.DASHBOARD}= Get DSC Component State ${DSC_NAME} dashboard ${OPERATOR_NS} ${SAVED_MANAGEMENT_STATES.DATASCIENCEPIPELINES}= Get DSC Component State ${DSC_NAME} datasciencepipelines ${OPERATOR_NS} + ${SAVED_MANAGEMENT_STATES.MODELMESHERVING}= Get DSC Component State ${DSC_NAME} modelmeshserving ${OPERATOR_NS} Set Suite Variable ${SAVED_MANAGEMENT_STATES} Suite Teardown @@ -201,11 +244,7 @@ Set DSC Component Removed State And Wait For Completion Set Component State ${component} Removed END - Wait Until Keyword Succeeds 5 min 0 sec - ... Is Resource Present Deployment ${deployment_name} ${APPLICATIONS_NS} ${IS_NOT_PRESENT} - - Wait Until Keyword Succeeds 5 min 0 sec - ... Check If Pod Does Not Exist ${label_selector} ${APPLICATIONS_NS} + Wait For Resources To Be Removed ${deployment_name} ${label_selector} Set DSC Component Managed State And Wait For Completion [Documentation] Set component management state to 'Managed', and wait for deployment and pod to be available. @@ -216,6 +255,11 @@ Set DSC Component Managed State And Wait For Completion Set Component State ${component} Managed END + Wait For Resources To Be Available ${deployment_name} ${label_selector} + +Wait For Resources To Be Available + [Documentation] Wait until Deployment and Pod(s) are Available + [Arguments] ${deployment_name} ${label_selector} Wait Until Keyword Succeeds 5 min 0 sec ... Is Resource Present Deployment ${deployment_name} ${APPLICATIONS_NS} ${IS_PRESENT} @@ -225,6 +269,16 @@ Set DSC Component Managed State And Wait For Completion Wait Until Keyword Succeeds 8 min 0 sec ... Is Pod Ready ${label_selector} +Wait For Resources To Be Removed + [Documentation] Wait until Deployment and Pod(s) to Removed + [Arguments] ${deployment_name} ${label_selector} + + Wait Until Keyword Succeeds 5 min 0 sec + ... Is Resource Present Deployment ${deployment_name} ${APPLICATIONS_NS} ${IS_NOT_PRESENT} + + Wait Until Keyword Succeeds 5 min 0 sec + ... Check If Pod Does Not Exist ${label_selector} ${APPLICATIONS_NS} + Restore DSC Component State [Documentation] Set component management state to original state, wait for component resources to be available. [Arguments] ${component} ${deployment_name} ${LABEL_SELECTOR} ${saved_state} @@ -249,3 +303,11 @@ Is Pod Ready # Log To Console "Pod Ready Status: ${output}" Should Be Equal As Integers ${rc} 0 Should Not Contain ${output} False + +Get DataScienceCluster Spec + [Documentation] Return the DSC Spec + [Arguments] ${DSC_NAME} + ${rc} ${output}= Run And Return Rc And Output + ... oc get DataScienceCluster/${DSC_NAME} -n ${OPERATOR_NS} -o "jsonpath={".spec"}" + Should Be Equal As Integers ${rc} 0 + RETURN ${output} diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/143__dsc_negative_no_servicemesh_op.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/143__dsc_negative_no_servicemesh_op.robot index e790458bc..8f1ce9b33 100644 --- a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/143__dsc_negative_no_servicemesh_op.robot +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/143__dsc_negative_no_servicemesh_op.robot @@ -28,7 +28,7 @@ ${IS_NOT_PRESENT} 1 Validate DSC and DSCI Created Without Service Mesh Operator #robocop:disable [Documentation] The purpose of this Test Case is to validate that DSC and DSCI are created ... without Service Mesh Operator installer, but with errors - [Tags] Operator Sanity Tier1 ODS-2584 + [Tags] Operator Tier3 ODS-2584 Log To Console message=Creating DSCInitialization CR via CLI Apply DSCInitialization CustomResource dsci_name=${DSCI_NAME} diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/144_dsc_negative_no_dependant_op.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/144_dsc_negative_no_dependant_op.robot index 3ca70d05e..02a08aafa 100644 --- a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/144_dsc_negative_no_dependant_op.robot +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/144_dsc_negative_no_dependant_op.robot @@ -28,7 +28,7 @@ ${IS_NOT_PRESENT} 1 Validate DSC and DSCI Created Without Service Mesh And Serverless Operators #robocop:disable [Documentation] The purpose of this Test Case is to validate that DSC and DSCI are created ... without dependant operators ((servicemesh, serverless) installed, but with errors - [Tags] Operator Sanity Tier1 ODS-2527 + [Tags] Operator Tier3 ODS-2527 Log To Console message=Creating DSCInitialization CR via CLI Apply DSCInitialization CustomResource dsci_name=${DSCI_NAME}