From 6debc78ba39488249ec029229c0b756d10c590cf Mon Sep 17 00:00:00 2001 From: Lin Wang Date: Mon, 4 Sep 2023 15:41:02 +0800 Subject: [PATCH] feat: add test cases for remote models (#813) Signed-off-by: Lin Wang --- .../ml-commons-dashboards/overview_spec.js | 318 ++++++++++++------ .../plugins/ml-commons-dashboards/commands.js | 48 ++- .../ml-commons-dashboards/constants.js | 3 + 3 files changed, 259 insertions(+), 110 deletions(-) diff --git a/cypress/integration/plugins/ml-commons-dashboards/overview_spec.js b/cypress/integration/plugins/ml-commons-dashboards/overview_spec.js index 9210ae37f..1b0c129d4 100644 --- a/cypress/integration/plugins/ml-commons-dashboards/overview_spec.js +++ b/cypress/integration/plugins/ml-commons-dashboards/overview_spec.js @@ -6,134 +6,236 @@ import { MLC_URL, MLC_DASHBOARD_API } from '../../../utils/constants'; if (Cypress.env('ML_COMMONS_DASHBOARDS_ENABLED')) { describe('MLC Overview page', () => { - let uploadedModelId; - let uploadedModelLoadedError; - const uploadModelName = `traced_small_model-${new Date() - .getTime() - .toString(34)}`; before(() => { // Disable only_run_on_ml_node to avoid model upload error in case of cluster no ml nodes cy.disableOnlyRunOnMLNode(); - cy.disableNativeMemoryCircuitBreaker(); - cy.enableRegisterModelViaURL(); - cy.wait(1000); - - cy.registerModelGroup({ - name: 'model-group', - }) - .then(({ model_group_id }) => - cy.uploadModelByUrl({ - name: uploadModelName, - version: '1.0.0', - model_format: 'TORCH_SCRIPT', - model_task_type: 'text_embedding', - model_group_id, - model_content_hash_value: - 'e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021', - model_config: { - model_type: 'bert', - embedding_dimension: 768, - framework_type: 'sentence_transformers', - all_config: - '{"architectures":["BertModel"],"max_position_embeddings":512,"model_type":"bert","num_attention_heads":12,"num_hidden_layers":6}', - }, - url: 'https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true', - }) - ) - .then(({ task_id: taskId }) => - cy.cyclingCheckTask({ - taskId, - }) - ) - .then(({ model_id: modelId }) => { - uploadedModelId = modelId; - return cy.loadMLCommonsModel(modelId); - }) - .then(({ task_id: taskId }) => - cy.cyclingCheckTask({ - taskId, - rejectOnError: false, - }) - ) - .then(({ error }) => { - if (error) { - uploadedModelLoadedError = error; - } - }); - }); - - after(() => { - if (uploadedModelId) { - cy.unloadMLCommonsModel(uploadedModelId); - cy.deleteMLCommonsModel(uploadedModelId); - } }); - it('should return to monitoring page when visit root', () => { cy.visit(MLC_URL.ROOT); cy.url().should('include', MLC_URL.OVERVIEW); }); - it('should display page header and deployed model name, status and id', () => { - cy.visit(MLC_URL.OVERVIEW); - - cy.contains('h1', 'Overview'); - cy.contains('h2', 'Deployed models'); + describe('custom upload model', () => { + let uploadedModelId; + let uploadedModelLoadedError; + const uploadModelName = `traced_small_model-${new Date() + .getTime() + .toString(34)}`; + before(() => { + cy.disableNativeMemoryCircuitBreaker(); + cy.enableRegisterModelViaURL(); + cy.wait(1000); + + cy.registerModelGroup({ + name: 'model-group', + }) + .then(({ model_group_id }) => + cy.uploadModelByUrl({ + name: uploadModelName, + version: '1.0.0', + model_format: 'TORCH_SCRIPT', + model_task_type: 'text_embedding', + model_group_id, + model_content_hash_value: + 'e13b74006290a9d0f58c1376f9629d4ebc05a0f9385f40db837452b167ae9021', + model_config: { + model_type: 'bert', + embedding_dimension: 768, + framework_type: 'sentence_transformers', + all_config: + '{"architectures":["BertModel"],"max_position_embeddings":512,"model_type":"bert","num_attention_heads":12,"num_hidden_layers":6}', + }, + url: 'https://github.com/opensearch-project/ml-commons/blob/2.x/ml-algorithms/src/test/resources/org/opensearch/ml/engine/algorithms/text_embedding/traced_small_model.zip?raw=true', + }) + ) + .then(({ task_id: taskId }) => + cy.cyclingCheckTask({ + taskId, + }) + ) + .then(({ model_id: modelId }) => { + uploadedModelId = modelId; + return cy.loadMLCommonsModel(modelId); + }) + .then(({ task_id: taskId }) => + cy.cyclingCheckTask({ + taskId, + rejectOnError: false, + }) + ) + .then(({ error }) => { + if (error) { + uploadedModelLoadedError = error; + } + }); + }); - cy.get('[aria-label="Search by name or ID"]').type(uploadModelName); + after(() => { + if (uploadedModelId) { + cy.unloadMLCommonsModel(uploadedModelId); + cy.deleteMLCommonsModel(uploadedModelId); + } + }); - cy.contains(uploadedModelId) - .closest('tr') - .contains(uploadedModelLoadedError ? 'Not responding' : 'Responding') - .closest('tr') - .contains(uploadModelName); + it('should display page header and deployed model name, status and source', () => { + cy.visit(MLC_URL.OVERVIEW); - cy.contains('h1', 'Overview'); - cy.contains('h2', 'Deployed models'); - }); + cy.contains('h1', 'Overview'); + cy.contains('h2', 'Models'); - it('should open preview panel after view detail button click', () => { - cy.visit(MLC_URL.OVERVIEW); - - cy.get('[aria-label="Search by name or ID"]').type(uploadModelName); + cy.get('[aria-label="Search by model name or ID"]').type( + uploadedModelId + ); - cy.contains(uploadedModelId) - .closest('tr') - .find('[aria-label="view detail"]') - .click(); + cy.contains(uploadModelName) + .closest('tr') + .contains(uploadedModelLoadedError ? 'Not responding' : 'Responding') + .closest('tr') + .contains('Local'); + }); - cy.contains('.euiFlyoutHeader > h3', uploadModelName); - cy.contains('.euiFlyoutBody', uploadedModelId); - }); + it('should open preview panel after view detail button click', () => { + cy.visit(MLC_URL.OVERVIEW); - it('should show empty nodes when deployed model profiling loading', () => { - cy.intercept( - 'GET', - MLC_DASHBOARD_API.DEPLOYED_MODEL_PROFILE.replace( - ':modelID', + cy.get('[aria-label="Search by model name or ID"]').type( uploadedModelId - ), - (req) => { - req.on('response', (res) => { - res.setDelay(3000); - }); - } - ).as('getDeployedModelProfile'); - - cy.visit(MLC_URL.OVERVIEW); + ); + + cy.contains(uploadModelName) + .closest('tr') + .find('[aria-label="view detail"]') + .click(); + + cy.get('div[role="dialog"]').contains(uploadModelName); + cy.get('div[role="dialog"]').contains(uploadedModelId); + cy.get('div[role="dialog"]').contains('Local'); + cy.get('div[role="dialog"]').contains('Status by node'); + }); + + it('should show empty nodes when deployed model profiling loading', () => { + cy.intercept( + 'GET', + MLC_DASHBOARD_API.DEPLOYED_MODEL_PROFILE.replace( + ':modelID', + uploadedModelId + ), + (req) => { + req.on('response', (res) => { + res.setDelay(3000); + }); + } + ).as('getDeployedModelProfile'); - cy.get('[aria-label="Search by name or ID"]').type(uploadModelName); + cy.visit(MLC_URL.OVERVIEW); - cy.contains(uploadedModelId) - .closest('tr') - .find('[aria-label="view detail"]') - .click(); + cy.get('[aria-label="Search by model name or ID"]').type( + uploadedModelId + ); + + cy.contains(uploadModelName) + .closest('tr') + .find('[aria-label="view detail"]') + .click(); + + cy.get('div[role="dialog"] .ml-nodesTableNodeIdCellText', { + timeout: 0, + }).should('not.exist'); + cy.wait('@getDeployedModelProfile'); + cy.get('div[role="dialog"] .ml-nodesTableNodeIdCellText').should( + 'exist' + ); + }); + }); - cy.get('div[role="dialog"] .ml-nodesTableNodeIdCellText', { - timeout: 0, - }).should('not.exist'); - cy.wait('@getDeployedModelProfile'); - cy.get('div[role="dialog"] .ml-nodesTableNodeIdCellText').should('exist'); + describe('remote model', () => { + let registeredRemoteModelId; + let remoteModelName; + before(() => { + remoteModelName = `remote sagemaker model-${new Date().getTime()}`; + cy.registerModel({ + name: remoteModelName, + function_name: 'remote', + version: '1.0.0', + description: 'test model', + connector: { + name: 'sagemaker: embedding', + description: 'Test connector for Sagemaker embedding model', + version: 1, + protocol: 'aws_sigv4', + credential: { + access_key: '...', + secret_key: '...', + session_token: '...', + }, + parameters: { + region: 'us-west-2', + service_name: 'sagemaker', + }, + actions: [ + { + action_type: 'predict', + method: 'POST', + headers: { + 'content-type': 'application/json', + }, + url: 'https://runtime.sagemaker.us-west-2.amazonaws.com/endpoints/lmi-model-2023-06-24-01-35-32-275/invocations', + request_body: '["${parameters.inputs}"]', + }, + ], + }, + }) + .then(({ task_id: taskId }) => + cy.cyclingCheckTask({ + taskId, + }) + ) + .then(({ model_id: modelId }) => { + registeredRemoteModelId = modelId; + return cy.loadMLCommonsModel(modelId); + }) + .then(({ task_id: taskId }) => + cy.cyclingCheckTask({ + taskId, + rejectOnError: false, + }) + ); + }); + after(() => { + if (registeredRemoteModelId) { + cy.unloadMLCommonsModel(registeredRemoteModelId); + cy.wait(1000); + cy.deleteMLCommonsModel(registeredRemoteModelId); + cy.wait(1000); + } + }); + it('should show remote models with External source', () => { + cy.visit(MLC_URL.OVERVIEW); + + cy.get('[aria-label="Search by model name or ID"]').type( + registeredRemoteModelId + ); + + cy.contains('.euiTableRowCell', remoteModelName).should('exist'); + cy.contains('.euiTableRowCell', 'External').should('exist'); + }); + + it('should show show connector details after status details clicked', () => { + cy.visit(MLC_URL.OVERVIEW); + + cy.get('[aria-label="Search by model name or ID"]').type( + registeredRemoteModelId + ); + cy.contains(remoteModelName) + .closest('tr') + .find('[aria-label="view detail"]') + .click(); + cy.get('div[role="dialog"]').contains('External'); + cy.get('div[role="dialog"]').contains('Connector details'); + cy.get('div[role="dialog"]').contains('sagemaker: embedding'); + cy.get('div[role="dialog"]').contains( + 'Test connector for Sagemaker embedding model' + ); + }); }); }); } diff --git a/cypress/utils/plugins/ml-commons-dashboards/commands.js b/cypress/utils/plugins/ml-commons-dashboards/commands.js index a41797712..13767a377 100644 --- a/cypress/utils/plugins/ml-commons-dashboards/commands.js +++ b/cypress/utils/plugins/ml-commons-dashboards/commands.js @@ -9,7 +9,7 @@ Cypress.Commands.add('cyclingCheckTask', ({ taskId, rejectOnError = true }) => new Cypress.Promise((resolve, reject) => { const checkTask = () => { cy.getMLCommonsTask(taskId).then((payload) => { - if (payload.error) { + if (payload && payload.error) { if (rejectOnError) { reject(new Error(payload.error)); return; @@ -17,7 +17,7 @@ Cypress.Commands.add('cyclingCheckTask', ({ taskId, rejectOnError = true }) => resolve(payload); return; } - if (payload.state === 'COMPLETED') { + if (payload && payload.state === 'COMPLETED') { resolve(payload); return; } @@ -110,3 +110,47 @@ Cypress.Commands.add('enableRegisterModelViaURL', () => { failOnStatusCode: false, }); }); + +Cypress.Commands.add('disableConnectorAccessControl', () => { + cy.request('PUT', `${Cypress.env('openSearchUrl')}/_cluster/settings`, { + transient: { + 'plugins.ml_commons.connector_access_control_enabled': false, + }, + }); +}); + +Cypress.Commands.add( + 'setTrustedConnectorEndpointsRegex', + (trustedConnectorEndpointsRegex) => { + cy.request('PUT', `${Cypress.env('openSearchUrl')}/_cluster/settings`, { + transient: { + 'plugins.ml_commons.trusted_connector_endpoints_regex': + trustedConnectorEndpointsRegex, + }, + }); + } +); + +Cypress.Commands.add('createModelConnector', (body) => + cy + .request({ + method: 'POST', + url: MLC_API.CONNECTOR_CREATE, + body, + }) + .then(({ body }) => body) +); + +Cypress.Commands.add('registerModel', (body) => + cy + .request({ + method: 'POST', + url: MLC_API.MODEL_REGISTER, + body, + }) + .then(({ body }) => body) +); + +Cypress.Commands.add('deleteModelConnector', (connectorId) => + cy.request('DELETE', `${MLC_API.CONNECTOR_BASE}/${connectorId}`) +); diff --git a/cypress/utils/plugins/ml-commons-dashboards/constants.js b/cypress/utils/plugins/ml-commons-dashboards/constants.js index edb2b52bb..ac4fac3eb 100644 --- a/cypress/utils/plugins/ml-commons-dashboards/constants.js +++ b/cypress/utils/plugins/ml-commons-dashboards/constants.js @@ -25,6 +25,9 @@ export const MLC_API = { MODEL_UPLOAD: `${MLC_API_BASE}/models/_upload`, MODEL_GROUP_REGISTER: `${MLC_API_BASE}/model_groups/_register`, TASK_BASE: `${MLC_API_BASE}/tasks`, + CONNECTOR_BASE: `${MLC_API_BASE}/connectors`, + CONNECTOR_CREATE: `${MLC_API_BASE}/connectors/_create`, + MODEL_REGISTER: `${MLC_API_BASE}/models/_register`, }; const BASE_MLC_DASHBOARD_API = BASE_PATH + '/api/ml-commons';