Skip to content

Commit

Permalink
feat: mocked agent testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@mdonnalley
mdonnalley committed Nov 13, 2024
1 parent 8df61a9 commit 334988d
Show file tree
Hide file tree
Showing 5 changed files with 685 additions and 89 deletions.
4 changes: 3 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
"url": "https://github.com/forcedotcom/agents.git"
},
"dependencies": {
"@oclif/table": "^0.3.3",
"@salesforce/core": "^8.5.2",
"@salesforce/kit": "^3.2.3"
"@salesforce/kit": "^3.2.3",
"nock": "^13.5.6"
},
"devDependencies": {
"@salesforce/cli-plugins-testkit": "^5.3.20",
Expand Down
12 changes: 5 additions & 7 deletions src/agent.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import { inspect } from 'node:util';
import { Connection, Logger, SfError, SfProject } from '@salesforce/core';
import { Duration, sleep } from '@salesforce/kit';
import { mockOrRequest } from './mockDir';
import { MaybeMock } from './mockDir';
import {
type SfAgent,
type AgentCreateConfig,
Expand All @@ -20,9 +20,11 @@ import {

export class Agent implements SfAgent {
private logger: Logger;
private maybeMock: MaybeMock;

public constructor(private connection: Connection, private project: SfProject) {
public constructor(connection: Connection, private project: SfProject) {
this.logger = Logger.childFromRoot(this.constructor.name);
this.maybeMock = new MaybeMock(connection);
}

public async create(config: AgentCreateConfig): Promise<AgentCreateResponse> {
Expand All @@ -46,11 +48,7 @@ export class Agent implements SfAgent {
this.verifyAgentSpecConfig(config);

let agentSpec: AgentJobSpec;
const response = await mockOrRequest<AgentJobSpecCreateResponse>(
this.connection,
'GET',
this.buildAgentJobSpecUrl(config)
);
const response = await this.maybeMock.request<AgentJobSpecCreateResponse>('GET', this.buildAgentJobSpecUrl(config));

if (response.isSuccess && response.jobSpecs) {
agentSpec = response.jobSpecs;
Expand Down
164 changes: 154 additions & 10 deletions src/agentTester.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
* Licensed under the BSD 3-Clause license.
* For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
*/
import { Connection, PollingClient, StatusResult } from '@salesforce/core';
import { Duration } from '@salesforce/kit';
import { MaybeMock } from './mockDir';

import { Connection } from '@salesforce/core';
import { mockOrRequest } from './mockDir';
type Format = 'human' | 'tap' | 'junit' | 'json';

type AgentTestStartResponse = {
id: string;
Expand Down Expand Up @@ -37,25 +39,167 @@ type AgentTestDetailsResponse = {
};

export class AgentTester {
public constructor(private connection: Connection) {}
private maybeMock: MaybeMock;
public constructor(connection: Connection) {
this.maybeMock = new MaybeMock(connection);
}

public async start(suiteId: string): Promise<{ id: string }> {
const url = `/services/data/${this.connection.getApiVersion()}/einstein/ai-evaluations/runs`;
const url = '/einstein/ai-evaluations/runs';

return mockOrRequest<AgentTestStartResponse>(this.connection, 'POST', url, {
return this.maybeMock.request<AgentTestStartResponse>('POST', url, {
aiEvaluationSuiteDefinition: suiteId,
});
}

public async status(jobId: string): Promise<AgentTestStatusResponse> {
const url = `/services/data/${this.connection.getApiVersion()}/einstein/ai-evaluations/runs/${jobId}`;
const url = `/einstein/ai-evaluations/runs/${jobId}`;

return this.maybeMock.request<AgentTestStatusResponse>('GET', url);
}

public async poll(
jobId: string,
{
format = 'human',
timeout = Duration.minutes(5),
}: {
format?: Format;
timeout?: Duration;
}
): Promise<{ response: AgentTestDetailsResponse; formatted: string }> {
const client = await PollingClient.create({
poll: async (): Promise<StatusResult> => {
const { status } = await this.status(jobId);
if (status === 'COMPLETED') {
return { payload: await this.details(jobId, format), completed: true };
}

return mockOrRequest<AgentTestStatusResponse>(this.connection, 'GET', url);
return { completed: false };
},
frequency: Duration.seconds(1),
timeout,
});

const result = await client.subscribe<{ response: AgentTestDetailsResponse; formatted: string }>();
return result;
}

public async details(jobId: string): Promise<AgentTestDetailsResponse> {
const url = `/services/data/${this.connection.getApiVersion()}/einstein/ai-evaluations/runs/${jobId}/details`;
public async details(
jobId: string,
format: Format = 'human'
): Promise<{ response: AgentTestDetailsResponse; formatted: string }> {
const url = `/einstein/ai-evaluations/runs/${jobId}/details`;

const response = await this.maybeMock.request<AgentTestDetailsResponse>('GET', url);
return {
response,
formatted:
format === 'human'
? await humanFormat(response)
: format === 'tap'
? await tapFormat(response)
: format === 'junit'
? await junitFormat(response)
: await jsonFormat(response),
};
}
}

return mockOrRequest<AgentTestDetailsResponse>(this.connection, 'GET', url);
export async function humanFormat(details: AgentTestDetailsResponse): Promise<string> {
// TODO: these tables need to follow the same defaults that sf-plugins-core uses
// TODO: the api response isn't finalized so this is just a POC
const { makeTable } = await import('@oclif/table');
const tables: string[] = [];
for (const aiEvalDef of details.tests) {
for (const result of aiEvalDef.results) {
const table = makeTable({
title: `Test Results for ${aiEvalDef.AiEvaluationDefinition} (#${result.test_number})`,
data: result.results.map((r) => ({
'TEST NAME': r.name,
OUTCOME: r.is_pass ? 'Pass' : 'Fail',
MESSAGE: r.error ?? '',
'RUNTIME (MS)': r.execution_time_ms,
})),
});
tables.push(table);
}
}

return tables.join('\n');
}

export async function junitFormat(details: AgentTestDetailsResponse): Promise<string> {
// APEX EXAMPLE
// <?xml version="1.0" encoding="UTF-8"?>
// <testsuites>
// <testsuite name="force.apex" timestamp="2024-11-13T19:19:23.000Z" hostname="https://energy-site-1368-dev-ed.scratch.my.salesforce.com" tests="11" failures="0" errors="0" time="2.57">
// <properties>
// <property name="outcome" value="Successful"/>
// <property name="testsRan" value="11"/>
// <property name="passing" value="11"/>
// <property name="failing" value="0"/>
// <property name="skipped" value="0"/>
// <property name="passRate" value="100%"/>
// <property name="failRate" value="0%"/>
// <property name="testStartTime" value="Wed Nov 13 2024 12:19:23 PM"/>
// <property name="testSetupTimeInMs" value="0"/>
// <property name="testExecutionTime" value="2.57 s"/>
// <property name="testTotalTime" value="2.57 s"/>
// <property name="commandTime" value="0.17 s"/>
// <property name="hostname" value="https://energy-site-1368-dev-ed.scratch.my.salesforce.com"/>
// <property name="orgId" value="00DEi000006OlrxMAC"/>
// <property name="username" value="test-mgoe8ogsltwe@example.com"/>
// <property name="testRunId" value="707Ei00000dTRSa"/>
// <property name="userId" value="005Ei00000FkGU9IAN"/>
// </properties>
// <testcase name="importSampleData" classname="TestSampleDataController" time="0.27">
// </testcase>
// <testcase name="blankAddress" classname="GeocodingServiceTest" time="0.01">
// </testcase>
// <testcase name="errorResponse" classname="GeocodingServiceTest" time="0.01">
// </testcase>
// <testcase name="successResponse" classname="GeocodingServiceTest" time="0.01">
// </testcase>
// <testcase name="createFileFailsWhenIncorrectBase64Data" classname="FileUtilitiesTest" time="0.10">
// </testcase>
// <testcase name="createFileFailsWhenIncorrectFilename" classname="FileUtilitiesTest" time="0.03">
// </testcase>
// <testcase name="createFileFailsWhenIncorrectRecordId" classname="FileUtilitiesTest" time="0.35">
// </testcase>
// <testcase name="createFileSucceedsWhenCorrectInput" classname="FileUtilitiesTest" time="0.22">
// </testcase>
// <testcase name="testGetPagedPropertyList" classname="TestPropertyController" time="1.01">
// </testcase>
// <testcase name="testGetPicturesNoResults" classname="TestPropertyController" time="0.06">
// </testcase>
// <testcase name="testGetPicturesWithResults" classname="TestPropertyController" time="0.51">
// </testcase>
// </testsuite>
// </testsuites>
await Promise.reject(new Error('Not implemented'));
return JSON.stringify(details, null, 2);
}

export async function tapFormat(details: AgentTestDetailsResponse): Promise<string> {
// APEX EXAMPLE
// 1..11
// ok 1 TestPropertyController.testGetPagedPropertyList
// ok 2 TestPropertyController.testGetPicturesNoResults
// ok 3 TestPropertyController.testGetPicturesWithResults
// ok 4 FileUtilitiesTest.createFileFailsWhenIncorrectBase64Data
// ok 5 FileUtilitiesTest.createFileFailsWhenIncorrectFilename
// ok 6 FileUtilitiesTest.createFileFailsWhenIncorrectRecordId
// ok 7 FileUtilitiesTest.createFileSucceedsWhenCorrectInput
// ok 8 TestSampleDataController.importSampleData
// ok 9 GeocodingServiceTest.blankAddress
// ok 10 GeocodingServiceTest.errorResponse
// ok 11 GeocodingServiceTest.successResponse
// # Run "sf apex get test -i 707Ei00000dUJry -o test-mgoe8ogsltwe@example.com --result-format <format>" to retrieve test results in a different format.
await Promise.reject(new Error('Not implemented'));
return JSON.stringify(details, null, 2);
}

export async function jsonFormat(details: AgentTestDetailsResponse): Promise<string> {
return Promise.resolve(JSON.stringify(details, null, 2));
}
Loading

0 comments on commit 334988d

Please sign in to comment.