From 3f6e44ad301ea609de093edab0ae8f8cca383211 Mon Sep 17 00:00:00 2001 From: Lucas <31957045+SketchingDev@users.noreply.github.com> Date: Sat, 17 Feb 2024 16:22:29 +0000 Subject: [PATCH] Allow prompt placeholders (#170) --- examples/cli-ai-tests/chatgpt-example.yml | 6 ++- .../package.json | 2 +- .../commands/aiTest/createAiTestCommand.ts | 48 +++---------------- .../prompt/generation/promptGenerator.spec.ts | 42 ++++++++++++++++ .../prompt/generation/promptGenerator.ts | 33 +++++++++++++ .../generation/replacePlaceholders.spec.ts | 13 +++++ .../prompt/generation/replacePlaceholders.ts | 11 +++++ .../aiTest/prompt/shouldEndConversation.ts | 37 +++++++------- .../commands/aiTest/testScript/modelSchema.ts | 5 +- .../commands/aiTest/testScript/modelTypes.ts | 1 + .../testScript/validatePromptScript.spec.ts | 6 +++ .../src/commands/aiTest/ui.ts | 23 ++++----- 12 files changed, 149 insertions(+), 78 deletions(-) create mode 100644 packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.spec.ts create mode 100644 packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.ts create mode 100644 packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.spec.ts create mode 100644 packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.ts diff --git a/examples/cli-ai-tests/chatgpt-example.yml b/examples/cli-ai-tests/chatgpt-example.yml index e4c20da..13d57c4 100644 --- a/examples/cli-ai-tests/chatgpt-example.yml +++ b/examples/cli-ai-tests/chatgpt-example.yml @@ -8,8 +8,12 @@ config: scenarios: "Accept survey": setup: + placeholders: + NAME: + - John + - Jane prompt: | - I want you to play the role of a customer talking to a company's online chatbot. You must not + I want you to play the role of a customer called {NAME}, talking to a company's online chatbot. You must not break from this role, and all of your responses must be based on how a customer would realistically talk to a company's chatbot. To help you play the role of a customer consider the following points when writing a response: diff --git a/packages/genesys-web-messaging-tester-cli/package.json b/packages/genesys-web-messaging-tester-cli/package.json index 8c996a7..09ae4bd 100644 --- a/packages/genesys-web-messaging-tester-cli/package.json +++ b/packages/genesys-web-messaging-tester-cli/package.json @@ -1,6 +1,6 @@ { "name": "@ovotech/genesys-web-messaging-tester-cli", - "version": "3.0.0", + "version": "3.0.1", "main": "lib/index.js", "types": "lib/index.d.ts", "license": "Apache-2.0", diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts index 5803957..8f20648 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts @@ -20,6 +20,7 @@ import * as googleAi from './chatCompletionClients/googleVertexAi/createChatComp import * as openAi from './chatCompletionClients/chatGpt/createChatCompletionClient'; import { ChatCompletionClient, Utterance } from './chatCompletionClients/chatCompletionClient'; import { validateOpenAiEnvVariables } from './chatCompletionClients/chatGpt/validateOpenAiEnvVariables'; +import { promptGenerator } from './prompt/generation/promptGenerator'; export interface AiTestCommandDependencies { command?: Command; @@ -155,11 +156,15 @@ export function createAiTestCommand({ const convo = conversationFactory(session); const messages: Utterance[] = []; + const generatedPrompt = promptGenerator(scenario.setup); + outputConfig.writeOut(ui.displayPrompt(generatedPrompt)); + outputConfig.writeOut(ui.conversationStartHeader()); + let endConversation: ShouldEndConversationResult = { hasEnded: false, }; do { - const utterance = await chatCompletionClient.predict(scenario.setup.prompt, messages); + const utterance = await chatCompletionClient.predict(generatedPrompt.prompt, messages); if (utterance) { messages.push(utterance); @@ -198,47 +203,6 @@ export function createAiTestCommand({ if (scenario.followUp) { outputConfig.writeOut(ui.followUpDetailsUnderDevelopment()); } - // if (scenario.followUp) { - // const content = substituteTemplatePlaceholders(scenario.followUp.prompt, transcript); - // const { choices } = await openai.chat.completions.create({ - // model: chatGptModel, - // n: 1, // Number of choices - // temperature, - // messages: [ - // { - // role: 'system', - // content, - // }, - // ], - // }); - // - // if (choices[0].message?.content) { - // const result = containsTerminatingPhrases(choices[0].message.content, { - // fail: scenario.setup.terminatingPhrases.fail, - // pass: scenario.setup.terminatingPhrases.pass, - // }); - // - // outputConfig.writeOut(ui.followUpDetails(choices[0].message.content)); - // if (result.phraseFound) { - // outputConfig.writeOut(ui.followUpResult(result)); - // if (result.phraseIndicates === 'fail') { - // throw new CommandExpectedlyFailedError(); - // } - // } - // } - // - // // endConversation = shouldEndConversation( - // // messages, - // // scenario.setup.terminatingPhrases.fail, - // // scenario.setup.terminatingPhrases.pass, - // // ); - // // if (choices[0].message?.content) { - // // messages.push({ role: 'assistant', content: choices[0].message.content }); - // // await convo.sendText(choices[0].message.content); - // // } else { - // // messages.push({ role: 'assistant', content: '' }); - // // } - // } }, ); } diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.spec.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.spec.ts new file mode 100644 index 0000000..5826bf1 --- /dev/null +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.spec.ts @@ -0,0 +1,42 @@ +import { AiScenarioFollowUpSection } from '../../testScript/modelTypes'; +import { promptGenerator } from './promptGenerator'; + +test('Placeholders are replaced if value present', () => { + const scenario: Pick = { + placeholders: { + FIRST_NAME: ['John'], + SECOND_NAME: ['Doe'], + }, + prompt: 'Your first name is {FIRST_NAME} and your second name is {SECOND_NAME}', + }; + + expect(promptGenerator(scenario)).toStrictEqual({ + prompt: 'Your first name is John and your second name is Doe', + placeholderValues: { FIRST_NAME: 'John', SECOND_NAME: 'Doe' }, + }); +}); + +test('Placeholders ignored if no values present', () => { + const scenario: Pick = { + placeholders: { + FIRST_NAME: [], + }, + prompt: 'Your first name is {FIRST_NAME}', + }; + + expect(promptGenerator(scenario)).toStrictEqual({ + prompt: 'Your first name is {FIRST_NAME}', + placeholderValues: {}, + }); +}); + +test('Original prompt returned if placeholder values not present', () => { + const scenario: Pick = { + prompt: 'Your first name is {FIRST_NAME}', + }; + + expect(promptGenerator(scenario)).toStrictEqual({ + prompt: 'Your first name is {FIRST_NAME}', + placeholderValues: {}, + }); +}); diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.ts new file mode 100644 index 0000000..6d2d0af --- /dev/null +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.ts @@ -0,0 +1,33 @@ +import { AiScenarioFollowUpSection } from '../../testScript/modelTypes'; +import { replacePlaceholders } from './replacePlaceholders'; + +export interface PromptGeneratorResult { + placeholderValues: Record; + prompt: string; +} + +export function promptGenerator( + scenario: Pick, + updatePrompt: typeof replacePlaceholders = replacePlaceholders, + randomIndex = (max: number) => Math.floor(Math.random() * max), +): PromptGeneratorResult { + if (!scenario.placeholders) { + return { + placeholderValues: {}, + prompt: scenario.prompt, + }; + } + + const chosenValues: Record = Object.fromEntries( + Object.entries(scenario.placeholders) + .filter(([, values]) => values.length > 0) + .map(([placeholder, values]) => { + return [placeholder, values[randomIndex(values.length)]]; + }), + ); + + return { + placeholderValues: chosenValues, + prompt: updatePrompt(scenario.prompt, chosenValues), + }; +} diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.spec.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.spec.ts new file mode 100644 index 0000000..cb7ae32 --- /dev/null +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.spec.ts @@ -0,0 +1,13 @@ +import { replacePlaceholders } from './replacePlaceholders'; + +test('Placeholders are replaced', () => { + expect( + replacePlaceholders('{FIRST_NAME} {LAST_NAME} {ABC}', { FIRST_NAME: 'John', LAST_NAME: 'Doe' }), + ).toStrictEqual('John Doe {ABC}'); +}); + +test('Prompt with missing placeholders are ignored', () => { + expect(replacePlaceholders('{FIRST_NAME} {LAST_NAME}', { FIRST_NAME: 'John' })).toStrictEqual( + 'John {LAST_NAME}', + ); +}); diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.ts new file mode 100644 index 0000000..cda5a18 --- /dev/null +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.ts @@ -0,0 +1,11 @@ +export function replacePlaceholders( + prompt: string, + placeholderValues: Record, +): string { + return Object.entries(placeholderValues).reduce( + (previousValue, [placeholderKey, placeholderValue]) => { + return previousValue.replace(`{${placeholderKey}}`, placeholderValue); + }, + prompt, + ); +} diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts index fe262c6..5cb659d 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts @@ -37,10 +37,9 @@ export function shouldEndConversation( }; } - const lastAiMsg = utterances.filter((m) => m.role === 'customer').slice(-1); - - if (lastAiMsg[0]?.content) { - const phraseResult = containsTerminatingPhrases(lastAiMsg[0].content, { + const lastMsg = utterances.slice(-1); + if (lastMsg[0]?.content) { + const phraseResult = containsTerminatingPhrases(lastMsg[0].content, { pass: passPhrases, fail: failPhrases, }); @@ -50,7 +49,7 @@ export function shouldEndConversation( hasEnded: true, reason: { type: phraseResult.phraseIndicates, - description: `Terminating phrase found in response: '${lastAiMsg[0].content}'`, + description: `Terminating phrase found in response: '${lastMsg[0].content}'`, }, }; } @@ -70,20 +69,20 @@ export function shouldEndConversation( // } // } - const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2); - if (lastTwoChatBotMsgs.length === 2) { - const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content; - if (areMessagesTheSame) { - return { - hasEnded: true, - - reason: { - type: 'fail', - description: 'The Chatbot repeated itself', - }, - }; - } - } + // const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2); + // if (lastTwoChatBotMsgs.length === 2) { + // const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content; + // if (areMessagesTheSame) { + // return { + // hasEnded: true, + // + // reason: { + // type: 'fail', + // description: 'The Chatbot repeated itself', + // }, + // }; + // } + // } return { hasEnded: false }; } diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelSchema.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelSchema.ts index 99267b1..6b1c905 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelSchema.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelSchema.ts @@ -34,7 +34,7 @@ export const schema = Joi.object({ temperature: Joi.number(), }), }), - }), + }).required(), }).required(), scenarios: Joi.object() .min(1) @@ -42,6 +42,9 @@ export const schema = Joi.object({ /./, Joi.object({ setup: Joi.object({ + placeholders: Joi.object() + .min(1) + .pattern(/./, Joi.array().items(Joi.string()).required()), prompt: Joi.string().required(), terminatingPhrases: Joi.object({ pass: Joi.array().items(Joi.string()).min(1).required(), diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelTypes.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelTypes.ts index 5b7ed9b..b91daca 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelTypes.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelTypes.ts @@ -8,6 +8,7 @@ export interface AiScenarioSetupSection { export interface AiScenarioFollowUpSection { readonly prompt: string; + readonly placeholders?: Record; readonly terminatingPhrases: { readonly pass: string[]; readonly fail: string[]; diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/validatePromptScript.spec.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/validatePromptScript.spec.ts index 66306eb..d4fe98a 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/validatePromptScript.spec.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/validatePromptScript.spec.ts @@ -17,6 +17,9 @@ test('Valid', () => { scenarios: { 'test-name-of-test-1': { setup: { + placeholders: { + TEST: ['test-1', 'test-2'], + }, prompt: 'test-prompt-1', terminatingPhrases: { fail: ['test-failing-phrase-1'], @@ -65,6 +68,9 @@ test('Valid', () => { 'test-name-of-test-1': { setup: { prompt: 'test-prompt-1', + placeholders: { + TEST: ['test-1', 'test-2'], + }, terminatingPhrases: { fail: ['test-failing-phrase-1'], pass: ['test-passing-phrase-1'], diff --git a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/ui.ts b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/ui.ts index 63a393f..9aac176 100644 --- a/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/ui.ts +++ b/packages/genesys-web-messaging-tester-cli/src/commands/aiTest/ui.ts @@ -2,8 +2,8 @@ import chalk from 'chalk'; import { ValidationError } from 'joi'; import { ShouldEndConversationEndedResult } from './prompt/shouldEndConversation'; import { TranscribedMessage } from '@ovotech/genesys-web-messaging-tester'; -import { PhraseFound } from './prompt/containsTerminatingPhrases'; import { PreflightError } from './chatCompletionClients/chatCompletionClient'; +import { PromptGeneratorResult } from './prompt/generation/promptGenerator'; export class Ui { /** @@ -51,6 +51,14 @@ export class Ui { ); } + public displayPrompt({ prompt }: PromptGeneratorResult): string { + return Ui.trailingNewline(chalk.grey(prompt)); + } + + public conversationStartHeader(): string { + return Ui.trailingNewline(['Conversation', '------------'].join('\n')); + } + public testResult(result: ShouldEndConversationEndedResult): string { const resultMessage = result.reason.type === 'pass' @@ -74,17 +82,4 @@ export class Ui { chalk.bold.yellow('Follow up definitions ignored, as functionality is under development'), ); } - - public followUpDetails(feedback: string): string { - return Ui.trailingNewline(['\n---------------------', feedback].join('\n')); - } - - public followUpResult(result: PhraseFound): string { - const resultMessage = - result.phraseIndicates === 'fail' - ? chalk.bold.red(`FAILED: ${result.subject}`) - : chalk.bold.green(`PASSED: ${result.subject}`); - - return Ui.trailingNewline(['\n---------------------', resultMessage].join('\n')); - } }