Skip to content

Commit

Permalink
Allow prompt placeholders (#170)
Browse files Browse the repository at this point in the history
  • Loading branch information
SketchingDev authored Feb 17, 2024
1 parent 5474f39 commit 3f6e44a
Show file tree
Hide file tree
Showing 12 changed files with 149 additions and 78 deletions.
6 changes: 5 additions & 1 deletion examples/cli-ai-tests/chatgpt-example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ config:
scenarios:
"Accept survey":
setup:
placeholders:
NAME:
- John
- Jane
prompt: |
I want you to play the role of a customer talking to a company's online chatbot. You must not
I want you to play the role of a customer called {NAME}, talking to a company's online chatbot. You must not
break from this role, and all of your responses must be based on how a customer would realistically talk to a company's chatbot.
To help you play the role of a customer consider the following points when writing a response:
Expand Down
2 changes: 1 addition & 1 deletion packages/genesys-web-messaging-tester-cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@ovotech/genesys-web-messaging-tester-cli",
"version": "3.0.0",
"version": "3.0.1",
"main": "lib/index.js",
"types": "lib/index.d.ts",
"license": "Apache-2.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import * as googleAi from './chatCompletionClients/googleVertexAi/createChatComp
import * as openAi from './chatCompletionClients/chatGpt/createChatCompletionClient';
import { ChatCompletionClient, Utterance } from './chatCompletionClients/chatCompletionClient';
import { validateOpenAiEnvVariables } from './chatCompletionClients/chatGpt/validateOpenAiEnvVariables';
import { promptGenerator } from './prompt/generation/promptGenerator';

export interface AiTestCommandDependencies {
command?: Command;
Expand Down Expand Up @@ -155,11 +156,15 @@ export function createAiTestCommand({
const convo = conversationFactory(session);
const messages: Utterance[] = [];

const generatedPrompt = promptGenerator(scenario.setup);
outputConfig.writeOut(ui.displayPrompt(generatedPrompt));
outputConfig.writeOut(ui.conversationStartHeader());

let endConversation: ShouldEndConversationResult = {
hasEnded: false,
};
do {
const utterance = await chatCompletionClient.predict(scenario.setup.prompt, messages);
const utterance = await chatCompletionClient.predict(generatedPrompt.prompt, messages);

if (utterance) {
messages.push(utterance);
Expand Down Expand Up @@ -198,47 +203,6 @@ export function createAiTestCommand({
if (scenario.followUp) {
outputConfig.writeOut(ui.followUpDetailsUnderDevelopment());
}
// if (scenario.followUp) {
// const content = substituteTemplatePlaceholders(scenario.followUp.prompt, transcript);
// const { choices } = await openai.chat.completions.create({
// model: chatGptModel,
// n: 1, // Number of choices
// temperature,
// messages: [
// {
// role: 'system',
// content,
// },
// ],
// });
//
// if (choices[0].message?.content) {
// const result = containsTerminatingPhrases(choices[0].message.content, {
// fail: scenario.setup.terminatingPhrases.fail,
// pass: scenario.setup.terminatingPhrases.pass,
// });
//
// outputConfig.writeOut(ui.followUpDetails(choices[0].message.content));
// if (result.phraseFound) {
// outputConfig.writeOut(ui.followUpResult(result));
// if (result.phraseIndicates === 'fail') {
// throw new CommandExpectedlyFailedError();
// }
// }
// }
//
// // endConversation = shouldEndConversation(
// // messages,
// // scenario.setup.terminatingPhrases.fail,
// // scenario.setup.terminatingPhrases.pass,
// // );
// // if (choices[0].message?.content) {
// // messages.push({ role: 'assistant', content: choices[0].message.content });
// // await convo.sendText(choices[0].message.content);
// // } else {
// // messages.push({ role: 'assistant', content: '' });
// // }
// }
},
);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { AiScenarioFollowUpSection } from '../../testScript/modelTypes';
import { promptGenerator } from './promptGenerator';

test('Placeholders are replaced if value present', () => {
const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
placeholders: {
FIRST_NAME: ['John'],
SECOND_NAME: ['Doe'],
},
prompt: 'Your first name is {FIRST_NAME} and your second name is {SECOND_NAME}',
};

expect(promptGenerator(scenario)).toStrictEqual({
prompt: 'Your first name is John and your second name is Doe',
placeholderValues: { FIRST_NAME: 'John', SECOND_NAME: 'Doe' },
});
});

test('Placeholders ignored if no values present', () => {
const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
placeholders: {
FIRST_NAME: [],
},
prompt: 'Your first name is {FIRST_NAME}',
};

expect(promptGenerator(scenario)).toStrictEqual({
prompt: 'Your first name is {FIRST_NAME}',
placeholderValues: {},
});
});

test('Original prompt returned if placeholder values not present', () => {
const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
prompt: 'Your first name is {FIRST_NAME}',
};

expect(promptGenerator(scenario)).toStrictEqual({
prompt: 'Your first name is {FIRST_NAME}',
placeholderValues: {},
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { AiScenarioFollowUpSection } from '../../testScript/modelTypes';
import { replacePlaceholders } from './replacePlaceholders';

export interface PromptGeneratorResult {
placeholderValues: Record<string, string>;
prompt: string;
}

export function promptGenerator(
scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'>,
updatePrompt: typeof replacePlaceholders = replacePlaceholders,
randomIndex = (max: number) => Math.floor(Math.random() * max),
): PromptGeneratorResult {
if (!scenario.placeholders) {
return {
placeholderValues: {},
prompt: scenario.prompt,
};
}

const chosenValues: Record<string, string> = Object.fromEntries(
Object.entries(scenario.placeholders)
.filter(([, values]) => values.length > 0)
.map(([placeholder, values]) => {
return [placeholder, values[randomIndex(values.length)]];
}),
);

return {
placeholderValues: chosenValues,
prompt: updatePrompt(scenario.prompt, chosenValues),
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { replacePlaceholders } from './replacePlaceholders';

test('Placeholders are replaced', () => {
expect(
replacePlaceholders('{FIRST_NAME} {LAST_NAME} {ABC}', { FIRST_NAME: 'John', LAST_NAME: 'Doe' }),
).toStrictEqual('John Doe {ABC}');
});

test('Prompt with missing placeholders are ignored', () => {
expect(replacePlaceholders('{FIRST_NAME} {LAST_NAME}', { FIRST_NAME: 'John' })).toStrictEqual(
'John {LAST_NAME}',
);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export function replacePlaceholders(
prompt: string,
placeholderValues: Record<string, string>,
): string {
return Object.entries(placeholderValues).reduce(
(previousValue, [placeholderKey, placeholderValue]) => {
return previousValue.replace(`{${placeholderKey}}`, placeholderValue);
},
prompt,
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,9 @@ export function shouldEndConversation(
};
}

const lastAiMsg = utterances.filter((m) => m.role === 'customer').slice(-1);

if (lastAiMsg[0]?.content) {
const phraseResult = containsTerminatingPhrases(lastAiMsg[0].content, {
const lastMsg = utterances.slice(-1);
if (lastMsg[0]?.content) {
const phraseResult = containsTerminatingPhrases(lastMsg[0].content, {
pass: passPhrases,
fail: failPhrases,
});
Expand All @@ -50,7 +49,7 @@ export function shouldEndConversation(
hasEnded: true,
reason: {
type: phraseResult.phraseIndicates,
description: `Terminating phrase found in response: '${lastAiMsg[0].content}'`,
description: `Terminating phrase found in response: '${lastMsg[0].content}'`,
},
};
}
Expand All @@ -70,20 +69,20 @@ export function shouldEndConversation(
// }
// }

const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2);
if (lastTwoChatBotMsgs.length === 2) {
const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content;
if (areMessagesTheSame) {
return {
hasEnded: true,

reason: {
type: 'fail',
description: 'The Chatbot repeated itself',
},
};
}
}
// const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2);
// if (lastTwoChatBotMsgs.length === 2) {
// const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content;
// if (areMessagesTheSame) {
// return {
// hasEnded: true,
//
// reason: {
// type: 'fail',
// description: 'The Chatbot repeated itself',
// },
// };
// }
// }

return { hasEnded: false };
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,17 @@ export const schema = Joi.object<TestPromptFile>({
temperature: Joi.number(),
}),
}),
}),
}).required(),
}).required(),
scenarios: Joi.object()
.min(1)
.pattern(
/./,
Joi.object({
setup: Joi.object({
placeholders: Joi.object()
.min(1)
.pattern(/./, Joi.array().items(Joi.string()).required()),
prompt: Joi.string().required(),
terminatingPhrases: Joi.object({
pass: Joi.array().items(Joi.string()).min(1).required(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export interface AiScenarioSetupSection {

export interface AiScenarioFollowUpSection {
readonly prompt: string;
readonly placeholders?: Record<string, string[]>;
readonly terminatingPhrases: {
readonly pass: string[];
readonly fail: string[];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ test('Valid', () => {
scenarios: {
'test-name-of-test-1': {
setup: {
placeholders: {
TEST: ['test-1', 'test-2'],
},
prompt: 'test-prompt-1',
terminatingPhrases: {
fail: ['test-failing-phrase-1'],
Expand Down Expand Up @@ -65,6 +68,9 @@ test('Valid', () => {
'test-name-of-test-1': {
setup: {
prompt: 'test-prompt-1',
placeholders: {
TEST: ['test-1', 'test-2'],
},
terminatingPhrases: {
fail: ['test-failing-phrase-1'],
pass: ['test-passing-phrase-1'],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import chalk from 'chalk';
import { ValidationError } from 'joi';
import { ShouldEndConversationEndedResult } from './prompt/shouldEndConversation';
import { TranscribedMessage } from '@ovotech/genesys-web-messaging-tester';
import { PhraseFound } from './prompt/containsTerminatingPhrases';
import { PreflightError } from './chatCompletionClients/chatCompletionClient';
import { PromptGeneratorResult } from './prompt/generation/promptGenerator';

export class Ui {
/**
Expand Down Expand Up @@ -51,6 +51,14 @@ export class Ui {
);
}

public displayPrompt({ prompt }: PromptGeneratorResult): string {
return Ui.trailingNewline(chalk.grey(prompt));
}

public conversationStartHeader(): string {
return Ui.trailingNewline(['Conversation', '------------'].join('\n'));
}

public testResult(result: ShouldEndConversationEndedResult): string {
const resultMessage =
result.reason.type === 'pass'
Expand All @@ -74,17 +82,4 @@ export class Ui {
chalk.bold.yellow('Follow up definitions ignored, as functionality is under development'),
);
}

public followUpDetails(feedback: string): string {
return Ui.trailingNewline(['\n---------------------', feedback].join('\n'));
}

public followUpResult(result: PhraseFound): string {
const resultMessage =
result.phraseIndicates === 'fail'
? chalk.bold.red(`FAILED: ${result.subject}`)
: chalk.bold.green(`PASSED: ${result.subject}`);

return Ui.trailingNewline(['\n---------------------', resultMessage].join('\n'));
}
}

0 comments on commit 3f6e44a

Please sign in to comment.