From 47355146c4ec8f1448ce99d278bd36b2511f3dbf Mon Sep 17 00:00:00 2001
From: Chris <66376200+crickman@users.noreply.github.com>
Date: Thu, 17 Aug 2023 17:14:42 -0700
Subject: [PATCH] Re-ordered plug-in result to be included _after_ chat-history
 (#182)

### Motivation and Context

<!-- Thank you for your contribution to the copilot-chat repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

Customer reported stale plug-in results. Re-ordering prompt to include
plug-in results last may impact prioritizing this data for certain
cases. (Isolated testing indicates that the model can respond to
priority hints via ordering, but other factors may influence model
behavior as well...such as chat history or memory presenting stronger
semantic context.)

### Description
<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

Move planner results to after chat-history:

```
[SOURCE START]

This is the result of invoking the functions listed after "PLUGINS USED:" to retrieve additional information outside of the data you were trained on. You can use this data to help answer the user's query.

PLUGINS USED: ListPluginPlugin.GetList.

RESULT: {
"content": "eba26d18-b4bb-46d4-b6b6-cbf60d4312e8\n12ed4781-5b22-48a0-b4e9-a351e546e7ff\n90605eb2-51ce-4058-8ae0-dfbe5ed455f7\nf51d2923-f97a-41b8-97d4-991b9f46e260",
"contentType": "text/plain"
}

[SOURCE END]
```

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [Contribution
Guidelines](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/copilot-chat/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone :smile:
---
 webapi/Skills/ChatSkills/ChatSkill.cs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/webapi/Skills/ChatSkills/ChatSkill.cs b/webapi/Skills/ChatSkills/ChatSkill.cs
index d06485ef0..8d03d898f 100644
--- a/webapi/Skills/ChatSkills/ChatSkill.cs
+++ b/webapi/Skills/ChatSkills/ChatSkill.cs
@@ -383,10 +383,13 @@ private async Task<ChatMessage> GetChatResponseAsync(string chatId, string userI
         var documentMemories = tasks[1];
 
         // Fill in the chat history if there is any token budget left
-        var chatContextComponents = new List<string>() { chatMemories, documentMemories, planResult };
+        var chatContextComponents = new List<string>() { chatMemories, documentMemories };
         var chatContextText = string.Join("\n\n", chatContextComponents.Where(c => !string.IsNullOrEmpty(c)));
-        var chatHistoryTokenLimit = remainingToken - TokenUtilities.TokenCount(chatContextText);
+        var chatHistoryTokenLimit = remainingToken - TokenUtilities.TokenCount(chatContextText) - TokenUtilities.TokenCount(planResult);
+
         string chatHistory = string.Empty;
+
+        // Append the chat history, if allowed.
         if (chatHistoryTokenLimit > 0)
         {
             await this.UpdateBotResponseStatusOnClient(chatId, "Extracting chat history");
@@ -395,6 +398,12 @@ private async Task<ChatMessage> GetChatResponseAsync(string chatId, string userI
             chatContextText = $"{chatContextText}\n{chatHistory}";
         }
 
+        // Append the plan result last, if exists, to imply precedence.
+        if (!string.IsNullOrWhiteSpace(planResult))
+        {
+            chatContextText = $"{chatContextText}\n{planResult}";
+        }
+
         // Set variables needed in prompt
         chatContext.Variables.Set("audience", audience);
         chatContext.Variables.Set("userIntent", userIntent);