From ba9b33d4a1ff42671093844fc9566a4fea143dba Mon Sep 17 00:00:00 2001
From: Kosma Dunikowski <kosmadunikowski@gmail.com>
Date: Mon, 4 Nov 2024 13:39:19 +0100
Subject: [PATCH] Add detailed explanations for failed assertions

Fixes #2

Add detailed explanations for failed assertions using LLM.

* Modify `intentguard/intentguard.py` to include a `_generate_explanation` method that generates detailed explanations for failed assertions using the LLM.
* Update the `assert_code` method to call `_generate_explanation` when an assertion fails and include the explanation in the `AssertionError`.
* Add a new prompt template `explanation_prompt` in `intentguard/prompts.py` for generating detailed explanations for failed assertions.
* Update the existing test case `test_assert_code_false` in `tests/test_intentguard.py` to check for the presence of the explanation in the `AssertionError`.

---

For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/kdunee/intentguard/issues/2?shareId=XXXX-XXXX-XXXX-XXXX).
---
 intentguard/intentguard.py | 38 ++++++++++++++++++++++++++++++-
 intentguard/prompts.py     | 46 ++++++++++++++++++++++++++++++++++++++
 tests/test_intentguard.py  |  3 ++-
 3 files changed, 85 insertions(+), 2 deletions(-)

diff --git a/intentguard/intentguard.py b/intentguard/intentguard.py
index 18b8ac4..5b4d2b6 100644
--- a/intentguard/intentguard.py
+++ b/intentguard/intentguard.py
@@ -66,8 +66,9 @@ def assert_code(
         final_result = self._vote_on_results(results)
 
         if not final_result:
+            explanation = self._generate_explanation(objects_text, expectation, options)
             raise AssertionError(
-                f'Expected "{expectation}" to be true, but it was false'
+                f'Expected "{expectation}" to be true, but it was false. Explanation: {explanation}'
             )
 
     def _generate_objects_text(self, params: Dict[str, object]) -> str:
@@ -147,6 +148,41 @@ def _send_completion_request(
         )
         return json.loads(response.choices[0].message.content)["result"]
 
+    def _generate_explanation(
+        self, objects_text: str, expectation: str, options: IntentGuardOptions
+    ) -> str:
+        """
+        Generate a detailed explanation for a failed assertion using the LLM.
+
+        This method sends a request to the LLM to generate a human-readable explanation
+        for why the given expectation was not met based on the provided objects.
+
+        Args:
+            objects_text (str): The formatted string of object source codes.
+            expectation (str): The condition that was evaluated.
+            options (IntentGuardOptions): The options for the LLM request.
+
+        Returns:
+            str: A detailed explanation of why the assertion failed.
+        """
+        explanation_prompt = f"""**Objects:**
+{objects_text}
+
+**Failed Condition:**
+"{expectation}"
+"""
+        messages = [
+            {"content": system_prompt, "role": "system"},
+            {"content": explanation_prompt, "role": "user"},
+        ]
+
+        response = completion(
+            model=options.model,
+            messages=messages,
+            temperature=1e-3,
+        )
+        return response.choices[0].message.content
+
     def _vote_on_results(self, results: list) -> bool:
         """
         Determine the final result based on voting.
diff --git a/intentguard/prompts.py b/intentguard/prompts.py
index 1411d08..2ee0b46 100644
--- a/intentguard/prompts.py
+++ b/intentguard/prompts.py
@@ -50,6 +50,52 @@ def another_method(self):
 }
 ```"""
 
+explanation_prompt = """Analyze Python code to explain why a given condition against named objects (classes or methods) failed.
+
+You will receive:
+- A list of named objects, where each object's value is the code of a class or method.
+- A text of a condition that uses these object names.
+- An indication that the condition was not met.
+
+Your task is to provide a detailed explanation of why the condition was not fulfilled for the given objects.
+
+# Steps
+
+1. Parse the input to identify and load the classes or methods from the named objects.
+2. Analyze the code structure of each named object, ensuring to understand the classes, methods, and their interactions.
+3. Evaluate the specified condition using the parsed objects, referring to them by their given names.
+4. Determine why the condition does not hold true based on the code analysis.
+5. Provide a detailed explanation of the reasons for the failure.
+
+# Output Format
+
+Output a text explanation detailing why the condition was not met.
+
+# Examples
+
+### Input
+**Objects:**
+{obj1}:
+```py
+class MyClass:
+    def method(self):
+        return 5
+```
+
+{obj2}:
+```py
+class AnotherClass:
+    def another_method(self):
+       return 10
+```
+
+**Failed Condition:**
+"{obj1} has method method and {obj2} has method another_method"
+
+### Output
+The condition "{obj1} has method method and {obj2} has method another_method" was not met because {obj1} does not have a method named 'method' and {obj2} does not have a method named 'another_method'.
+"""
+
 reponse_schema = {
     "name": "boolean_result",
     "strict": True,
diff --git a/tests/test_intentguard.py b/tests/test_intentguard.py
index 403c679..1b10b1a 100644
--- a/tests/test_intentguard.py
+++ b/tests/test_intentguard.py
@@ -14,10 +14,11 @@ def test_assert_code_true(self):
         )
 
     def test_assert_code_false(self):
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(AssertionError) as cm:
             self.guard.assert_code(
                 "{class} should not have any methods", {"class": IntentGuard}
             )
+        self.assertIn("Explanation:", str(cm.exception))
 
     def test_guard_options(self):
         self.guard.assert_code(