Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Enable explanations even w/o explanation template #6190

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: Enable explanations even w/o explanation template #6190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e53a828
feat: Enable explanations even w/o explanation template
anticorrelator Jan 28, 2025
9a9a6a1
Update test
anticorrelator Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions packages/phoenix-evals/src/phoenix/evals/templates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,17 @@ def __init__(
self.rails = rails
self.template = self._normalize_template(template)
self.explanation_template: Optional[List[PromptPartTemplate]]

if explanation_template:
self.explanation_template = self._normalize_template(explanation_template)
self.explanation_parser = parse_label_from_chain_of_thought_response
else:
self.explanation_template = None
self.explanation_label_parser = explanation_label_parser
self.explanation_template = patched_explanation_template(self.template)
self.explanation_parser = parse_label_from_patched_explanation_response

if explanation_label_parser:
self.explanation_label_parser = explanation_label_parser

self._start_delim, self._end_delim = delimiters
self.variables: List[str] = []
for _template in [self.template, self.explanation_template]:
Expand All @@ -174,9 +180,8 @@ def prompt(self, options: Optional[PromptOptions] = None) -> List[PromptPartTemp
return self.template

def extract_label_from_explanation(self, raw_string: str) -> str:
if parser := self.explanation_label_parser:
return parser(raw_string)
return parse_label_from_chain_of_thought_response(raw_string)
parser = self.explanation_parser
return parser(raw_string)

def score(self, rail: str) -> float:
if self._scores is None:
Expand All @@ -195,6 +200,14 @@ def parse_label_from_chain_of_thought_response(raw_string: str) -> str:
return NOT_PARSABLE


def parse_label_from_patched_explanation_response(raw_string: str) -> str:
explanation_delimiter = r"\W*EXPLANATION\W*"
parts = re.split(explanation_delimiter, raw_string, maxsplit=1, flags=re.IGNORECASE)
if parts:
return parts[0]
return NOT_PARSABLE


def normalize_classification_template(
rails: List[str], template: Union[PromptTemplate, ClassificationTemplate, str]
) -> ClassificationTemplate:
Expand Down Expand Up @@ -240,6 +253,27 @@ def normalize_prompt_template(template: Union[PromptTemplate, str]) -> PromptTem
)


def patched_explanation_template(
template_parts: List[PromptPartTemplate],
) -> List[PromptPartTemplate]:
"""
Attempts to patch a template to additionally include an explanation part.
"""
patched_explanation_template_text = (
"*****\n\n"
"After following the previous instructions, add a paragraph that starts with "
"`EXPLANATION: ` and then provide a concise explanation of your reasoning."
)
Comment on lines +262 to +266
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how do we know that the original template doesn't contain this instruction? I think there might be downsides to the fact that now the template is not declarative but constructed inside of the function itself?

The scenario I'm worried about is that Arize is now trying to build templates that are "evaluation" templates. Depending on how you pass it into this, the explanation postfix will now get added (possibly twice).

We should probably be careful here so I'm gonna block for now just to make sure we don't break any workflows.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the postfix only gets added if the explanation template isn't provided, it's just fallback behavior


return [
*template_parts,
PromptPartTemplate(
content_type=PromptPartContentType.TEXT,
template=patched_explanation_template_text,
),
]


def map_template(
dataframe: pd.DataFrame,
template: PromptTemplate,
Expand Down
6 changes: 5 additions & 1 deletion packages/phoenix-evals/tests/phoenix/evals/templates/test_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,14 @@ def test_classification_template_can_beinstantiated_with_no_explanation_template
template = ClassificationTemplate(
rails=["relevant", "irrelevant"], template="is this irrelevant?"
)
assert template.explanation_template is None
assert template.explanation_template is not None
assert len(template.explanation_template) == 2

explanation_options = PromptOptions(provide_explanation=True)
assert template.prompt(options=explanation_options)[0].template == "is this irrelevant?"
assert (
"provide a concise explanation" in template.prompt(options=explanation_options)[1].template
)


def test_template_with_default_delimiters_uses_python_string_formatting():
Expand Down
Loading