feat: add mistral as a new guardrail provider

plugins/index.ts

@@ -32,6 +32,20 @@ import { handler as patronusnoRacialBias } from './patronus/noRacialBias';
 import { handler as patronusretrievalAnswerRelevance } from './patronus/retrievalAnswerRelevance';
 import { handler as patronustoxicity } from './patronus/toxicity';
 import { handler as patronuscustom } from './patronus/custom';
+import { mistralGuardrailHandler } from './mistral';
+import { PluginHandler } from './types';
+
+const mistralGuardCategories = [
+  'sexual',
+  'hate_and_discrimination',
+  'violence_and_threats',
+  'dangerous_and_criminal_content',
+  'selfharm',
+  'health',
+  'financial',
+  'law',
+  'pii',
+];
 
 export const plugins = {
   default: {
@@ -80,4 +94,11 @@ export const plugins = {
     toxicity: patronustoxicity,
     custom: patronuscustom,
   },
+  mistral: mistralGuardCategories.reduce(
+    (config, category) => {
+      config[category] = mistralGuardrailHandler;
+      return config;
+    },
+    {} as Record<string, PluginHandler>
+  ),
 };

plugins/mistral/index.ts

@@ -0,0 +1,119 @@
+import {
+  HookEventType,
+  PluginContext,
+  PluginHandler,
+  PluginParameters,
+} from '../types';
+import { getText, post } from '../utils';
+
+interface MistralResponse {
+  id: string;
+  model: string;
+  results: [
+    {
+      categories: {
+        sexual: boolean;
+        hate_and_discrimination: boolean;
+        violence_and_threats: boolean;
+        dangerous_and_criminal_content: boolean;
+        selfharm: boolean;
+        health: boolean;
+        financial: boolean;
+        law: boolean;
+        pii: boolean;
+      };
+      category_score: {
+        sexual: number;
+        hate_and_discrimination: number;
+        violence_and_threats: number;
+        dangerous_and_criminal_content: number;
+        selfharm: number;
+        health: number;
+        financial: number;
+        law: number;
+        pii: number;
+      };
+    },
+  ];
+}
+
+type GuardrailFunction = keyof MistralResponse['results'][0]['categories'];
+
+export const mistralGuardrailHandler: PluginHandler = async (
+  context: PluginContext,
+  parameters: PluginParameters,
+  eventType: HookEventType,
+  _options,
+  fn: string
+) => {
+  let error = null;
+  let verdict = true;
+  let data = null;
+
+  const creds = parameters.credentials as Record<string, string>;
+  if (!creds.apiKey) {
+    return {
+      error: 'Mistral API key not provided.',
+      verdict: false,
+      data: null,
+    };
+  }
+
+  let model = 'mistral-moderation-latest';
+
+  if (parameters.model) {
+    // Model can be passed dynamically
+    model = parameters.model;
+  }
+
+  const guardrailFunction = fn as GuardrailFunction;
+
+  const text = getText(context, eventType);
+  const messages =
+    eventType === 'beforeRequestHook'
+      ? context.request?.json?.messages
+      : context.response?.json?.messages;
+
+  // should contain text or should contain messages array
+  if (
+    (!text && !Array.isArray(messages)) ||
+    (Array.isArray(messages) && messages.length === 0)
+  ) {
+    return {
+      error: 'Mistral: Invalid Request body',
+      verdict: false,
+      data: null,
+    };
+  }
+
+  // Use conversation guardrail if it's a chatcomplete and before hook
+  const shouldUseConversation =
+    eventType === 'beforeRequestHook' && context.requestType === 'chatComplete';
+  const url = shouldUseConversation
+    ? 'https://api.mistral.ai/v1/chat/moderations'
+    : 'https://api.mistral.ai/v1/moderations';
+
+  try {
+    const request = await post<MistralResponse>(
+      url,
+      {
+        model: model,
+        ...(!shouldUseConversation && { input: [text] }),
+        ...(shouldUseConversation && { input: [messages] }),
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${creds.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+      }
+    );
+
+    verdict = request.results?.[0]?.categories[guardrailFunction];
+  } catch (error) {
+    error = error;
+    verdict = false;
+  }
+
+  return { error, verdict, data };
+};

plugins/mistral/manifest.json

@@ -0,0 +1,135 @@
+{
+  "id": "mistral",
+  "description": "Mistral Content Moderation classifier leverages the most relevant policy categories for effective guardrails and introduces a pragmatic approach to LLM safety by addressing model-generated harms such as unqualified advice and PII",
+  "credentials": {
+    "type": "object",
+    "properties": {
+      "apiKey": {
+        "type": "string",
+        "label": "API Key",
+        "description": "Find your API key in the Mistral la-plateforme",
+        "encrypted": true
+      }
+    },
+    "required": ["apiKey"]
+  },
+  "functions": [
+    {
+      "name": "Detect PII",
+      "id": "pii",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that requests, shares, or attempts to elicit personal identifying information such as full names, addresses, phone numbers, social security numbers, or financial account details."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Sexual Content",
+      "id": "sexual",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Material that explicitly depicts, describes, or promotes sexual activities, nudity, or sexual services. This includes pornographic content, graphic descriptions of sexual acts, and solicitation for sexual purposes. Educational or medical content about sexual health presented in a non-explicit, informational context is generally exempted."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Hate & Discrimination",
+      "id": "hate_and_discrimination",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that expresses prejudice, hostility, or advocates discrimination against individuals or groups based on protected characteristics such as race, ethnicity, religion, gender, sexual orientation, or disability. This includes slurs, dehumanizing language, calls for exclusion or harm targeted at specific groups, and persistent harassment or bullying of individuals based on these characteristics."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Violent & Thereat",
+      "id": "violence_and_threats",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that describes, glorifies, incites, or threatens physical violence against individuals or groups. This includes graphic depictions of injury or death, explicit threats of harm, and instructions for carrying out violent acts. This category covers both targeted threats and general promotion or glorification of violence."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Dangerous & Criminal Content",
+      "id": "dangerous_and_criminal_content",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that promotes or provides instructions for illegal activities or extremely hazardous behaviors that pose a significant risk of physical harm, death, or legal consequences. This includes guidance on creating weapons or explosives, encouragement of extreme risk-taking behaviors, and promotion of non-violent crimes such as fraud, theft, or drug trafficking."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Selfharm",
+      "id": "selfharm",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that promotes, instructs, plans, or encourages deliberate self-injury, suicide, eating disorders, or other self-destructive behaviors. This includes detailed methods, glorification, statements of intent, dangerous challenges, and related slang terms"
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Health",
+      "id": "health",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that contains or tries to elicit detailed or tailored medical advice."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Finance",
+      "id": "financial",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that contains or tries to elicit detailed or tailored financial advice."
+        }
+      ],
+      "parameters": {}
+    },
+    {
+      "name": "Detect Law",
+      "id": "law",
+      "supportedHooks": ["beforeRequestHook", "afterRequestHook"],
+      "type": "guardrail",
+      "description": [
+        {
+          "type": "subHeading",
+          "text": "Content that contains or tries to elicit detailed or tailored legal advice."
+        }
+      ],
+      "parameters": {}
+    }
+  ]
+}