EN-7889 slack alert on offensive content

app/models/chat_message.rb

@@ -3,6 +3,7 @@ class ChatMessage < ApplicationRecord
   include ChatServices::Spam
   include ChatServices::PrivateConversation
   include Deeplinkable
+  include Offensable
   include Translatable
   include Reactionnable
   include Surveyable

app/models/concerns/offensable.rb

@@ -0,0 +1,33 @@
+module Offensable
+  extend ActiveSupport::Concern
+
+  alias_attribute :name, :content
+
+  FIELDS = {
+    chat_messages: "content"
+  }
+
+  included do
+    has_one :openai_request, as: :instance
+
+    after_commit :check_offense, if: :check_offense?
+  end
+
+  private
+
+  def build_openai_request(attributes = {})
+    super(attributes.merge(instance_class: self.class.name))
+  end
+
+  def check_offense
+    build_openai_request(module_type: :offense).save!
+  end
+
+  def check_offense?
+    key = self.class.table_name.to_sym
+
+    return false unless FIELDS.has_key?(key)
+
+    previous_changes.keys.include?(FIELDS[key])
+  end
+end

app/models/openai_request.rb

@@ -8,7 +8,8 @@ class OpenaiRequest < ApplicationRecord
   # 4. create a response class that inherits from BasicResponse. Check MatchingResponse for evample
   # 5. add this class to performer_response method
   enum module_type: {
-    matching: 'matching'
+    matching: 'matching',
+    offense: 'offense'
   }
 
   after_commit :run, on: :create
@@ -46,13 +47,19 @@ def response_instance
     @response_instance ||= begin
       if matching?
         OpenaiServices::MatchingResponse.new(response: safe_json_parse(response))
+      elsif offense?
+        OpenaiServices::OffenseResponse.new(response: safe_json_parse(response))
       end
     end
   end
 
   # add module_type case if needed
   def performer_instance
-    return OpenaiServices::MatchingPerformer.new(openai_request: self) if matching?
+    if matching?
+      OpenaiServices::MatchingPerformer.new(openai_request: self)
+    elsif offense?
+      OpenaiServices::OffensePerformer.new(openai_request: self)
+    end
   end
 
   attr_accessor :forced_matching
@@ -62,6 +69,8 @@ def run
   end
 
   def safe_json_parse json_string
+    return Hash.new unless json_string.present?
+
     JSON.parse(json_string)
   rescue JSON::ParserError
     {}

app/services/openai_services/basic_response.rb

@@ -9,6 +9,10 @@ def valid?
       raise NotImplementedError, "this method valid? has to be defined in your class"
     end
 
+    def display_result
+      raise NotImplementedError, "this method display_result has to be defined in your class"
+    end
+
     def parsed_response
       return unless @response
       return unless content = @response["content"]

app/services/openai_services/matching_performer.rb

@@ -2,9 +2,6 @@ module OpenaiServices
   class MatchingPerformer < BasicPerformer
     attr_reader :user
 
-    class MatcherCallback < Callback
-    end
-
     def user_message
       {
         role: "user",

app/services/openai_services/matching_response.rb

@@ -17,6 +17,10 @@ def valid?
       recommandations.any?
     end
 
+    def display_result
+      recommandations.count
+    end
+
     def recommandations
       return [] unless @parsed_response
 

app/services/openai_services/offense_performer.rb

@@ -0,0 +1,33 @@
+module OpenaiServices
+  class OffensePerformer < BasicPerformer
+    def user_message
+      {
+        role: "user",
+        content: [
+          { type: "text", text: get_formatted_prompt },
+        ]
+      }
+    end
+
+    def get_response_class
+      OffenseResponse
+    end
+
+    private
+
+    def handle_success response
+      super(response)
+
+      puts "-- response.offensive?: #{response.offensive?}"
+
+      if response.offensive?
+        puts "-- offensive!"
+        SlackServices::OffensiveText.new(instance: instance, text: instance.content).notify
+      end
+    end
+
+    def get_formatted_prompt
+      @configuration.prompt.gsub("{{text}}", instance.content)
+    end
+  end
+end

app/services/openai_services/offense_response.rb

@@ -0,0 +1,19 @@
+module OpenaiServices
+  class OffenseResponse < BasicResponse
+    def valid?
+      result.in? [true, false]
+    end
+
+    def offensive?
+      result == true
+    end
+
+    def display_result
+      result
+    end
+
+    def result
+      @parsed_response["result"]
+    end
+  end
+end

app/services/slack_services/offensive_text.rb

@@ -0,0 +1,30 @@
+module SlackServices
+  class OffensiveText < Notifier
+    def initialize instance:, text:
+      @instance = instance
+      @text = text
+    end
+
+    def env
+      ENV['SLACK_SIGNAL']
+    end
+
+    def payload
+      {
+        text: "<@#{slack_moderator_id(@instance.user)}> ou team modération (département : #{departement(@instance.user) || 'n/a'}) pouvez-vous vérifier ce texte ?",
+        attachments: [
+          {
+            text: "Texte offensant : #{@text}"
+          }
+        ]
+      }
+    end
+
+    def payload_adds
+      {
+        username: "Texte offensant",
+        channel: webhook('channel'),
+      }
+    end
+  end
+end

app/views/admin/openai_requests/index.html.erb

@@ -16,13 +16,15 @@
           <thead>
             <th>Nom</th>
             <th>Créé le</th>
+            <th>Résultat</th>
           </thead>
           <% @openai_requests.each_with_index do |openai_request, i| %>
             <% parite = i.even? ? 'pair' : 'impair' %>
 
             <tr class="above-row row-<%= parite %>">
               <td><%= link_to openai_request.instance.name, admin_openai_request_path(openai_request) %></td>
               <td><%= l openai_request.created_at %></td>
+              <td><%= openai_request.response_instance.display_result %></td>
             </tr>
           <% end %>
         </table>

db/migrate/20241217142700_create_openai_offense_assistant_configuration_instance.rb

@@ -0,0 +1,15 @@
+class CreateOpenaiOffenseAssistantConfigurationInstance < ActiveRecord::Migration[6.1]
+  def change
+    unless Rails.env.test?
+      OpenaiAssistant.new(
+        module_type: :offense,
+        version: 2,
+        api_key: ENV['OPENAI_API_KEY'],
+        assistant_id: ENV['OPENAI_API_OFFENSE_ASSISTANT_ID'],
+        prompt: "{{text}}",
+        poi_from_file: false,
+        resource_from_file: false
+      ).save
+    end
+  end
+end