From 8f3eb7666822dd11f4624bf4c1fab15c1f9be3d9 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Tue, 2 May 2023 17:36:49 +0100
Subject: [PATCH 1/6] feat/neon_transformers

---
 ovos_core/intent_services/__init__.py         | 124 ++++++++-------
 ovos_core/intent_services/adapt_service.py    |  34 ++--
 ovos_core/intent_services/commonqa_service.py |  23 +--
 ovos_core/intent_services/converse_service.py |   7 +-
 ovos_core/intent_services/fallback_service.py |  16 +-
 .../intent_services/padatious_service.py      |  20 +--
 ovos_core/transformers.py                     | 149 ++++++++++++++++++
 setup.py                                      |   4 +-
 8 files changed, 276 insertions(+), 101 deletions(-)
 create mode 100644 ovos_core/transformers.py

diff --git a/ovos_core/intent_services/__init__.py b/ovos_core/intent_services/__init__.py
index e76850d2410d..048c1c103de7 100644
--- a/ovos_core/intent_services/__init__.py
+++ b/ovos_core/intent_services/__init__.py
@@ -12,14 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-
 from collections import namedtuple
 
 from ovos_config.config import Configuration
 from ovos_config.locale import setup_locale
 
-from ovos_bus_client.message import Message
+from ovos_core.transformers import MetadataTransformersService, UtteranceTransformersService
 from ovos_core.intent_services.adapt_service import AdaptService
 from ovos_core.intent_services.commonqa_service import CommonQAService
 from ovos_core.intent_services.converse_service import ConverseService
@@ -42,41 +40,6 @@
                          )
 
 
-def _normalize_all_utterances(utterances):
-    """Create normalized versions and pair them with the original utterance.
-
-    This will create a list of tuples with the original utterance as the
-    first item and if normalizing changes the utterance the normalized version
-    will be set as the second item in the tuple, if normalization doesn't
-    change anything the tuple will only have the "raw" original utterance.
-
-    Args:
-        utterances (list): list of utterances to normalize
-
-    Returns:
-        list of tuples, [(original utterance, normalized) ... ]
-    """
-    try:
-        from lingua_franca.parse import normalize
-        # normalize() changes "it's a boy" to "it is a boy", etc.
-        norm_utterances = [normalize(u.lower(), remove_articles=False)
-                           for u in utterances]
-    except:
-        norm_utterances = utterances
-
-    # Create pairs of original and normalized counterparts for each entry
-    # in the input list.
-    combined = []
-    for utt, norm in zip(utterances, norm_utterances):
-        if utt == norm:
-            combined.append((utt,))
-        else:
-            combined.append((utt, norm))
-
-    LOG.debug(f"Utterances: {combined}")
-    return combined
-
-
 class IntentService:
     """Mycroft intent service. parses utterances using a variety of systems.
 
@@ -100,6 +63,8 @@ def __init__(self, bus):
         self.fallback = FallbackService(bus)
         self.converse = ConverseService(bus)
         self.common_qa = CommonQAService(bus)
+        self.utterance_plugins = UtteranceTransformersService(bus, config=config)
+        self.metadata_plugins = MetadataTransformersService(bus, config=config)
 
         self.bus.on('register_vocab', self.handle_register_vocab)
         self.bus.on('register_intent', self.handle_register_intent)
@@ -193,39 +158,87 @@ def reset_converse(self, message):
             LOG.exception(f"Failed to set lingua_franca default lang to {lang}")
         self.converse.converse_with_skills([], lang, message)
 
+    def _handle_transformers(self, message):
+        """
+        Pipe utterance through transformer plugins to get more metadata.
+        Utterances may be modified by any parser and context overwritten
+        """
+        lang = get_message_lang(message)  # per query lang or default Configuration lang
+        original = utterances = message.data.get('utterances', [])
+        message.context["lang"] = lang
+        utterances, message.context = self.utterance_plugins.transform(utterances, message.context)
+        if original != utterances:
+            message.data["utterances"] = utterances
+            LOG.debug(f"utterances transformed: {original} -> {utterances}")
+        message.context = self.metadata_plugins.transform(message.context)
+        return message
+
+    @staticmethod
+    def disambiguate_lang(message):
+        """ disambiguate language of the query via pre-defined context keys
+        1 - stt_lang -> tagged in stt stage  (STT used this lang to transcribe speech)
+        2 - request_lang -> tagged in source message (wake word/request volunteered lang info)
+        3 - detected_lang -> tagged by transformers  (text classification, free form chat)
+        4 - config lang (or from message.data)
+        """
+        cfg = Configuration()
+        default_lang = get_message_lang(message)
+        valid_langs = set([cfg.get("lang", "en-us")] + cfg.get("secondary_langs'", []))
+        lang_keys = ["stt_lang",
+                     "request_lang",
+                     "detected_lang"]
+        for k in lang_keys:
+            if k in message.context:
+                v = message.context[k]
+                if v in valid_langs:
+                    if v != default_lang:
+                        LOG.info(f"replaced {default_lang} with {k}: {v}")
+                    return v
+                else:
+                    LOG.warning(f"ignoring {k}, {v} is not in enabled languages: {valid_langs}")
+
+        return default_lang
+
     def handle_utterance(self, message):
-        """Main entrypoint for handling user utterances with Mycroft skills
+        """Main entrypoint for handling user utterances
 
         Monitor the messagebus for 'recognizer_loop:utterance', typically
         generated by a spoken interaction but potentially also from a CLI
         or other method of injecting a 'user utterance' into the system.
 
         Utterances then work through this sequence to be handled:
-        1) Active skills attempt to handle using converse()
-        2) Padatious high match intents (conf > 0.95)
-        3) Adapt intent handlers
-        5) CommonQuery Skills
-        6) High Priority Fallbacks
-        7) Padatious near match intents (conf > 0.8)
-        8) General Fallbacks
-        9) Padatious loose match intents (conf > 0.5)
-        10) Catch all fallbacks including Unknown intent handler
+        1) UtteranceTransformers can modify the utterance and metadata in message.context
+        2) MetadataTransformers can modify the metadata in message.context
+        3) Language is extracted from message
+        4) Active skills attempt to handle using converse()
+        5) Padatious high match intents (conf > 0.95)
+        6) Adapt intent handlers
+        7) CommonQuery Skills
+        8) High Priority Fallbacks
+        9) Padatious near match intents (conf > 0.8)
+        10) General Fallbacks
+        11) Padatious loose match intents (conf > 0.5)
+        12) Catch all fallbacks including Unknown intent handler
 
         If all these fail the complete_intent_failure message will be sent
-        and a generic info of the failure will be spoken.
+        and a generic error sound played.
 
         Args:
             message (Message): The messagebus data
         """
         try:
-            lang = get_message_lang(message)
+
+            # Get utterance utterance_plugins additional context
+            message = self._handle_transformers(message)
+
+            # tag language of this utterance
+            lang = self.disambiguate_lang(message)
             try:
                 setup_locale(lang)
             except Exception as e:
                 LOG.exception(f"Failed to set lingua_franca default lang to {lang}")
 
             utterances = message.data.get('utterances', [])
-            combined = _normalize_all_utterances(utterances)
 
             stopwatch = Stopwatch()
 
@@ -242,11 +255,12 @@ def handle_utterance(self, message):
                 self.fallback.low_prio
             ]
 
+            # match
             match = None
             with stopwatch:
                 # Loop through the matching functions until a match is found.
                 for match_func in match_funcs:
-                    match = match_func(combined, lang, message)
+                    match = match_func(utterances, lang, message)
                     if match:
                         break
             if match:
@@ -374,7 +388,6 @@ def handle_get_intent(self, message):
         """
         utterance = message.data["utterance"]
         lang = get_message_lang(message)
-        combined = _normalize_all_utterances([utterance])
 
         # Create matchers
         padatious_matcher = PadatiousMatcher(self.padatious_service)
@@ -394,7 +407,7 @@ def handle_get_intent(self, message):
         ]
         # Loop through the matching functions until a match is found.
         for match_func in match_funcs:
-            match = match_func(combined, lang, message)
+            match = match_func([utterance], lang, message)
             if match:
                 if match.intent_type:
                     intent_data = match.intent_data
@@ -436,8 +449,7 @@ def handle_get_adapt(self, message):
         """
         utterance = message.data["utterance"]
         lang = get_message_lang(message)
-        combined = _normalize_all_utterances([utterance])
-        intent = self.adapt_service.match_intent(combined, lang)
+        intent = self.adapt_service.match_intent([utterance], lang)
         intent_data = intent.intent_data if intent else None
         self.bus.emit(message.reply("intent.service.adapt.reply",
                                     {"intent": intent_data}))
diff --git a/ovos_core/intent_services/adapt_service.py b/ovos_core/intent_services/adapt_service.py
index 8a2d1f081685..e0c62fc2d45f 100644
--- a/ovos_core/intent_services/adapt_service.py
+++ b/ovos_core/intent_services/adapt_service.py
@@ -13,14 +13,15 @@
 # limitations under the License.
 #
 """An intent parsing service using the Adapt parser."""
-import time
 from threading import Lock
 
+import time
 from adapt.context import ContextManagerFrame
 from adapt.engine import IntentDeterminationEngine
 from ovos_config.config import Configuration
 
 import ovos_core.intent_services
+from ovos_utils import flatten_list
 from ovos_utils.log import LOG
 
 
@@ -211,6 +212,8 @@ def match_intent(self, utterances, lang=None, __=None):
         Returns:
             Intent structure, or None if no match was found.
         """
+        # we call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
         lang = lang or self.lang
         if lang not in self.engines:
             return None
@@ -226,21 +229,20 @@ def take_best(intent, utt):
                 # TODO - Shouldn't Adapt do this?
                 best_intent['utterance'] = utt
 
-        for utt_tup in utterances:
-            for utt in utt_tup:
-                try:
-                    intents = [i for i in self.engines[lang].determine_intent(
-                        utt, 100,
-                        include_tags=True,
-                        context_manager=self.context_manager)]
-                    if intents:
-                        utt_best = max(
-                            intents, key=lambda x: x.get('confidence', 0.0)
-                        )
-                        take_best(utt_best, utt_tup[0])
-
-                except Exception as err:
-                    LOG.exception(err)
+        for utt in utterances:
+            try:
+                intents = [i for i in self.engines[lang].determine_intent(
+                    utt, 100,
+                    include_tags=True,
+                    context_manager=self.context_manager)]
+                if intents:
+                    utt_best = max(
+                        intents, key=lambda x: x.get('confidence', 0.0)
+                    )
+                    take_best(utt_best, utt)
+
+            except Exception as err:
+                LOG.exception(err)
 
         if best_intent:
             self.update_context(best_intent)
diff --git a/ovos_core/intent_services/commonqa_service.py b/ovos_core/intent_services/commonqa_service.py
index d9c2b02718c1..9a826ffcc6dd 100644
--- a/ovos_core/intent_services/commonqa_service.py
+++ b/ovos_core/intent_services/commonqa_service.py
@@ -1,10 +1,12 @@
 import re
+from threading import Lock, Event
+
 import time
 from itertools import chain
-from threading import Lock, Event
+from ovos_bus_client.message import Message, dig_for_message
 
 import ovos_core.intent_services
-from ovos_bus_client.message import Message, dig_for_message
+from ovos_utils import flatten_list
 from ovos_utils.enclosure.api import EnclosureAPI
 from ovos_utils.log import LOG
 from ovos_utils.messagebus import get_message_lang
@@ -93,14 +95,17 @@ def match(self, utterances, lang, message):
         Returns:
             IntentMatch or None
         """
+        # we call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
         match = None
-        utterance = utterances[0][0]
-        if self.is_question_like(utterance, lang):
-            message.data["lang"] = lang  # only used for speak
-            message.data["utterance"] = utterance
-            answered = self.handle_question(message)
-            if answered:
-                match = ovos_core.intent_services.IntentMatch('CommonQuery', None, {}, None)
+        for utterance in utterances:
+            if self.is_question_like(utterance, lang):
+                message.data["lang"] = lang  # only used for speak
+                message.data["utterance"] = utterance
+                answered = self.handle_question(message)
+                if answered:
+                    match = ovos_core.intent_services.IntentMatch('CommonQuery', None, {}, None)
+                break
         return match
 
     def handle_question(self, message):
diff --git a/ovos_core/intent_services/converse_service.py b/ovos_core/intent_services/converse_service.py
index d11be04fed8b..ba0d4918f048 100644
--- a/ovos_core/intent_services/converse_service.py
+++ b/ovos_core/intent_services/converse_service.py
@@ -1,9 +1,9 @@
 import time
-
+from ovos_bus_client.message import Message
 from ovos_config.config import Configuration
 
 import ovos_core.intent_services
-from ovos_bus_client.message import Message
+from ovos_utils import flatten_list
 from ovos_utils.log import LOG
 from ovos_workshop.permissions import ConverseMode, ConverseActivationMode
 
@@ -252,7 +252,8 @@ def converse_with_skills(self, utterances, lang, message):
         Returns:
             IntentMatch if handled otherwise None.
         """
-        utterances = [item for tup in utterances or [] for item in tup]
+        # we call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
         # filter allowed skills
         self._check_converse_timeout()
         # check if any skill wants to handle utterance
diff --git a/ovos_core/intent_services/fallback_service.py b/ovos_core/intent_services/fallback_service.py
index c0d9afc6e919..29c65025b6f9 100644
--- a/ovos_core/intent_services/fallback_service.py
+++ b/ovos_core/intent_services/fallback_service.py
@@ -13,12 +13,15 @@
 # limitations under the License.
 #
 """Intent service for Mycroft's fallback system."""
-from collections import namedtuple
-from ovos_config import Configuration
 import operator
+from collections import namedtuple
+
 import time
-from ovos_utils.log import LOG
+from ovos_config import Configuration
+
 import ovos_core.intent_services
+from ovos_utils import flatten_list
+from ovos_utils.log import LOG
 from ovos_workshop.skills.fallback import FallbackMode
 
 FallbackRange = namedtuple('FallbackRange', ['start', 'stop'])
@@ -81,7 +84,7 @@ def _collect_fallback_skills(self, message, fb_range=FallbackRange(0, 100)):
 
         # filter skills outside the fallback_range
         in_range = [s for s, p in self.registered_fallbacks.items()
-                     if fb_range.start < p <= fb_range.stop]
+                    if fb_range.start < p <= fb_range.stop]
         skill_ids += [s for s in self.registered_fallbacks if s not in in_range]
 
         def handle_ack(msg):
@@ -150,9 +153,8 @@ def _fallback_range(self, utterances, lang, message, fb_range):
         Returns:
             IntentMatch or None
         """
-        # NOTE - utterances here is a list of tuple of [(ut, norm)]
-        # they have been munged along the way... TODO undo the munging at the source
-        utterances = [u[0] for u in utterances]
+        # we call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
         message.data["utterances"] = utterances  # all transcripts
         message.data["lang"] = lang
 
diff --git a/ovos_core/intent_services/padatious_service.py b/ovos_core/intent_services/padatious_service.py
index fd7e1246a3e6..f5b6270d8b2f 100644
--- a/ovos_core/intent_services/padatious_service.py
+++ b/ovos_core/intent_services/padatious_service.py
@@ -17,13 +17,14 @@
 from os.path import expanduser, isfile
 from subprocess import call
 from threading import Event
-from time import time as get_time, sleep
 
+from ovos_bus_client.message import Message
 from ovos_config.config import Configuration
 from padacioso import IntentContainer as FallbackIntentContainer
+from time import time as get_time, sleep
 
 import ovos_core.intent_services
-from ovos_bus_client.message import Message
+from ovos_utils import flatten_list
 from ovos_utils.log import LOG
 
 try:
@@ -81,18 +82,19 @@ def _match_level(self, utterances, limit, lang=None):
                                          with optional normalized version.
             limit (float): required confidence level.
         """
+        # we call flatten in case someone is sending the old style list of tuples
+        utterances = flatten_list(utterances)
         if not self.has_result:
             lang = lang or self.service.lang
             padatious_intent = None
             LOG.debug(f'Padatious Matching confidence > {limit}')
             for utt in utterances:
-                for variant in utt:
-                    intent = self.service.calc_intent(variant, lang)
-                    if intent:
-                        best = padatious_intent.conf if padatious_intent else 0.0
-                        if best < intent.conf:
-                            padatious_intent = intent
-                            padatious_intent.matches['utterance'] = utt[0]
+                intent = self.service.calc_intent(utt, lang)
+                if intent:
+                    best = padatious_intent.conf if padatious_intent else 0.0
+                    if best < intent.conf:
+                        padatious_intent = intent
+                        padatious_intent.matches['utterance'] = utt[0]
             if padatious_intent:
                 skill_id = padatious_intent.name.split(':')[0]
                 self.ret = ovos_core.intent_services.IntentMatch(
diff --git a/ovos_core/transformers.py b/ovos_core/transformers.py
new file mode 100644
index 000000000000..81e87e9b47c1
--- /dev/null
+++ b/ovos_core/transformers.py
@@ -0,0 +1,149 @@
+from typing import Optional, List
+
+from ovos_plugin_manager.metadata_transformers import find_metadata_transformer_plugins
+from ovos_plugin_manager.templates.transformers import UtteranceTransformer
+from ovos_plugin_manager.text_transformers import find_utterance_transformer_plugins
+
+from ovos_utils.json_helper import merge_dict
+from ovos_utils.log import LOG
+
+
+class UtteranceTransformersService:
+
+    def __init__(self, bus, config=None):
+        self.config_core = config or {}
+        self.loaded_plugins = {}
+        self.has_loaded = False
+        self.bus = bus
+        self.config = self.config_core.get("utterance_transformers") or {
+            "ovos-utterance-normalizer": {}
+        }
+        self.load_plugins()
+
+    def load_plugins(self):
+        for plug_name, plug in find_utterance_transformer_plugins().items():
+            if plug_name in self.config:
+                # if disabled skip it
+                if not self.config[plug_name].get("active", True):
+                    continue
+                try:
+                    self.loaded_plugins[plug_name] = plug()
+                    LOG.info(f"loaded utterance transformer plugin: {plug_name}")
+                except Exception as e:
+                    LOG.error(e)
+                    LOG.exception(f"Failed to load utterance transformer plugin: {plug_name}")
+
+    @property
+    def plugins(self):
+        """
+        Return loaded transformers in priority order, such that modules with a
+        higher `priority` rank are called first and changes from lower ranked
+        transformers are applied last
+
+        A plugin of `priority` 1 will override any existing context keys and
+        will be the last to modify utterances`
+        """
+        return sorted(self.loaded_plugins.values(),
+                      key=lambda k: k.priority, reverse=True)
+
+    def shutdown(self):
+        for module in self.plugins:
+            try:
+                module.shutdown()
+            except:
+                pass
+
+    def transform(self, utterances: List[str], context: Optional[dict] = None):
+        context = context or {}
+
+        for module in self.plugins:
+            try:
+                utterances, data = module.transform(utterances, context)
+                LOG.debug(f"{module.name}: {data}")
+                context = merge_dict(context, data)
+            except Exception as e:
+                LOG.warning(f"{module.name} transform exception: {e}")
+        return utterances, context
+
+
+class MetadataTransformersService:
+
+    def __init__(self, bus, config=None):
+        self.config_core = config or {}
+        self.loaded_plugins = {}
+        self.has_loaded = False
+        self.bus = bus
+        self.config = self.config_core.get("metadata_transformers") or {}
+        self.load_plugins()
+
+    def load_plugins(self):
+        for plug_name, plug in find_metadata_transformer_plugins().items():
+            if plug_name in self.config:
+                # if disabled skip it
+                if not self.config[plug_name].get("active", True):
+                    continue
+                try:
+                    self.loaded_plugins[plug_name] = plug()
+                    LOG.info(f"loaded metadata transformer plugin: {plug_name}")
+                except Exception as e:
+                    LOG.error(e)
+                    LOG.exception(f"Failed to load metadata transformer plugin: {plug_name}")
+
+    @property
+    def plugins(self):
+        """
+        Return loaded transformers in priority order, such that modules with a
+        higher `priority` rank are called first and changes from lower ranked
+        transformers are applied last.
+
+        A plugin of `priority` 1 will override any existing context keys
+        """
+        return sorted(self.loaded_plugins.values(),
+                      key=lambda k: k.priority, reverse=True)
+
+    def shutdown(self):
+        for module in self.plugins:
+            try:
+                module.shutdown()
+            except:
+                pass
+
+    def transform(self, context: Optional[dict] = None):
+        context = context or {}
+
+        for module in self.plugins:
+            try:
+                data = module.transform(context)
+                LOG.debug(f"{module.name}: {data}")
+                context = merge_dict(context, data)
+            except Exception as e:
+                LOG.warning(f"{module.name} transform exception: {e}")
+        return context
+
+
+class UtteranceNormalizer(UtteranceTransformer):
+
+    def __init__(self, name="ovos-utterance-normalizer", priority=1):
+        super().__init__(name, priority)
+
+    @staticmethod
+    def strip_punctuation(utterance: str):
+        return utterance.rstrip('.').rstrip('?').rstrip('!').rstrip(',').rstrip(';')
+
+    def transform(self, utterances: List[str],
+                  context: Optional[dict] = None) -> (list, dict):
+        context = context or {}
+        norm = [self.strip_punctuation(u) for u in utterances] + utterances
+        try:
+            # TODO - move to ovos-classifiers when it gets it's first stable release
+            # make ovos-lf provide this plugin with a lower priority afterwards
+            from lingua_franca.parse import normalize
+            lang = context.get("lang") or self.config.get("lang", "en-us")
+            norm += [normalize(u, lang=lang, remove_articles=False) for u in norm] + \
+                    [normalize(u, lang=lang, remove_articles=True) for u in norm]
+        except ImportError:
+            LOG.warning("lingua_franca not installed")
+        except:
+            LOG.exception("lingua_franca utterance normalization failed")
+
+        return list(set(norm)), context
diff --git a/setup.py b/setup.py
index 54eeec6baef4..b38f79b28bd0 100644
--- a/setup.py
+++ b/setup.py
@@ -87,6 +87,8 @@ def required(requirements_file):
             'mycroft-enclosure-client=ovos_PHAL.__main__:main',
             'mycroft-cli-client=mycroft.client.text.__main__:main',
             'mycroft-gui-service=mycroft.gui.__main__:main'
-        ]
+        ],
+        # default plugins
+        'neon.plugin.text': 'ovos-utterance-normalizer=ovos_core.transformers:UtteranceNormalizer'
     }
 )

From 5b731031a96f4e4f0eb77b236caf80e0df36ee77 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Tue, 2 May 2023 19:50:44 +0100
Subject: [PATCH 2/6] use lf in namespace for plugin to allow future
 replacement

---
 ovos_core/transformers.py | 4 ++--
 setup.py                  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ovos_core/transformers.py b/ovos_core/transformers.py
index 81e87e9b47c1..e270f3dd734b 100644
--- a/ovos_core/transformers.py
+++ b/ovos_core/transformers.py
@@ -16,7 +16,7 @@ def __init__(self, bus, config=None):
         self.has_loaded = False
         self.bus = bus
         self.config = self.config_core.get("utterance_transformers") or {
-            "ovos-utterance-normalizer": {}
+            "ovos-lf-utterance-normalizer": {}
         }
         self.load_plugins()
 
@@ -123,7 +123,7 @@ def transform(self, context: Optional[dict] = None):
 
 class UtteranceNormalizer(UtteranceTransformer):
 
-    def __init__(self, name="ovos-utterance-normalizer", priority=1):
+    def __init__(self, name="ovos-lf-utterance-normalizer", priority=1):
         super().__init__(name, priority)
 
     @staticmethod
diff --git a/setup.py b/setup.py
index b38f79b28bd0..54d5e9515eb7 100644
--- a/setup.py
+++ b/setup.py
@@ -89,6 +89,6 @@ def required(requirements_file):
             'mycroft-gui-service=mycroft.gui.__main__:main'
         ],
         # default plugins
-        'neon.plugin.text': 'ovos-utterance-normalizer=ovos_core.transformers:UtteranceNormalizer'
+        'neon.plugin.text': 'ovos-lf-utterance-normalizer=ovos_core.transformers:UtteranceNormalizer'
     }
 )

From 922dd35b2e4d2678d50b717909d63ee5fe4aa8de Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Tue, 2 May 2023 20:38:00 +0100
Subject: [PATCH 3/6] move default utt normalizer plugin to ovos-classifiers
 instead of bundling a hardcoded plugin

---
 ovos_core/intent_services/adapt_service.py |  1 -
 ovos_core/transformers.py                  | 28 +---------------------
 requirements/requirements.txt              |  5 +++-
 setup.py                                   |  4 +---
 4 files changed, 6 insertions(+), 32 deletions(-)

diff --git a/ovos_core/intent_services/adapt_service.py b/ovos_core/intent_services/adapt_service.py
index e0c62fc2d45f..b59d795bf50b 100644
--- a/ovos_core/intent_services/adapt_service.py
+++ b/ovos_core/intent_services/adapt_service.py
@@ -19,7 +19,6 @@
 from adapt.context import ContextManagerFrame
 from adapt.engine import IntentDeterminationEngine
 from ovos_config.config import Configuration
-
 import ovos_core.intent_services
 from ovos_utils import flatten_list
 from ovos_utils.log import LOG
diff --git a/ovos_core/transformers.py b/ovos_core/transformers.py
index e270f3dd734b..5014fd1aee3b 100644
--- a/ovos_core/transformers.py
+++ b/ovos_core/transformers.py
@@ -16,7 +16,7 @@ def __init__(self, bus, config=None):
         self.has_loaded = False
         self.bus = bus
         self.config = self.config_core.get("utterance_transformers") or {
-            "ovos-lf-utterance-normalizer": {}
+            "ovos-utterance-normalizer": {}
         }
         self.load_plugins()
 
@@ -121,29 +121,3 @@ def transform(self, context: Optional[dict] = None):
         return context
 
 
-class UtteranceNormalizer(UtteranceTransformer):
-
-    def __init__(self, name="ovos-lf-utterance-normalizer", priority=1):
-        super().__init__(name, priority)
-
-    @staticmethod
-    def strip_punctuation(utterance: str):
-        return utterance.rstrip('.').rstrip('?').rstrip('!').rstrip(',').rstrip(';')
-
-    def transform(self, utterances: List[str],
-                  context: Optional[dict] = None) -> (list, dict):
-        context = context or {}
-        norm = [self.strip_punctuation(u) for u in utterances] + utterances
-        try:
-            # TODO - move to ovos-classifiers when it gets it's first stable release
-            # make ovos-lf provide this plugin with a lower priority afterwards
-            from lingua_franca.parse import normalize
-            lang = context.get("lang") or self.config.get("lang", "en-us")
-            norm += [normalize(u, lang=lang, remove_articles=False) for u in norm] + \
-                    [normalize(u, lang=lang, remove_articles=True) for u in norm]
-        except ImportError:
-            LOG.warning("lingua_franca not installed")
-        except:
-            LOG.exception("lingua_franca utterance normalization failed")
-
-        return list(set(norm)), context
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index a38d0e6b5986..ee9dd0786886 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -12,4 +12,7 @@ ovos-plugin-manager<0.1.0, >=0.0.23a9
 ovos-config~=0.0,>=0.0.8
 ovos-lingua-franca>=0.4.7
 ovos_backend_client<0.1.0, >=0.0.6
-ovos_workshop<0.1.0, >=0.0.12a17
\ No newline at end of file
+ovos_workshop<0.1.0, >=0.0.12a17
+
+# provides plugins and classic machine learning framework
+ovos-classifiers<0.1.0, >=0.0.0a3
diff --git a/setup.py b/setup.py
index 54d5e9515eb7..54eeec6baef4 100644
--- a/setup.py
+++ b/setup.py
@@ -87,8 +87,6 @@ def required(requirements_file):
             'mycroft-enclosure-client=ovos_PHAL.__main__:main',
             'mycroft-cli-client=mycroft.client.text.__main__:main',
             'mycroft-gui-service=mycroft.gui.__main__:main'
-        ],
-        # default plugins
-        'neon.plugin.text': 'ovos-lf-utterance-normalizer=ovos_core.transformers:UtteranceNormalizer'
+        ]
     }
 )

From 1b0f6e7bd72e815d3056fb5b2e80ac2148c9b933 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Tue, 2 May 2023 21:37:13 +0100
Subject: [PATCH 4/6] also enable CoreferenceNormalizer by default since it is
 provided by a dependency and useful

---
 ovos_core/transformers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ovos_core/transformers.py b/ovos_core/transformers.py
index 5014fd1aee3b..43a3e213eb9a 100644
--- a/ovos_core/transformers.py
+++ b/ovos_core/transformers.py
@@ -16,7 +16,8 @@ def __init__(self, bus, config=None):
         self.has_loaded = False
         self.bus = bus
         self.config = self.config_core.get("utterance_transformers") or {
-            "ovos-utterance-normalizer": {}
+            "ovos-utterance-normalizer": {},
+            "ovos-utterance-coref-normalizer": {}
         }
         self.load_plugins()
 

From 047ad7ae932f5fa5b01af85012490feedfced882 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Wed, 3 May 2023 18:21:44 +0100
Subject: [PATCH 5/6] unittests

---
 test/unittests/skills/xformers.py | 112 ++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 test/unittests/skills/xformers.py

diff --git a/test/unittests/skills/xformers.py b/test/unittests/skills/xformers.py
new file mode 100644
index 000000000000..9fed98b284d8
--- /dev/null
+++ b/test/unittests/skills/xformers.py
@@ -0,0 +1,112 @@
+import unittest
+from copy import deepcopy
+from unittest.mock import Mock
+
+from ovos_plugin_manager.templates.transformers import UtteranceTransformer
+
+from ovos_utils.messagebus import FakeBus
+
+
+class MockTransformer(UtteranceTransformer):
+
+    def __init__(self):
+        super().__init__("mock_transformer")
+
+    def transform(self, utterances, context=None):
+        return utterances + ["transformer"], {}
+
+
+class MockContextAdder(UtteranceTransformer):
+
+    def __init__(self):
+        super().__init__("mock_context_adder")
+
+    def transform(self, utterances, context=None):
+        return utterances, {"old_context": False,
+                            "new_context": True,
+                            "new_key": "test"}
+
+
+class TextTransformersTests(unittest.TestCase):
+    def test_utterance_transformer_service_load(self):
+        from ovos_core.transformers import UtteranceTransformersService
+        bus = FakeBus()
+        service = UtteranceTransformersService(bus)
+        self.assertIsInstance(service.config, dict)
+        self.assertEqual(service.bus, bus)
+        self.assertFalse(service.has_loaded)
+        self.assertIsInstance(service.loaded_plugins, dict)
+        for plugin in service.loaded_plugins:
+            self.assertIsInstance(service.loaded_plugins[plugin],
+                                  UtteranceTransformer)
+        self.assertIsInstance(service.plugins, list)
+        self.assertEqual(len(service.loaded_plugins), len(service.plugins))
+        service.shutdown()
+
+    def test_utterance_transformer_service_transform(self):
+        from ovos_core.transformers import UtteranceTransformersService
+        bus = FakeBus()
+        service = UtteranceTransformersService(bus)
+        service.loaded_plugins = {"mock_transformer": MockTransformer()}
+        utterances = ["test", "utterance"]
+        context = {"old_context": True,
+                   "new_context": False}
+        utterances, context = service.transform(utterances, context)
+        self.assertEqual(" ".join(utterances), "test utterance transformer")
+        self.assertEqual(context, {"old_context": True,
+                                   "new_context": False})
+
+        service.loaded_plugins["mock_context_adder"] = MockContextAdder()
+        utterances, context = service.transform(utterances, context)
+        self.assertEqual(utterances, ["test", "utterance",
+                                      "transformer", "transformer"])
+        self.assertEqual(context, {"old_context": False,
+                                   "new_context": True,
+                                   "new_key": "test"})
+        service.shutdown()
+
+    def test_utterance_transformer_service_priority(self):
+        from ovos_core.transformers import UtteranceTransformersService
+
+        utterances = ["test 1", "test one"]
+        lang = "en-us"
+
+        def mod_1_parse(utterances, lang):
+            utterances.append("mod 1 parsed")
+            return utterances, {"parser_context": "mod_1"}
+
+        def mod_2_parse(utterances, lang):
+            utterances.append("mod 2 parsed")
+            return utterances, {"parser_context": "mod_2"}
+
+        bus = FakeBus()
+        service = UtteranceTransformersService(bus)
+
+        mod_1 = Mock()
+        mod_1.priority = 2
+        mod_1.transform = mod_1_parse
+        mod_2 = Mock()
+        mod_2.priority = 1
+        mod_2.transform = mod_2_parse
+
+        service.loaded_plugins = \
+            {"test_mod_1": mod_1,
+             "test_mod_2": mod_2}
+
+        # Check transformers adding utterances
+        new_utterances, context = service.transform(deepcopy(utterances),
+                                                    {'lang': lang})
+        self.assertEqual(context["parser_context"], "mod_2")
+        self.assertNotEqual(new_utterances, utterances)
+        self.assertEqual(len(new_utterances),
+                         len(utterances) + 2)
+
+        # Check context change on priority swap
+        mod_2.priority = 100
+        _, context = service.transform(deepcopy(utterances),
+                                       {'lang': lang})
+        self.assertEqual(context["parser_context"], "mod_1")
+
+
+if __name__ == "__main__":
+    unittest.main()

From 41e160e8b37fc54157db74450e55e4ddf4ea3e62 Mon Sep 17 00:00:00 2001
From: JarbasAi <jarbasai@mailfence.com>
Date: Wed, 3 May 2023 19:29:32 +0100
Subject: [PATCH 6/6] move pipaudit to its own workflow

---
 .github/workflows/build_tests.yml |  7 ------
 .github/workflows/pipaudit.yml    | 38 +++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 7 deletions(-)
 create mode 100644 .github/workflows/pipaudit.yml

diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml
index f35b33b66cda..eb775ff6f31a 100644
--- a/.github/workflows/build_tests.yml
+++ b/.github/workflows/build_tests.yml
@@ -40,10 +40,3 @@ jobs:
       - name: Install package
         run: |
           pip install .[mycroft,lgpl,deprecated,skills-essential]
-      - uses: pypa/gh-action-pip-audit@v1.0.0
-        with:
-          # Ignore setuptools vulnerability we can't do much about
-          # Ignore numpy vulnerability affecting latest version for Py3.7
-          ignore-vulns: |
-            GHSA-r9hx-vwmv-q579
-            GHSA-fpfv-jqm9-f5jm
diff --git a/.github/workflows/pipaudit.yml b/.github/workflows/pipaudit.yml
new file mode 100644
index 000000000000..4ea03af64266
--- /dev/null
+++ b/.github/workflows/pipaudit.yml
@@ -0,0 +1,38 @@
+name: Run PipAudit
+on:
+  push:
+    branches:
+      - master
+      - dev
+  workflow_dispatch:
+
+jobs:
+  build_tests:
+    strategy:
+      max-parallel: 2
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Build Tools
+        run: |
+          python -m pip install build wheel
+      - name: Install System Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt install python3-dev swig libssl-dev
+      - name: Install package
+        run: |
+          pip install .[skills-essential]
+      - uses: pypa/gh-action-pip-audit@v1.0.0
+        with:
+          # Ignore setuptools vulnerability we can't do much about
+          # Ignore numpy vulnerability affecting latest version for Py3.7
+          ignore-vulns: |
+            GHSA-r9hx-vwmv-q579
+            GHSA-fpfv-jqm9-f5jm