Added 'multiple' operator to allow complex criteria matching on payload items

CHANGELOG.rst

@@ -60,6 +60,10 @@ Added
 
   Contributed by @khushboobhatia01
 
+* Enhanced 'search' operator to allow complex criteria matching on payload items. #5482
+
+  Contributed by @erceth
+
 Fixed
 ~~~~~
 

st2common/st2common/operators.py

@@ -58,8 +58,10 @@ def search(value, criteria_pattern, criteria_condition, check_function):
 
     value: the payload list to search
     condition: one of:
-      * any - return true if any items of the list match and false if none of them match
-      * all - return true if all items of the list match and false if any of them do not match
+      * any - return true if any payload items of the list match all criteria items
+      * all - return true if all payload items of the list match all criteria items
+      * all2any - return true if all payload items of the list match any criteria items
+      * any2any - return true if any payload items match any criteria items
     pattern: a dictionary of criteria to apply to each item of the list
 
     This operator has O(n) algorithmic complexity in terms of number of child patterns.
@@ -86,78 +88,74 @@ def search(value, criteria_pattern, criteria_condition, check_function):
         ]
     }
 
-    And an example usage in criteria:
+    Example #1
 
     ---
     criteria:
       trigger.fields:
         type: search
         # Controls whether this criteria has to match any or all items of the list
-        condition: any  # or all
+        condition: any  # or all or all2any or any2any
         pattern:
           # Here our context is each item of the list
           # All of these patterns have to match the item for the item to match
           # These are simply other operators applied to each item in the list
+          # "#" and text after are ignored.
+          # This allows dictionary keys to be unique but refer to the same field
           item.field_name:
             type: "equals"
             pattern: "Status"
 
           item.to_value:
             type: "equals"
             pattern: "Approved"
+
+          item.field_name#1:
+            type: "greaterthan"
+            pattern: 40
+
+          item.field_name#2:
+            type: "lessthan"
+            pattern: 50
     """
+    if isinstance(value, dict):
+        value = [value]
+    payloadItemMatch = all
+    patternMatch = all
     if criteria_condition == "any":
-        # Any item of the list can match all patterns
-        rtn = any(
-            [
-                # Any payload item can match
-                all(
-                    [
-                        # Match all patterns
-                        check_function(
-                            child_criterion_k,
-                            child_criterion_v,
-                            PayloadLookup(
-                                child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX
-                            ),
-                        )
-                        for child_criterion_k, child_criterion_v in six.iteritems(
-                            criteria_pattern
-                        )
-                    ]
-                )
-                for child_payload in value
-            ]
-        )
-    elif criteria_condition == "all":
-        # Every item of the list must match all patterns
-        rtn = all(
-            [
-                # All payload items must match
-                all(
-                    [
-                        # Match all patterns
-                        check_function(
-                            child_criterion_k,
-                            child_criterion_v,
-                            PayloadLookup(
-                                child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX
-                            ),
-                        )
-                        for child_criterion_k, child_criterion_v in six.iteritems(
-                            criteria_pattern
-                        )
-                    ]
-                )
-                for child_payload in value
-            ]
-        )
-    else:
+        payloadItemMatch = any
+    elif criteria_condition == "all2any":
+        patternMatch = any
+    elif criteria_condition == "any2any":
+        payloadItemMatch = any
+        patternMatch = any
+    elif criteria_condition != "all":
         raise UnrecognizedConditionError(
-            "The '%s' search condition is not recognized, only 'any' "
-            "and 'all' are allowed" % criteria_condition
+            "The '%s' condition is not recognized for type search, 'any', 'all', 'any2any'"
+            " and 'all2any' are allowed" % criteria_condition
         )
 
+    rtn = payloadItemMatch(
+        [
+            # any/all payload item can match
+            patternMatch(
+                [
+                    # Match any/all patterns
+                    check_function(
+                        child_criterion_k,
+                        child_criterion_v,
+                        PayloadLookup(
+                            child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX
+                        ),
+                    )
+                    for child_criterion_k, child_criterion_v in six.iteritems(
+                        criteria_pattern
+                    )
+                ]
+            )
+            for child_payload in value
+        ]
+    )
     return rtn
 
 

st2common/tests/unit/test_operators.py

@@ -564,6 +564,203 @@ def record_function_args(criterion_k, criterion_v, payload_lookup):
             ],
         )
 
+    def _test_function(self, criterion_k, criterion_v, payload_lookup):
+        op = operators.get_operator(criterion_v["type"])
+        return op(payload_lookup.get_value("item.to_value")[0], criterion_v["pattern"])
+
+    def test_search_any2any(self):
+        # true if any payload items match any criteria
+        op = operators.get_operator("search")
+
+        payload = [
+            {
+                "field_name": "waterLevel",
+                "to_value": 30,
+            },
+            {
+                "field_name": "waterLevel",
+                "to_value": 45,
+            },
+        ]
+
+        criteria_pattern = {
+            "item.waterLevel#1": {
+                "type": "lessthan",
+                "pattern": 40,
+            },
+            "item.waterLevel#2": {
+                "type": "greaterthan",
+                "pattern": 50,
+            },
+        }
+
+        result = op(payload, criteria_pattern, "any2any", self._test_function)
+        self.assertTrue(result)
+
+        payload[0]["to_value"] = 44
+
+        result = op(payload, criteria_pattern, "any2any", self._test_function)
+        self.assertFalse(result)
+
+    def test_search_any(self):
+        # true if any payload items match all criteria
+        op = operators.get_operator("search")
+        payload = [
+            {
+                "field_name": "waterLevel",
+                "to_value": 45,
+            },
+            {
+                "field_name": "waterLevel",
+                "to_value": 20,
+            },
+        ]
+
+        criteria_pattern = {
+            "item.waterLevel#1": {
+                "type": "greaterthan",
+                "pattern": 40,
+            },
+            "item.waterLevel#2": {
+                "type": "lessthan",
+                "pattern": 50,
+            },
+            "item.waterLevel#3": {
+                "type": "equals",
+                "pattern": 46,
+            },
+        }
+
+        result = op(payload, criteria_pattern, "any", self._test_function)
+        self.assertFalse(result)
+
+        payload[0]["to_value"] = 46
+
+        result = op(payload, criteria_pattern, "any", self._test_function)
+        self.assertTrue(result)
+
+        payload[0]["to_value"] = 45
+        del criteria_pattern["item.waterLevel#3"]
+
+        result = op(payload, criteria_pattern, "any", self._test_function)
+        self.assertTrue(result)
+
+    def test_search_all2any(self):
+        # true if all payload items match any criteria
+        op = operators.get_operator("search")
+        payload = [
+            {
+                "field_name": "waterLevel",
+                "to_value": 45,
+            },
+            {
+                "field_name": "waterLevel",
+                "to_value": 20,
+            },
+        ]
+
+        criteria_pattern = {
+            "item.waterLevel#1": {
+                "type": "greaterthan",
+                "pattern": 40,
+            },
+            "item.waterLevel#2": {
+                "type": "lessthan",
+                "pattern": 50,
+            },
+            "item.waterLevel#3": {
+                "type": "equals",
+                "pattern": 46,
+            },
+        }
+
+        result = op(payload, criteria_pattern, "all2any", self._test_function)
+        self.assertTrue(result)
+
+        criteria_pattern["item.waterLevel#2"]["type"] = "greaterthan"
+
+        result = op(payload, criteria_pattern, "all2any", self._test_function)
+        self.assertFalse(result)
+
+    def test_search_all(self):
+        # true if all payload items match all criteria items
+        op = operators.get_operator("search")
+        payload = [
+            {
+                "field_name": "waterLevel",
+                "to_value": 45,
+            },
+            {
+                "field_name": "waterLevel",
+                "to_value": 46,
+            },
+        ]
+
+        criteria_pattern = {
+            "item.waterLevel#1": {
+                "type": "greaterthan",
+                "pattern": 40,
+            },
+            "item.waterLevel#2": {
+                "type": "lessthan",
+                "pattern": 50,
+            },
+        }
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertTrue(result)
+
+        payload[0]["to_value"] = 30
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertFalse(result)
+
+        payload[0]["to_value"] = 45
+
+        criteria_pattern["item.waterLevel#3"] = {
+            "type": "equals",
+            "pattern": 46,
+        }
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertFalse(result)
+
+    def test_search_payload_dict(self):
+        op = operators.get_operator("search")
+        payload = {
+            "field_name": "waterLevel",
+            "to_value": 45,
+        }
+
+        criteria_pattern = {
+            "item.waterLevel#1": {
+                "type": "greaterthan",
+                "pattern": 40,
+            },
+            "item.waterLevel#2": {
+                "type": "lessthan",
+                "pattern": 50,
+            },
+        }
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertTrue(result)
+
+        payload["to_value"] = 30
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertFalse(result)
+
+        payload["to_value"] = 45
+
+        criteria_pattern["item.waterLevel#3"] = {
+            "type": "equals",
+            "pattern": 46,
+        }
+
+        result = op(payload, criteria_pattern, "all", self._test_function)
+        self.assertFalse(result)
+
 
 class OperatorTest(unittest2.TestCase):
     def test_matchwildcard(self):

st2reactor/st2reactor/rules/filter.py

@@ -154,8 +154,10 @@ def _check_criterion(self, criterion_k, criterion_v, payload_lookup):
 
             return (False, None, None)
 
+        # Avoids the dict unique keys limitation. Allows multiple evaluations of the same payload item by a rule.
+        criterion_k_hash_strip = criterion_k.split("#", 1)[0]
         try:
-            matches = payload_lookup.get_value(criterion_k)
+            matches = payload_lookup.get_value(criterion_k_hash_strip)
             # pick value if only 1 matches else will end up being an array match.
             if matches:
                 payload_value = matches[0] if len(matches) > 0 else matches

st2reactor/tests/unit/test_filter.py

@@ -414,3 +414,25 @@ class MockSystemLookup(object):
         }
         f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule)
         self.assertTrue(f.filter())
+
+    def test_hash_strip_int_value(self):
+        rule = MOCK_RULE_1
+        rule.criteria = {
+            "trigger.int": {"type": "gt", "pattern": 0},
+            "trigger.int#2": {"type": "lt", "pattern": 2},
+        }
+        f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule)
+        self.assertTrue(f.filter(), "equals check should have passed.")
+
+        rule = MOCK_RULE_1
+        rule.criteria = {
+            "trigger.int": {"type": "gt", "pattern": 2},
+            "trigger.int#2": {"type": "lt", "pattern": 3},
+        }
+        f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule)
+        self.assertFalse(f.filter(), "trigger value is gt than 0 but didn't match.")
+
+        rule = MOCK_RULE_1
+        rule.criteria = {"trigger.int#1": {"type": "lt", "pattern": 2}}
+        f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule)
+        self.assertTrue(f.filter(), "trigger value is gt than 0 but didn't match.")