Refs. #71 Adjust parser and conversion script

- Treat some edge cases
- Repair `convert_cf_to_yaml.py` script
- Make tests able to run with both setups (yaml and standard configs)
- Treat `.yml` as `.pre` and `.yaml` as `.cf` files for conversion and parse script

pad/rules/parser.py

@@ -223,7 +223,7 @@ def _handle_yaml_element(self, yaml_dict, _depth):
 
                 # Put in the result just the valid rule params
                 for param in value:
-                    if param in YAML_RULE_PARAMS:
+                    if param in YAML_RULE_PARAMS or param in self.ctxt.cmds:
 
                         # Score and priority should be converted to string
                         if param == "score" or param == "priority":
@@ -255,6 +255,12 @@ def _handle_yaml_element(self, yaml_dict, _depth):
                                 if locale_language.startswith(lang):
                                     self.results[key]["describe"] = desc
 
+                        # Set rule type to uri_detail and set rule value
+                        # to the value of uri_detail key
+                        elif param == "uri_detail":
+                            self.results[key]["type"] = "uri_detail"
+                            param = "value"
+                            self.results[key][param] = value["uri_detail"]
                         else:
                             self.results[key][param] = value[param]
 
@@ -316,6 +322,7 @@ def _handle_line(self, filename, line, line_no, _depth=0):
             except ValueError:
                 raise pad.errors.InvalidSyntax(filename, line_no, line,
                                                "Missing argument")
+
             if rtype == "tflags":
                 value = value.split()
 

tests/util/__init__.py

@@ -2,6 +2,8 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
+from tests.util.convert_cf_to_yaml import convert
+
 import os
 import sys
 import shutil
@@ -72,6 +74,20 @@ def setup_conf(self, config=DEFAULT_CONFIG, pre_config=DEFAULT_PRE_CONFIG):
         with open(os.path.join(self.test_conf, "20.cf"), "w") as conf:
             conf.write(config)
 
+        # If USE_YAML environment variable is set, convert file to yml
+        # and remove the old ones
+        if os.environ.get("USE_YAML") == "1":
+            convert(os.path.join(self.test_conf, "v310.pre"))
+            os.remove(os.path.join(self.test_conf, "v310.pre"))
+            convert(os.path.join(self.test_conf, "v320.pre"))
+            os.remove(os.path.join(self.test_conf, "v320.pre"))
+            convert(os.path.join(self.test_conf, "20.cf"))
+            os.remove(os.path.join(self.test_conf, "20.cf"))
+
+
+
+
+
     def check_pad(self, message, message_only=False, report_only=True,
                   extra_args=None, debug=False, expect_failure=False, env=None):
         """Run the match script and return the result.

tests/util/convert_cf_to_yaml.py

@@ -1,17 +1,24 @@
 import sys
 import yaml
+import re
 
 RULES_TYPES = {
     "full", "body", "rawbody", "uri", "meta", "header", "mimeheader", "eval"
 }
 
 RULES_SETTINGS = {
-    "score", "priority", "describe", "lang", "tflags"
+    "score", "priority", "describe", "lang", "tflags", "uri_detail"
 }
 
 
 def convert(filename):
-    new_filename = filename.rsplit(".")[0] + ".yml"
+
+    extension = filename.rsplit(".")[1]
+
+    if extension == "pre":
+        new_filename = filename.rsplit(".")[0] + ".yml"
+    elif extension == "cf":
+        new_filename = filename.rsplit(".")[0] + ".yaml"
 
     with open(filename, "r") as old, open(new_filename, "w+") as new:
         for line in old:
@@ -29,13 +36,13 @@ def convert(filename):
                 yaml_dict["ifplugin"] = line.split()[1].strip()
 
             elif line.startswith("loadplugin"):
-                yaml_dict["loadplugin"] = line.split()[1].strip()
+                yaml_dict["loadplugin"] = line.split(' ', 1)[1].strip()
 
             else:
                 rtype = line.split()[0]
                 if rtype in RULES_TYPES:
                     rname = line.split()[1]
-                    rvalue = line.rsplit(' ', 2)[2]
+                    rvalue = line.split(' ', 2)[2]
                     yaml_dict[rname] = dict()
                     yaml_dict[rname]["type"] = rtype
                     yaml_dict[rname]["value"] = rvalue
@@ -62,9 +69,11 @@ def convert(filename):
                         yaml_dict[rname] = dict()
                         yaml_dict[rname][rtype] = rvalue
                 else:
-                    rvalue = line.split(' ', 1)[1]
-                    yaml_dict[rtype] = rvalue
-                    yaml_dict
+                    try:
+                        rvalue = line.split(' ', 1)[1]
+                        yaml_dict[rtype] = rvalue
+                    except IndexError:
+                        pass
 
             yaml.dump(yaml_dict, new, default_flow_style=False)