Create PythonMemberFunction and PythonObject

.vscode/settings.json

@@ -1,11 +1,18 @@
 {
     "python.testing.pytestArgs": [
-        "."
+        "tests"
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter"
     },
-    "python.formatting.provider": "none"
+    "python.formatting.provider": "none",
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./tests",
+        "-p",
+        "test_*.py"
+    ]
 }

dlg_paletteGen/classes.py

@@ -106,15 +106,18 @@ class DetailedDescription:
         "casa": r"\n-{2,20}? parameter",
     }
 
-    def __init__(self, descr: str):
+    def __init__(self, descr: str, name=None):
         """
         :param descr: Text of the detaileddescription node
         """
+        self.name = name
         self.description = descr
         self.format = ""
         self._identify_format()
         self.main_descr, self.params = self.process_descr()
-        self.brief_descr = self.main_descr.split(".")[0] + "." if self.main_descr else ""
+        self.brief_descr = (
+            self.main_descr.split(".")[0] + "." if self.main_descr else ""
+        )
 
     def _process_rEST(self, detailed_description) -> tuple:
         """
@@ -339,7 +342,9 @@ def _process_casa(self, dd: str):
         dList = dStr.split("\n")
         try:
             start_ind = [
-                idx for idx, s in enumerate(dList) if re.findall(r"-{1,20} parameter", s)
+                idx
+                for idx, s in enumerate(dList)
+                if re.findall(r"-{1,20} parameter", s)
             ][0] + 1
         except IndexError:
             start_ind = 0
@@ -414,10 +419,9 @@ def process_descr(self):
             self.description = pd[0]
             self._gen_code_block()
             return self.description, pd[1]
-        else:
-            logger.debug("Format not recognized or can't execute %s", do)
-            logger.debug("Returning description unparsed!")
-            return (self._gen_code_block(), {})
+        logger.debug("Format not recognized or can't execute %s", do)
+        logger.debug("Returning description unparsed!")
+        return (self._gen_code_block(), {})
 
 
 class GreatGrandChild:

dlg_paletteGen/module_base.py

@@ -1,3 +1,4 @@
+# pylint: disable=invalid-name
 """
 dlg_paletteGen base functionality for the treatment of installed modules.
 """
@@ -101,7 +102,7 @@ def inspect_member(member, module=None, parent=None):
         )
     ):
         logger.debug(f"Process documentation of {type(member).__name__} {name}")
-        dd = DetailedDescription(doc)
+        dd = DetailedDescription(doc, name=name)
         node["description"] = f"{dd.description.strip()}"
         if len(dd.params) > 0:
             logger.debug("Identified parameters: %s", dd.params)
@@ -115,7 +116,8 @@ def inspect_member(member, module=None, parent=None):
             module.__name__,
             member.__name__,
         )
-        dd = DetailedDescription(inspect.getdoc(module))
+        node["category"] = "PythonMemberFunction"
+        dd = DetailedDescription(inspect.getdoc(module), name=module.__name__)
         node["description"] += f"\n{dd.description.strip()}"
     elif hasattr(member, "__name__"):
         logger.debug("Member '%s' has no description!", name)
@@ -167,6 +169,7 @@ def inspect_member(member, module=None, parent=None):
     for k, field in fields.items():
         ind += 1
         if k == "self" and ind == 0:
+            node["category"] = "PythonMemberFunction"
             if member.__name__ in ["__init__", "__cls__"]:
                 fields["self"]["usage"] = "OutputPort"
             elif inspect.ismethoddescriptor(member):
@@ -238,7 +241,7 @@ def get_members(mod: types.ModuleType, module_members=[], parent=None, member=No
 
             for name, node in nodes.items():
                 if name in module_members:
-                    logger.debug("!!!!! found duplicate: %s", name)
+                    logger.info("!!!!! found duplicate: %s", name)
                 else:
                     module_members.append(name)
                     logger.debug(
@@ -257,7 +260,7 @@ def get_members(mod: types.ModuleType, module_members=[], parent=None, member=No
                         # pass
             if member:  # we've found what we wanted
                 break
-    logger.debug("Analysed %d members in module %s", count, module_name)
+    logger.info("Analysed %d members in module %s", count, module_name)
     return members
 
 

dlg_paletteGen/source_base.py

@@ -30,6 +30,7 @@
 KNOWN_DATA_CATEGORIES = [
     "File",
     "Memory",
+    "PythonObject",
     "SharedMemory",
     "NGAS",
     "S3",
@@ -93,6 +94,7 @@ def check_required_fields_for_category(message: str, fields: list, catg: str):
     if catg in [
         "DynlibApp",
         "PythonApp",
+        "PythonMemberFunction",
         "Branch",
         "BashShellApp",
         "Mpi",
@@ -104,6 +106,7 @@ def check_required_fields_for_category(message: str, fields: list, catg: str):
     if catg in [
         "DynlibApp",
         "PythonApp",
+        "PythonMemberFunction",
         "Branch",
         "BashShellApp",
         "Docker",
@@ -116,6 +119,7 @@ def check_required_fields_for_category(message: str, fields: list, catg: str):
     if catg in [
         "File",
         "Memory",
+        "PythonObject",
         "NGAS",
         "ParameterSet",
         "Plasma",
@@ -127,6 +131,7 @@ def check_required_fields_for_category(message: str, fields: list, catg: str):
     if catg in [
         "File",
         "Memory",
+        "PythonObject",
         "NGAS",
         "ParameterSet",
         "Plasma",
@@ -426,7 +431,11 @@ def create_palette_node_from_params(params) -> tuple:
             # check that type is a known value
             if not FieldType.has_key(field_type):
                 logger.warning(
-                    text + " '" + internal_name + "' field_type is Unknown: " + field_type
+                    text
+                    + " '"
+                    + internal_name
+                    + "' field_type is Unknown: "
+                    + field_type
                 )
 
             # check that usage is a known value
@@ -1009,7 +1018,9 @@ def params_to_nodes(params: dict, tag: str) -> list:
 
             # if a construct is found, add to nodes
             if data["construct"] != "":
-                logger.info("Adding component: " + data["construct"] + "/" + node["text"])
+                logger.info(
+                    "Adding component: " + data["construct"] + "/" + node["text"]
+                )
                 construct_node = create_construct_node(data["construct"], node)
                 construct_node["repositoryUrl"] = git_repo
                 construct_node["commitHash"] = version

dlg_paletteGen/support_functions.py

@@ -192,6 +192,26 @@ def write_palette_json(
         json.dump(palette, outfile, indent=4)
 
 
+def get_field_by_name(name: str, node, value_key: str = "") -> dict:
+    """
+    Get field dictionary from node providing a name.
+
+    :param name: the name of the field to retrieve
+    :param node: the node data structure
+    :value_key: the key of the attribute of the field to return
+
+    :returns the field dictionary or empty dict
+    """
+    try:
+        field = [f for f in node["fields"] if f["name"] == name][0]
+        if value_key and value_key in field:
+            return field[value_key]
+        else:
+            return field
+    except IndexError:
+        return []
+
+
 def prepare_and_write_palette(nodes: list, output_filename: str, module_doc: str = ""):
     """
     Prepare and write the palette in JSON format.
@@ -202,18 +222,29 @@ def prepare_and_write_palette(nodes: list, output_filename: str, module_doc: str
     """
     # add signature for whole palette using BlockDAG
     vertices = {}
+    v_ind = -1
     GITREPO = os.environ.get("GIT_REPO")
     VERSION = os.environ.get("PROJECT_VERSION")
 
+    funcs = []
+    f_nodes = []
     for i in range(len(nodes)):
-        vertices[i] = nodes[i]
+        func_name = get_field_by_name("func_name", nodes[i], value_key="value")
+        if func_name and func_name not in funcs:
+            funcs.append(func_name)
+            f_nodes.append(nodes[i])
+            v_ind += 1
+            vertices[v_ind] = nodes[i]
+            logger.debug("Added function %s: %s", func_name, v_ind)
+        elif func_name and func_name in funcs:
+            logger.debug("Removing duplicate function: %s", func_name)
     block_dag = build_block_dag(vertices, [], data_fields=BLOCKDAG_DATA_FIELDS)
 
     # write the output json file
     write_palette_json(
         output_filename,
         module_doc,
-        nodes,
+        f_nodes,
         GITREPO,
         VERSION,
         block_dag,
@@ -341,7 +372,9 @@ def import_using_name(mod_name: str, traverse: bool = False):
                             mod = importlib.import_module(".".join(parts[:-1]))
                             break
                         except Exception as e:
-                            raise ValueError("Problem importing module %s, %s" % (mod, e))
+                            raise ValueError(
+                                "Problem importing module %s, %s" % (mod, e)
+                            )
                 logger.debug("Loaded module: %s", mod_name)
             else:
                 logger.debug("Recursive import failed! %s", parts[0] in sys.modules)
@@ -456,7 +489,7 @@ def populateFields(parameters: dict, dd, member=None) -> dict:
             elif p != "self":
                 descr_miss.append(p)
             elif p == "self":
-                param_desc["desc"] = "Reference to object"
+                param_desc["desc"] = f"{dd.name} Object"
 
         # populate the field itself
         field[p]["value"] = field[p]["defaultValue"] = param_desc["value"]
@@ -507,7 +540,9 @@ def populateFields(parameters: dict, dd, member=None) -> dict:
             field[p]["type"] = param_desc["type"]
         if isinstance(field[p]["value"], numpy.ndarray):
             try:
-                field[p]["value"] = field[p]["defaultValue"] = field[p]["value"].tolist()
+                field[p]["value"] = field[p]["defaultValue"] = field[p][
+                    "value"
+                ].tolist()
             except NotImplementedError:
                 field[p]["value"] = []
         fields.update(field)
@@ -576,7 +611,9 @@ def populateDefaultFields(Node):  # pylint: disable=invalid-name
     et[n]["value"] = 2
     et[n]["defaultValue"] = 2
     et[n]["type"] = "Integer"
-    et[n]["description"] = "Estimate of execution time (in seconds) for this application."
+    et[n][
+        "description"
+    ] = "Estimate of execution time (in seconds) for this application."
     et[n]["parameterType"] = "ConstraintParameter"
     Node["fields"].update(et)