Port to libyang 3

Refactor the code to switch to libyang 3:
- the cdefs.h file is updated to match the new definitions
- the flags used in lyd_new_* are regrouped in a newvaloptions bitmap
- the log callback management is reworked
- the system-ordered lists / leaf-lists are now ordered by key (hence the unit
  test change)

Fixes: #105
Link: https://github.com/CESNET/libyang/blob/master/doc/transition_2_3.dox
Signed-off-by: Stefan Gula <steweg@gmail.com>
Signed-off-by: Samuel Gauthier <samuel.gauthier@6wind.com>

cffi/cdefs.h

@@ -179,10 +179,10 @@ int ly_log_options(int);
 
 LY_LOG_LEVEL ly_log_level(LY_LOG_LEVEL);
 extern "Python" void lypy_log_cb(LY_LOG_LEVEL, const char *, const char *);
-void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *), int);
-struct ly_err_item *ly_err_first(const struct ly_ctx *);
+void ly_set_log_clb(void (*)(LY_LOG_LEVEL, const char *, const char *, const char *, uint64_t));
+const struct ly_err_item *ly_err_first(const struct ly_ctx *);
+const struct ly_err_item *ly_err_last(const struct ly_ctx *);
 void ly_err_clean(struct ly_ctx *, struct ly_err_item *);
-LY_VECODE ly_vecode(const struct ly_ctx *);
 
 #define LYS_UNKNOWN ...
 #define LYS_CONTAINER ...
@@ -238,14 +238,15 @@ struct lysc_node {
 
 struct ly_err_item {
     LY_LOG_LEVEL level;
-    LY_ERR no;
+    LY_ERR err;
     LY_VECODE vecode;
     char *msg;
-    char *path;
+    char *data_path;
+    char *schema_path;
+    uint64_t line;
     char *apptag;
     struct ly_err_item *next;
     struct ly_err_item *prev;
-    ...;
 };
 
 struct lyd_node {
@@ -261,11 +262,12 @@ struct lyd_node {
 
 LY_ERR lys_set_implemented(struct lys_module *,	const char **);
 
+#define LYD_NEW_VAL_OUTPUT ...
+#define LYD_NEW_VAL_BIN ...
+#define LYD_NEW_VAL_CANON ...
+#define LYD_NEW_META_CLEAR_DFLT ...
 #define LYD_NEW_PATH_UPDATE ...
-#define LYD_NEW_PATH_OUTPUT ...
-#define LYD_NEW_PATH_OPAQ   ...
-#define LYD_NEW_PATH_BIN_VALUE ...
-#define LYD_NEW_PATH_CANON_VALUE ...
+#define LYD_NEW_PATH_OPAQ ...
 LY_ERR lyd_new_path(struct lyd_node *, const struct ly_ctx *, const char *, const char *, uint32_t, struct lyd_node **);
 LY_ERR lyd_find_xpath(const struct lyd_node *, const char *, struct ly_set **);
 void lyd_unlink_siblings(struct lyd_node *node);
@@ -614,6 +616,7 @@ struct lysp_node_list {
 };
 
 struct lysc_type {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -641,6 +644,7 @@ struct lysp_type {
 struct lysp_qname {
     const char *str;
     const struct lysp_module *mod;
+    ...;
 };
 
 struct lysp_node {
@@ -682,7 +686,6 @@ struct lysc_ext {
     struct lysc_ext_instance *exts;
     struct lyplg_ext *plugin;
     struct lys_module *module;
-    uint32_t refcount;
     uint16_t flags;
 };
 
@@ -703,11 +706,10 @@ typedef enum {
     LYD_PATH_STD_NO_LAST_PRED
 } LYD_PATH_TYPE;
 
-LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **);
+LY_ERR lyd_new_term(struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_node **);
 char* lyd_path(const struct lyd_node *, LYD_PATH_TYPE, char *, size_t);
 LY_ERR lyd_new_inner(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **);
-LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, ly_bool, struct lyd_node **, ...);
-LY_ERR lyd_new_list2(struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_node **);
+LY_ERR lyd_new_list(struct lyd_node *, const struct lys_module *, const char *, uint32_t, struct lyd_node **node, ...);
 
 struct lyd_node_inner {
     union {
@@ -821,6 +823,7 @@ struct lysp_restr {
 };
 
 struct lysc_type_num {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -829,6 +832,7 @@ struct lysc_type_num {
 };
 
 struct lysc_type_dec {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -838,6 +842,7 @@ struct lysc_type_dec {
 };
 
 struct lysc_type_str {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -859,6 +864,7 @@ struct lysc_type_bitenum_item {
 };
 
 struct lysc_type_enum {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -867,6 +873,7 @@ struct lysc_type_enum {
 };
 
 struct lysc_type_bits {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -875,18 +882,19 @@ struct lysc_type_bits {
 };
 
 struct lysc_type_leafref {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
     uint32_t refcount;
     struct lyxp_expr *path;
     struct lysc_prefix *prefixes;
-    const struct lys_module *cur_mod;
     struct lysc_type *realtype;
     uint8_t require_instance;
 };
 
 struct lysc_type_identityref {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -895,6 +903,7 @@ struct lysc_type_identityref {
 };
 
 struct lysc_type_instanceid {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -903,6 +912,7 @@ struct lysc_type_instanceid {
 };
 
 struct lysc_type_union {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -911,6 +921,7 @@ struct lysc_type_union {
 };
 
 struct lysc_type_bin {
+    const char *name;
     struct lysc_ext_instance *exts;
     struct lyplg_type *plugin;
     LY_DATA_TYPE basetype;
@@ -1053,7 +1064,7 @@ LY_ERR lyd_merge_module(struct lyd_node **, const struct lyd_node *, const struc
 LY_ERR lyd_new_implicit_tree(struct lyd_node *, uint32_t, struct lyd_node **);
 LY_ERR lyd_new_implicit_all(struct lyd_node **, const struct ly_ctx *, uint32_t, struct lyd_node **);
 
-LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, ly_bool, struct lyd_meta **);
+LY_ERR lyd_new_meta(const struct ly_ctx *, struct lyd_node *, const struct lys_module *, const char *, const char *, uint32_t, struct lyd_meta **);
 
 struct ly_opaq_name {
     const char *name;

cffi/source.c

@@ -6,9 +6,6 @@
 #include <libyang/libyang.h>
 #include <libyang/version.h>
 
-#if (LY_VERSION_MAJOR != 2)
-#error "This version of libyang bindings only works with libyang 2.x"
-#endif
-#if (LY_VERSION_MINOR < 37)
-#error "Need at least libyang 2.37"
+#if (LY_VERSION_MAJOR != 3)
+#error "This version of libyang bindings only works with libyang 3.x"
 #endif

libyang/context.py

@@ -11,8 +11,8 @@
     DNode,
     data_format,
     data_type,
+    newval_flags,
     parser_flags,
-    path_flags,
     validation_flags,
 )
 from .schema import Module, SNode, schema_in_format
@@ -117,8 +117,12 @@ def error(self, msg: str, *args) -> LibyangError:
             while err:
                 if err.msg:
                     msg += ": %s" % c2str(err.msg)
-                if err.path:
-                    msg += ": %s" % c2str(err.path)
+                if err.data_path:
+                    msg += ": Data path: %s" % c2str(err.data_path)
+                if err.schema_path:
+                    msg += ": Schema path: %s" % c2str(err.schema_path)
+                if err.line != 0:
+                    msg += " (line %u)" % err.line
                 err = err.next
             lib.ly_err_clean(self.cdata, ffi.NULL)
 
@@ -244,7 +248,7 @@ def create_data_path(
                 value = str(value).lower()
             elif not isinstance(value, str):
                 value = str(value)
-        flags = path_flags(update=update, rpc_output=rpc_output)
+        flags = newval_flags(update=update, rpc_output=rpc_output)
         dnode = ffi.new("struct lyd_node **")
         ret = lib.lyd_new_path(
             parent.cdata if parent else ffi.NULL,
@@ -256,7 +260,8 @@ def create_data_path(
         )
         dnode = dnode[0]
         if ret != lib.LY_SUCCESS:
-            if lib.ly_vecode(self.cdata) != lib.LYVE_SUCCESS:
+            err = lib.ly_err_last(self.cdata)
+            if err != ffi.NULL and err.vecode != lib.LYVE_SUCCESS:
                 raise self.error("cannot create data path: %s", path)
             lib.ly_err_clean(self.cdata, ffi.NULL)
         if not dnode and not force_return_value:

libyang/data.py

@@ -77,12 +77,30 @@ def data_format(fmt_string: str) -> int:
 
 
 # -------------------------------------------------------------------------------------
-def path_flags(update: bool = False, rpc_output: bool = False) -> int:
+def newval_flags(
+    rpc_output: bool = False,
+    bin_value: bool = False,
+    canon_value: bool = False,
+    meta_clear_default: bool = False,
+    update: bool = False,
+    opaq: bool = False,
+) -> int:
+    """
+    Translate from booleans to newvaloptions flags.
+    """
     flags = 0
+    if rpc_output:
+        flags |= lib.LYD_NEW_VAL_OUTPUT
+    if bin_value:
+        flags |= lib.LYD_NEW_VAL_BIN
+    if canon_value:
+        flags |= lib.LYD_NEW_VAL_CANON
+    if meta_clear_default:
+        flags |= lib.LYD_NEW_META_CLEAR_DFLT
     if update:
         flags |= lib.LYD_NEW_PATH_UPDATE
-    if rpc_output:
-        flags |= lib.LYD_NEW_PATH_OUTPUT
+    if opaq:
+        flags |= lib.LYD_NEW_PATH_OPAQ
     return flags
 
 
@@ -297,13 +315,14 @@ def meta_free(self, name):
             item = item.next
 
     def new_meta(self, name: str, value: str, clear_dflt: bool = False):
+        flags = newval_flags(meta_clear_default=clear_dflt)
         ret = lib.lyd_new_meta(
             ffi.NULL,
             self.cdata,
             ffi.NULL,
             str2c(name),
             str2c(value),
-            clear_dflt,
+            flags,
             ffi.NULL,
         )
         if ret != lib.LY_SUCCESS:
@@ -364,20 +383,15 @@ def new_path(
         opt_bin_value: bool = False,
         opt_canon_value: bool = False,
     ):
-        opt = 0
-        if opt_update:
-            opt |= lib.LYD_NEW_PATH_UPDATE
-        if opt_output:
-            opt |= lib.LYD_NEW_PATH_OUTPUT
-        if opt_opaq:
-            opt |= lib.LYD_NEW_PATH_OPAQ
-        if opt_bin_value:
-            opt |= lib.LYD_NEW_PATH_BIN_VALUE
-        if opt_canon_value:
-            opt |= lib.LYD_NEW_PATH_CANON_VALUE
-
+        flags = newval_flags(
+            update=opt_update,
+            rpc_output=opt_output,
+            opaq=opt_opaq,
+            bin_value=opt_bin_value,
+            canon_value=opt_canon_value,
+        )
         ret = lib.lyd_new_path(
-            self.cdata, ffi.NULL, str2c(path), str2c(value), opt, ffi.NULL
+            self.cdata, ffi.NULL, str2c(path), str2c(value), flags, ffi.NULL
         )
         if ret != lib.LY_SUCCESS:
             raise self.context.error("cannot get module")
@@ -1003,7 +1017,10 @@ def _get_path(cdata) -> str:
 @DNode.register(SNode.CONTAINER)
 class DContainer(DNode):
     def create_path(
-        self, path: str, value: Any = None, rpc_output: bool = False
+        self,
+        path: str,
+        value: Any = None,
+        rpc_output: bool = False,
     ) -> Optional[DNode]:
         return self.context.create_data_path(
             path, parent=self, value=value, rpc_output=rpc_output
@@ -1177,8 +1194,14 @@ def _create_leaf(_parent, module, name, value, in_rpc_output=False):
                 value = str(value)
 
         n = ffi.new("struct lyd_node **")
+        flags = newval_flags(rpc_output=in_rpc_output)
         ret = lib.lyd_new_term(
-            _parent, module.cdata, str2c(name), str2c(value), in_rpc_output, n
+            _parent,
+            module.cdata,
+            str2c(name),
+            str2c(value),
+            flags,
+            n,
         )
 
         if ret != lib.LY_SUCCESS:
@@ -1209,11 +1232,12 @@ def _create_container(_parent, module, name, in_rpc_output=False):
 
     def _create_list(_parent, module, name, key_values, in_rpc_output=False):
         n = ffi.new("struct lyd_node **")
+        flags = newval_flags(rpc_output=in_rpc_output)
         ret = lib.lyd_new_list(
             _parent,
             module.cdata,
             str2c(name),
-            in_rpc_output,
+            flags,
             n,
             *[str2c(str(i)) for i in key_values],
         )

libyang/log.py

@@ -20,13 +20,18 @@
 
 
 @ffi.def_extern(name="lypy_log_cb")
-def libyang_c_logging_callback(level, msg, path):
+def libyang_c_logging_callback(level, msg, data_path, schema_path, line):
     args = [c2str(msg)]
-    if path:
-        fmt = "%s: %s"
-        args.append(c2str(path))
-    else:
-        fmt = "%s"
+    fmt = "%s"
+    if data_path:
+        fmt += ": %s"
+        args.append(c2str(data_path))
+    if schema_path:
+        fmt += ": %s"
+        args.append(c2str(schema_path))
+    if line != 0:
+        fmt += " line %u"
+        args.append(str(line))
     LOG.log(LOG_LEVELS.get(level, logging.NOTSET), fmt, *args)
 
 
@@ -51,10 +56,10 @@ def configure_logging(enable_py_logger: bool, level: int = logging.ERROR) -> Non
             break
     if enable_py_logger:
         lib.ly_log_options(lib.LY_LOLOG | lib.LY_LOSTORE)
-        lib.ly_set_log_clb(lib.lypy_log_cb, True)
+        lib.ly_set_log_clb(lib.lypy_log_cb)
     else:
         lib.ly_log_options(lib.LY_LOSTORE)
-        lib.ly_set_log_clb(ffi.NULL, False)
+        lib.ly_set_log_clb(ffi.NULL)
 
 
 configure_logging(False, logging.ERROR)