C-API: New no-copy xmldb_get_cache function for performance

Added SKIP flag to XML for skipping nodes in xml_diff

CHANGELOG.md

@@ -17,6 +17,7 @@ Expected: January 2025
 
 ### Features
 
+* C-API: New no-copy `xmldb_get_cache` function for performance
 * New: CLI generic pipe callbacks
   * Add scripts in `CLICON_CLI_PIPE_DIR`
 * New: [feature request: support xpath functions for strings](https://github.com/clicon/clixon/issues/556)

apps/backend/backend_commit.c

@@ -441,9 +441,8 @@ startup_validate(clixon_handle  h,
     }
     retval = 1;
  done:
-     if (td){
-         transaction_free(td);
-     }
+     if (td)
+         transaction_free1(td, 1);
     return retval;
  fail: /* cbret should be set */
     retval = 0;
@@ -662,7 +661,7 @@ candidate_validate(clixon_handle h,
      if (td){
          if (retval < 1)
              plugin_transaction_abort_all(h, td);
-         transaction_free(td);
+        transaction_free1(td, 1);
      }
     return retval;
  fail:
@@ -772,7 +771,7 @@ candidate_commit(clixon_handle  h,
     if (td){
         if (retval < 1)
             plugin_transaction_abort_all(h, td);
-        transaction_free(td);
+        transaction_free1(td, 1);
     }
     if (xret)
         xml_free(xret);
@@ -1081,9 +1080,8 @@ from_client_restart_one(clixon_handle    h,
         goto fail;
     retval = 1;
  done:
-    if (td){
-         transaction_free(td);
-     }
+    if (td)
+        transaction_free1(td, 1);
     return retval;
  fail:
     retval = 0;

apps/backend/backend_plugin.c

@@ -555,10 +555,32 @@ transaction_new(void)
 int
 transaction_free(transaction_data_t *td)
 {
-    if (td->td_src)
-        xml_free(td->td_src);
-    if (td->td_target)
-        xml_free(td->td_target);
+    return transaction_free1(td, 1);
+}
+
+/*! Free transaction structure
+ *
+ * @param[in]  td    Transaction data will be deallocated after the call
+ * @param[in]  copy  0: XML trees are no-copy, clear and dont free, 1: free XML trees
+ */
+int
+transaction_free1(transaction_data_t *td,
+                  int                 copy)
+{
+    if (td->td_src){
+        if (copy)
+            xml_free(td->td_src);
+        else
+            xml_apply(td->td_src, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
+                      (void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE|XML_FLAG_SKIP|XML_FLAG_MARK));
+    }
+    if (td->td_target){
+        if (copy)
+            xml_free(td->td_target);
+        else
+            xml_apply(td->td_target, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset,
+                      (void*)(XML_FLAG_NONE|XML_FLAG_ADD|XML_FLAG_DEL|XML_FLAG_CHANGE|XML_FLAG_SKIP|XML_FLAG_MARK));
+    }
     if (td->td_dvec)
         free(td->td_dvec);
     if (td->td_avec)

apps/backend/clixon_backend_plugin.h

@@ -111,6 +111,7 @@ int clixon_pagination_free(clixon_handle h);
 
 transaction_data_t * transaction_new(void);
 int transaction_free(transaction_data_t *);
+int transaction_free1(transaction_data_t *, int copy);
 
 int plugin_transaction_begin_one(clixon_plugin_t *cp, clixon_handle h, transaction_data_t *td);
 int plugin_transaction_begin_all(clixon_handle h, transaction_data_t *td);

include/clixon_custom.h

@@ -213,15 +213,23 @@
  * This causes xml_cmp to show that the datastores are unequal and may cause a wrong diff, or
  * worse case an overwrite.
  */
-
 #undef SYSTEM_ONLY_CONFIG_CANDIDATE_CLEAR
+
 /*! In full XPath namespace resolve, match even if namespace not resolved
  *
  * In the case of xpath lookup functions (eg xpath_vec_ctx) where nsc is defined, then
  * matching with XML requires equal namespaces.
  * However, some code is OK with the XPATH NSC being unresolved to NULL, even if the XML
  * namespace is defined.
  * This seems wrong and should be changed, but need further investigation
+ * @see https://github.com/clicon/clixon/issues/588
  */
 #define XPATH_NS_ACCEPT_UNRESOLVED
 
+/*! Default algorithm needs two passes to resolve "when"-protected non-presence containers
+ *
+ * A non-presence container may be protected by a YANG "when" statement which relies on
+ * default values that have not yet been resolved.
+ * A more intelligent algorithm is needed
+ */
+#define XML_DEFAULT_WHEN_TWICE

lib/clixon/clixon_datastore.h

@@ -76,6 +76,8 @@ int xmldb_get(clixon_handle h, const char *db, cvec *nsc, char *xpath, cxobj **x
 int xmldb_get0(clixon_handle h, const char *db, yang_bind yb,
                cvec *nsc, const char *xpath, int copy, withdefaults_type wdef,
                cxobj **xret, modstate_diff_t *msd, cxobj **xerr);
+int xmldb_get_cache(clixon_handle h, const char *db, yang_bind yb,
+                    cxobj **xtp, modstate_diff_t *msdiff, cxobj **xerr);
 /* in clixon_datastore_write.[ch]: */
 int xmldb_put(clixon_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret);
 int xmldb_dump(clixon_handle h, FILE *f, cxobj *xt, enum format_enum format, int pretty, withdefaults_type wdef, int multi, const char *multidb);

lib/clixon/clixon_xml.h

@@ -209,6 +209,7 @@ enum format_enum{
 #define XML_FLAG_BODYKEY  0x100 /* Text parsing key to be translated from body to key */
 #define XML_FLAG_ANYDATA  0x200 /* Treat as anydata, eg mount-points before bound */
 #define XML_FLAG_CACHE_DIRTY 0x400 /* This part of XML tree is not synced to disk */
+#define XML_FLAG_SKIP      0x800 /* Node is skipped in xml_diff */
 
 /*
  * Prototypes

lib/src/clixon_datastore.c

@@ -717,6 +717,7 @@ xmldb_db_reset(clixon_handle h,
  * @param[in]  h    Clixon handle
  * @param[in]  db   Database name
  * @retval     xml  XML cached tree or NULL
+ * @see xmldb_get_cache  Read from store if miss
  */
 cxobj *
 xmldb_cache_get(clixon_handle h,
@@ -950,6 +951,7 @@ xmldb_rename(clixon_handle h,
  * @retval     1      OK
  * @retval     0      YANG assigment and default assignment not made
  * @retval    -1      General error, check specific clicon_errno, clicon_suberrno
+ * @see xmldb_get_cache  Consider using this instead
  */
 int
 xmldb_populate(clixon_handle h,