System-only config

New `ca_system_only` backend callback for reading system-only data
New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option
API: Added `system_only` parameter to clixon_xml2file1()
Cleared running on commit and inited candidate on startup with system-only data
Added callback code in main example

CHANGELOG.md

@@ -19,6 +19,12 @@ Expected: January 2025
 
 * New: [feature request: support xpath functions for strings](https://github.com/clicon/clixon/issues/556)
   * Added: re-match, substring, string, string-length, translate, substring-before, substring-after, starts-with
+* Added support for system-only-config data
+  * A mechanism to not store sensitive data in the datastore, instead use application callbacks to store the data in system state.
+  * New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option
+  * New `system-only-config` extension
+  * New `ca_system_only` backend callback for reading system-only data
+
 ### Corrected Bugs
 
 * Fixed: [string length validation doesn't work for the entry "" in case it has default value specified](https://github.com/clicon/clixon/issues/563)

apps/backend/backend_commit.c

@@ -720,6 +720,10 @@ candidate_commit(clixon_handle  h,
      */
     if (xmldb_copy(h, db, "running") < 0)
         goto done;
+    /* Remove system-only-config data from destination */
+    if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){
+        xmldb_clear(h, "running");
+    }
     xmldb_modified_set(h, db, 0); /* reset dirty bit */
     /* Here pointers to old (source) tree are obsolete */
     if (td->td_dvec){
@@ -731,7 +735,6 @@ candidate_commit(clixon_handle  h,
         free(td->td_scvec);
         td->td_scvec = NULL;
     }
-
     /* 9. Call plugin transaction end callbacks */
     plugin_transaction_end_all(h, td);
     retval = 1;

apps/backend/backend_main.c

@@ -1030,6 +1030,13 @@ main(int    argc,
     /* Initiate the shared candidate. */
     if (xmldb_copy(h, "running", "candidate") < 0)
         goto done;
+    /* Add system-only config to candidate */
+    if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){
+        cxobj *x;
+        if ((x = xmldb_cache_get(h, "candidate")) != NULL)
+            if (xmldb_system_only_config(h, "/", NULL, &x) < 0)
+                goto done;
+    }
     if (xmldb_modified_set(h, "candidate", 0) <0)
         goto done;
     

example/main/example_backend.c

@@ -42,11 +42,14 @@
   *  -r  enable the reset function
   *  -s  enable the state function
   *  -S <file>  read state data from file, otherwise construct it programmatically (requires -s)
+  *  -o <xpath> System-only-config of xpath saved in mem
+  *  -O <file> read/write system-only-config to/from this file
   *  -i  read state file on init not by request for optimization (requires -sS <file>)
   *  -u  enable upgrade function - auto-upgrade testing
   *  -U  general-purpose upgrade
   *  -t  enable transaction logging (call syslog for every transaction)
   *  -V <xpath> Failing validate and commit if <xpath> is present (synthetic error)
+
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -70,7 +73,7 @@
 #include <clixon/clixon_backend.h>
 
 /* Command line options to be passed to getopt(3) */
-#define BACKEND_EXAMPLE_OPTS "a:m:M:n:rsS:x:iuUtV:"
+#define BACKEND_EXAMPLE_OPTS "a:m:M:n:o:O:rsS:x:iuUtV:"
 
 /* Enabling this improves performance in tests, but there may trigger the "double XPath"
  * problem.
@@ -103,6 +106,20 @@ static char *_mount_namespace = NULL;
  */
 static int _notification_stream_s = 0;
 
+/*! System-only config xpath
+ *
+ * Start backend with -o <xpath>
+ * Combined with -O <file> to write to file
+ */
+static char *_system_only_xpath = NULL;
+
+/*! System-only config file
+ *
+ * Start backend with -O <file>
+ * Combined with -o <file> to write to file
+ */
+static char *_system_only_file = NULL;
+
 /*! Variable to control if reset code is run.
  *
  * The reset code inserts "extra XML" which assumes ietf-interfaces is
@@ -185,6 +202,7 @@ static int   _validate_fail_toggle = 0; /* fail at validate and commit */
 
 /* forward */
 static int example_stream_timer_setup(clixon_handle h, int sec);
+static int main_system_only_commit(clixon_handle h, transaction_data td);
 
 int
 main_begin(clixon_handle    h,
@@ -224,17 +242,29 @@ main_complete(clixon_handle    h,
 }
 
 /*! This is called on commit. Identify modifications and adjust machine state
+ *
+ * Somewhat complex due to the different test-cases
+ * @param[in]  h   Clixon handle
+ * @param[in]  td  Transaction data
+ * @retval     0   OK
+ * @retval    -1   Error
  */
 int
 main_commit(clixon_handle    h,
             transaction_data td)
 {
+    int     retval = -1;
     cxobj  *target = transaction_target(td); /* wanted XML tree */
     cxobj **vec = NULL;
     int     i;
     size_t  len;
     cvec   *nsc = NULL;
 
+    if (_system_only_xpath != NULL){
+        if (main_system_only_commit(h, td) < 0)
+            goto done;
+        goto ok;
+    }
     if (_transaction_log)
         transaction_log(h, td, LOG_NOTICE, __FUNCTION__);
     if (_validate_fail_xpath){
@@ -242,7 +272,7 @@ main_commit(clixon_handle    h,
             xpath_first(transaction_target(td), NULL, "%s", _validate_fail_xpath)){
             _validate_fail_toggle = 0; /* toggle if triggered */
             clixon_err(OE_XML, 0, "User error");
-            return -1; /* induce fail */
+            goto done; /* simulate fail */
         }
     }
 
@@ -256,12 +286,14 @@ main_commit(clixon_handle    h,
     if (clixon_debug_get())
         for (i=0; i<len; i++)             /* Loop over added i/fs */
             xml_print(stdout, vec[i]); /* Print the added interface */
-  done:
+ ok:
+    retval = 0;
+ done:
     if (nsc)
         xml_nsctx_free(nsc);
     if (vec)
         free(vec);
-    return 0;
+    return retval;
 }
 
 int
@@ -637,6 +669,131 @@ example_statefile(clixon_handle     h,
     return retval;
 }
 
+/*! System-only config commit data
+ *
+ * @param[in]  h   Clixon handle
+ * @param[in]  td  Transaction data
+ * @retval     0   OK
+ * @retval    -1   Error
+ *
+ * System-only config data as defined by _ is not written to datastore.
+ * Instead, in this ocmmit action, it is written to file _state_file
+ * @see main_system_only_commit  callback for reading data
+ * @note  Only single system-only config data supported
+ */
+static int
+main_system_only_commit(clixon_handle    h,
+                        transaction_data td)
+{
+    int        retval = -1;
+    cxobj     *src;
+    cxobj     *target;
+    cvec      *nsc = NULL;
+    cxobj   **vec0 = NULL;
+    size_t    veclen0;
+    cxobj   **vec1 = NULL;
+    cxobj    *x0t;
+    cxobj    *x1t = NULL;
+    size_t    veclen1;
+    cxobj    *x;
+    int       i;
+    char     *xpath;
+    char     *file;
+    FILE     *fp = NULL;
+
+    clixon_debug(CLIXON_DBG_DEFAULT, "");
+    xpath = _system_only_xpath;
+    file = _system_only_file;
+    if (xpath == NULL || file == NULL){
+        clixon_err(OE_PLUGIN, EINVAL, "Both -o and -O must be given system-only config");
+        goto done;
+    }
+    src = transaction_src(td);    /* existing XML tree */
+    target = transaction_target(td); /* wanted XML tree */
+    if (xpath_vec_flag(target, nsc, "%s", XML_FLAG_ADD | XML_FLAG_CHANGE,
+                       &vec0, &veclen0, xpath) < 0)
+        goto done;
+    for (i=0; i<veclen0; i++){
+        x = vec0[i];
+        if (fp == NULL &&
+            (fp = fopen(file, "w")) == NULL){
+            clixon_err(OE_UNIX, errno, "open(%s)", file);
+            goto done;
+        }
+        xml_flag_set(x, XML_FLAG_MARK);
+        x0t = xml_root(x);
+        if ((x1t = xml_new(xml_name(x0t), NULL, CX_ELMNT)) == NULL)
+            goto done;
+        xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
+        if (xml_copy_marked(x0t, x1t) < 0) /* config */
+            goto done;
+        if (xml_apply(x0t, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
+            goto done;
+        if (clixon_xml2file(fp, x1t, 0, 1, NULL, fprintf, 1, 0) < 0)
+            goto done;
+        xml_flag_reset(x, XML_FLAG_MARK);
+        break; // XXX only single data
+    }
+    if (xpath_vec_flag(src, nsc, "%s", XML_FLAG_DEL,
+                       &vec1, &veclen1, xpath) < 0)
+        goto done;
+    for (i=0; i<veclen1; i++){
+        x = vec1[i];
+        if (fp == NULL &&
+            (fp = fopen(file, "w")) == NULL){
+            clixon_err(OE_UNIX, errno, "open(%s)", file);
+            goto done;
+        }
+        break; // XXX only single data
+    }
+    retval = 0;
+ done:
+    if (fp)
+        fclose(fp);
+    if (vec0)
+        free(vec0);
+    if (vec1)
+        free(vec1);
+    if (x1t)
+        xml_free(x1t);
+    return retval;
+}
+
+/*! Called to get system-only config data
+ *
+ * @param[in]  h       Clixon handle
+ * @param[in]  nsc     External XML namespace context, or NULL
+ * @param[in]  xpath   String with XPATH syntax. or NULL for all
+ * @param[out] xstate  XML tree, <config/> on entry.
+ * @retval     0       OK
+ * @retval    -1       Error
+ * @see example_statedata
+ * @see main_system_only_commit  where data is written
+ */
+int
+main_system_only_callback(clixon_handle h,
+                          cvec         *nsc,
+                          char         *xpath,
+                          cxobj        *xconfig)
+{
+    int    retval = -1;
+    char  *file;
+    FILE  *fp = NULL;
+
+    if ((file = _system_only_file) == NULL)
+        goto ok;
+    if ((fp = fopen(file, "r")) == NULL)
+        goto ok;
+    if (clixon_xml_parse_file(fp, YB_NONE, NULL, &xconfig, NULL) < 0)
+        goto done;
+ ok:
+    retval = 0;
+ done:
+    if (fp)
+        fclose(fp);
+    return retval;
+}
+
 /*! Example of state pagination callback and how to use pagination_data
  *
  * @param[in]  h        Generic handler
@@ -1411,6 +1568,7 @@ static clixon_plugin_api api = {
     .ca_daemon=example_daemon,              /* daemon */
     .ca_reset=example_reset,                /* reset */
     .ca_statedata=example_statedata,        /* statedata : Note fn is switched if -sS <file> */
+    .ca_system_only=main_system_only_callback, /* System-only-config callback */
     .ca_lockdb=example_lockdb,              /* Database lock changed state */
     .ca_trans_begin=main_begin,             /* trans begin */
     .ca_trans_validate=main_validate,       /* trans validate */
@@ -1461,6 +1619,12 @@ clixon_plugin_init(clixon_handle h)
         case 'n':
             _notification_stream_s = atoi(optarg);
             break;
+        case 'o':
+            _system_only_xpath = optarg;
+            break;
+        case 'O':
+            _system_only_file = optarg;
+            break;
         case 'r':
             _reset = 1;
             break;

lib/clixon/clixon_datastore.h

@@ -104,5 +104,6 @@ int xmldb_print(clixon_handle h, FILE *f);
 int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix);
 int xmldb_populate(clixon_handle h, const char *db);
 int xmldb_multi_upgrade(clixon_handle h, const char *db);
+int xmldb_system_only_config(clixon_handle h, const char *xpath, cvec *nsc, cxobj **xret);
 
 #endif /* _CLIXON_DATASTORE_H */

lib/clixon/clixon_plugin.h

@@ -214,12 +214,12 @@ typedef int (plgreset_t)(clixon_handle h, const char *db);
  * XXX: This callback may be replaced with a "dispatcher" type API in the future where the
  *      XPath binding is stricter, similar to the pagination API.
  *
- * @param[in]  h          Clixon handle
+ * @param[in]  h      Clixon handle
- * @param[in]  xpath      Part of state requested
+ * @param[in]  xpath  Part of state requested
- * @param[in]  nsc        XPath namespace context.
+ * @param[in]  nsc    XPath namespace context.
- * @param[out] xtop       XML tree where statedata is added
+ * @param[out] xtop   XML tree where data is added
- * @retval     0          OK
+ * @retval     0      OK
- * @retval    -1          Fatal error
+ * @retval    -1      Fatal error
  *
  * @note The system will make an xpath check and filter out non-matching trees
  * @note The system does not validate the xml, unless CLICON_VALIDATE_STATE_XML is set
@@ -251,7 +251,13 @@ typedef int (plglockdb_t)(clixon_handle h, char *db, int lock, int id);
  */
 typedef void *transaction_data;
 
-/* Transaction callback */
+/* Transaction callback
+ *
+ * @param[in]  h   Clixon handle
+ * @param[in]  td  Transaction data
+ * @retval     0    OK
+ * @retval    -1    Error
+ */
 typedef int (trans_cb_t)(clixon_handle h, transaction_data td);
 
 /*! Hook to override default prompt with explicit function
@@ -396,6 +402,7 @@ struct clixon_plugin_api{
             plgdaemon_t      *cb_daemon;         /* Plugin daemonized (always called) */
             plgreset_t       *cb_reset;          /* Reset system status */
             plgstatedata_t   *cb_statedata;      /* Provide state data XML from plugin */
+            plgstatedata_t   *cb_system_only;    /* Provide system-only config XML from plugin */
             plglockdb_t      *cb_lockdb;         /* Database lock changed state */
             trans_cb_t       *cb_trans_begin;    /* Transaction start */
             trans_cb_t       *cb_trans_validate; /* Transaction validation */
@@ -419,6 +426,7 @@ struct clixon_plugin_api{
 #define ca_daemon         u.cau_backend.cb_daemon
 #define ca_reset          u.cau_backend.cb_reset
 #define ca_statedata      u.cau_backend.cb_statedata
+#define ca_system_only    u.cau_backend.cb_system_only
 #define ca_lockdb         u.cau_backend.cb_lockdb
 #define ca_trans_begin    u.cau_backend.cb_trans_begin
 #define ca_trans_validate u.cau_backend.cb_trans_validate
@@ -508,6 +516,8 @@ int clixon_plugin_datastore_upgrade_all(clixon_handle h, const char *db, cxobj *
 int clixon_plugin_yang_mount_one(clixon_plugin_t *cp, clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib);
 int clixon_plugin_yang_mount_all(clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib);
 
+int clixon_plugin_system_only_all(clixon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath, cxobj **xtop);
+
 int clixon_plugin_yang_patch_one(clixon_plugin_t *cp, clixon_handle h, yang_stmt *ymod);
 int clixon_plugin_yang_patch_all(clixon_handle h, yang_stmt *ymod);
 

lib/clixon/clixon_xml_io.h

@@ -45,7 +45,7 @@
  */
 int   clixon_xml2file1(FILE *f, cxobj *xn, int level, int pretty, char *prefix,
                        clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef,
-                       int multi);
+                       int multi, int system_only);
 int   clixon_xml2file(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext);
 int   clixon_xml2file_multi(clixon_handle h, const char *db, cxobj *xn, int level, int pretty,
                             char *prefix, clicon_output_cb *fn, int skiptop, int autocliext,

lib/src/clixon_datastore.c

@@ -1021,3 +1021,40 @@ xmldb_multi_upgrade(clixon_handle h,
         free(tofile);
     return retval;
 }
+
+/*! Get system-only config data by calling user callback
+ *
+ * @param[in]     h       Clixon handle
+ * @param[in]     xpath   XPath selection, may be used to filter early
+ * @param[in]     nsc     XML Namespace context for xpath
+ * @param[in,out] xret    Existing XML tree, merge x into this, or rpc-error
+ * @retval        1       OK
+ * @retval        0       Statedata callback failed (error in xret)
+ * @retval       -1       Error (fatal)
+ */
+int
+xmldb_system_only_config(clixon_handle h,
+                         const char   *xpath,
+                         cvec         *nsc,
+                         cxobj       **xret)
+{
+    int        retval = -1;
+    yang_stmt *yspec;
+    int        ret;
+
+    clixon_debug(CLIXON_DBG_BACKEND, "");
+    if ((yspec = clicon_dbspec_yang(h)) == NULL){
+        clixon_err(OE_YANG, ENOENT, "No yang spec");
+        goto done;
+    }
+    if ((ret = clixon_plugin_system_only_all(h, yspec, nsc, (char*)xpath, xret)) < 0)
+        goto done;
+    if (ret == 0)
+        goto fail;
+    retval = 1; /* OK */
+ done:
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}

lib/src/clixon_datastore_read.c

@@ -886,6 +886,11 @@ xmldb_get_cache(clixon_handle     h,
         if (xml_apply(x1t, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
             goto done;
     }
+    if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG") &&
+        strcmp(db, "running") == 0){
+        if (xmldb_system_only_config(h, xpath?xpath:"/", nsc, &x1t) < 0)
+            goto done;
+    }
     /* If empty NACM config, then disable NACM if loaded
      */
     if (clicon_option_bool(h, "CLICON_NACM_DISABLED_ON_EMPTY")){

lib/src/clixon_datastore_write.c

@@ -1560,7 +1560,7 @@ xmldb_multi_write_applyfn(cxobj *x,
                     goto done;
                 }
                 /* Dont recurse multi-file yet */
-                if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0) < 0)
+                if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0, 0) < 0)
                     goto done;
             }
             retval = 2; /* Locally abort */
@@ -1627,7 +1627,9 @@ xmldb_dump(clixon_handle     h,
     }
     switch (format){
     case FORMAT_XML:
-        if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi) < 0)
+        if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi,
+                             clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")
+                             ) < 0)
             goto done;
         if (multi){
             mw.mw_h = h;

lib/src/clixon_plugin.c

@@ -68,6 +68,11 @@
 #include "clixon_options.h"
 #include "clixon_xml_nsctx.h"
 #include "clixon_xml_map.h"
+#include "clixon_xml_bind.h"
+#include "clixon_xml_sort.h"
+#include "clixon_xml_default.h"
+#include "clixon_xpath_ctx.h"
+#include "clixon_xpath.h"
 #include "clixon_yang_module.h"
 #include "clixon_netconf_lib.h"
 #include "clixon_validate.h"
@@ -927,6 +932,166 @@ clixon_plugin_yang_mount_all(clixon_handle   h,
     return retval;
 }
 
+/*! Call single backend statedata callback
+ *
+ * Create an xml tree (xret) for one callback only on the form:
+ *   <config>...</config>,
+ * call a user supplied function (ca_system_only) which can do two things:
+ *  - Fill in config XML in the tree and return 0
+ *  - Call cli_error() and return -1
+ * In the former case, this function returns the config XML tree to the caller (which
+ * typically merges the tree with other config trees).
+ * In the latter error case, this function returns 0 (invalid) to the caller with no tree
+ * If a fatal error occurs in this function, -1 is returned.
+ *
+ * @param[in]  cp      Plugin handle
+ * @param[in]  h       clicon handle
+ * @param[in]  nsc     namespace context for xpath
+ * @param[in]  xpath   String with XPATH syntax. or NULL for all
+ * @param[out] xp      If retval=1, tree created and returned: <config>...
+ * @retval     1       OK if callback found (and called) xret is set
+ * @retval     0       Callback failed. no XML tree returned
+ * @retval    -1       Fatal error
+ */
+static int
+clixon_plugin_system_only_one(clixon_plugin_t *cp,
+                              clixon_handle    h,
+                              cvec            *nsc,
+                              char            *xpath,
+                              cxobj          **xp)
+{
+    int             retval = -1;
+    plgstatedata_t *fn;          /* Plugin statedata fn */
+    cxobj          *x = NULL;
+    void           *wh = NULL;
+
+    if ((fn = clixon_plugin_api_get(cp)->ca_system_only) != NULL){
+        if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
+            goto done;
+        wh = NULL;
+        if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+            goto done;
+        if (fn(h, nsc, xpath, x) < 0){
+            if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+                goto done;
+            if (clixon_err_category() < 0)
+                clixon_log(h, LOG_WARNING, "%s: Internal error: Stateonly callback in plugin: %s returned -1 but did not make a clixon_err call",
+                           __FUNCTION__, clixon_plugin_name_get(cp));
+            goto fail;  /* Dont quit here on user callbacks */
+        }
+        if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
+            goto done;
+    }
+    if (xp && x){
+        *xp = x;
+        x = NULL;
+    }
+    retval = 1;
+ done:
+    if (x)
+        xml_free(x);
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
+/*! Go through all backend system-only callbacks and collect config data
+ *
+ * @param[in]     h       clicon handle
+ * @param[in]     yspec   Yang spec
+ * @param[in]     nsc     Namespace context
+ * @param[in]     xpath   String with XPATH syntax. or NULL for all
+ * @param[in,out] xret    Config XML tree is merged with existing tree.
+ * @retval        1       OK
+ * @retval        0       Callback failed (xret set with netconf-error)
+ * @retval       -1       Error
+ * @note xret can be replaced in this function
+ */
+int
+clixon_plugin_system_only_all(clixon_handle   h,
+                              yang_stmt      *yspec,
+                              cvec           *nsc,
+                              char           *xpath,
+                              cxobj         **xret)
+{
+    int              retval = -1;
+    int              ret;
+    cxobj           *x = NULL;
+    clixon_plugin_t *cp = NULL;
+    cbuf            *cberr = NULL;
+    cxobj           *xerr = NULL;
+
+    clixon_debug(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, "");
+    while ((cp = clixon_plugin_each(h, cp)) != NULL) {
+        if ((ret = clixon_plugin_system_only_one(cp, h, nsc, xpath, &x)) < 0)
+            goto done;
+        if (ret == 0){
+            if ((cberr = cbuf_new()) == NULL){
+                clixon_err(OE_UNIX, errno, "cbuf_new");
+                goto done;
+            }
+            /* error reason should be in clixon_err_reason */
+            cprintf(cberr, "Internal error, system-only callback in plugin %s returned invalid XML: %s",
+                    clixon_plugin_name_get(cp), clixon_err_reason());
+            if (netconf_operation_failed_xml(&xerr, "application", cbuf_get(cberr)) < 0)
+                goto done;
+            xml_free(*xret);
+            *xret = xerr;
+            xerr = NULL;
+            goto fail;
+        }
+        if (x == NULL)
+            continue;
+        if (xml_child_nr(x) == 0){
+            xml_free(x);
+            x = NULL;
+            continue;
+        }
+        clixon_debug_xml(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, x, "%s SYSTEM-ONLY:", clixon_plugin_name_get(cp));
+        /* XXX: ret == 0 invalid yang binding should be handled as internal error */
+        if ((ret = xml_bind_yang(h, x, YB_MODULE, yspec, &xerr)) < 0)
+            goto done;
+        if (ret == 0){
+            if (clixon_netconf_internal_error(xerr,
+                                              ". Internal error, system-only callback returned invalid XML from plugin: ",
+                                              clixon_plugin_name_get(cp)) < 0)
+                goto done;
+            xml_free(*xret);
+            *xret = xerr;
+            xerr = NULL;
+            goto fail;
+        }
+        if (xml_sort_recurse(x) < 0)
+            goto done;
+        /* Remove global defaults and empty non-presence containers */
+        if (xml_default_nopresence(x, 2, 0) < 0)
+            goto done;
+        if (xpath_first(x, nsc, "%s", xpath) != NULL){
+            if ((ret = netconf_trymerge(x, yspec, xret)) < 0)
+                goto done;
+            if (ret == 0)
+                goto fail;
+        }
+        if (x){
+            xml_free(x);
+            x = NULL;
+        }
+    } /* while plugin */
+    retval = 1;
+ done:
+    if (xerr)
+        xml_free(xerr);
+    if (cberr)
+        cbuf_free(cberr);
+    if (x)
+        xml_free(x);
+    return retval;
+ fail:
+    retval = 0;
+    goto done;
+}
+
 /*! Call plugin YANG schema patch
  *
  * @param[in]  cp      Plugin handle

lib/src/clixon_xml_io.c

@@ -203,6 +203,7 @@ xml2output_wdef(cxobj            *x,
  * @param[in]   autocliext How to handle autocli extensions: 0: ignore 1: follow
  * @param[in]   wdef       With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
  * @param[in]   multi      Multi-file split datastore, see CLICON_XMLDB_MULTI
+ * @param[in]   system_only Enable checks for system-only-config extension
  * @retval      0          OK
  * @retval     -1          Error
  * One can use clixon_xml2cbuf to get common code, but using fprintf is
@@ -223,30 +224,32 @@ xml2file_recurse(FILE                *f,
                  clicon_output_cb    *fn,
                  int                  autocliext,
                  withdefaults_type    wdef,
-                 int                  multi)
+                 int                  multi,
+                 int                  system_only)
 {
-    int           retval = -1;
+    int        retval = -1;
-    char         *name;
+    char      *name;
-    char         *namespace;
+    char      *namespace;
-    cxobj        *xc;
+    cxobj     *xc;
-    int           hasbody;
+    int        hasbody;
-    int           haselement;
+    int        haselement;
-    char         *val;
+    char      *val;
-    char         *encstr = NULL; /* xml encoded string */
+    char      *encstr = NULL; /* xml encoded string */
-    int           exist;
+    int        exist;
-    yang_stmt    *y;
+    yang_stmt *y;
-    int           level1;
+    int        level1;
-    int           tag = 0;
+    int        tag = 0;
-    int           ret;
+    int        subfile = 0;   /* File is split into subfile */
-    int           subfile = 0;   /* File is split into subfile */
+    char      *xpath = NULL;
-    char         *xpath = NULL;
+    char      *hexstr = NULL;
-    char         *hexstr = NULL;
+    int        ret;
 
     if (x == NULL)
         goto ok;
     y = xml_spec(x);
-    /* Check if system-only, then do not write to datastore */
+    /* Check if system-only, then do not write to datastore
-    if (y != NULL) {
+     */
+    if (y != NULL && system_only){
         exist = 0;
         if (yang_extension_value(y, "system-only-config", CLIXON_LIB_NS, &exist, NULL) < 0)
             goto done;
@@ -302,7 +305,7 @@ xml2file_recurse(FILE                *f,
         while ((xc = xml_child_each(x, xc, -1)) != NULL) {
             switch (xml_type(xc)){
             case CX_ATTR:
-                if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi) < 0)
+                if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
                     goto done;
                 break;
             case CX_BODY:
@@ -360,7 +363,7 @@ xml2file_recurse(FILE                *f,
                 }
                 if (xml_type(xc) != CX_ATTR && !subfile)
                         if (xml2file_recurse(f, xc, level+1, pretty, prefix,
-                                             fn, autocliext, wdef, multi) <0)
+                                             fn, autocliext, wdef, multi, system_only) <0)
                             goto done;
                 if (xa){
                     if (xml_purge(xa) < 0)
@@ -412,6 +415,7 @@ xml2file_recurse(FILE                *f,
  * @param[in]  autocliext How to handle autocli extensions: 0: ignore 1: follow
  * @param[in]  wdef       With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
  * @param[in]  multi      Multi-file split datastore, see CLICON_XMLDB_MULTI
+ * @param[in]  system_only Enable checks for system-only-config extension
  * @retval     0          OK
  * @retval    -1          Error
  * @see clixon_xml2cbuf print to a cbuf string
@@ -428,7 +432,8 @@ clixon_xml2file1(FILE                *f,
                  int                  skiptop,
                  int                  autocliext,
                  withdefaults_type    wdef,
-                 int                  multi)
+                 int                  multi,
+                 int                  system_only)
 {
     int   retval = 1;
     cxobj *xc;
@@ -438,11 +443,11 @@ clixon_xml2file1(FILE                *f,
     if (skiptop){
         xc = NULL;
         while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL)
-            if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
+            if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
                 goto done;
     }
     else {
-        if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
+        if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
             goto done;
     }
     retval = 0;
@@ -478,7 +483,7 @@ clixon_xml2file(FILE             *f,
                 int               skiptop,
                 int               autocliext)
 {
-    return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0);
+    return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0, 0);
 }
 
 /*! Print an XML tree structure to an output stream
@@ -493,7 +498,7 @@ int
 xml_print(FILE  *f,
           cxobj *x)
 {
-    return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0);
+    return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0, 0);
 }
 
 /*! Dump cxobj structure with pointers and flags for debugging, internal function
@@ -541,7 +546,7 @@ xml_dump(FILE  *f,
     return xml_dump1(f, x, 0);
 }
 
-/*! Internal: print  XML tree structure to a cligen buffer and encode chars "<>&"
+/*! Internal: print XML tree structure to a cligen buffer and encode chars "<>&"
  *
  * @param[in,out] cb       Cligen buffer to write to
  * @param[in]     xn       Clixon xml tree
@@ -724,14 +729,14 @@ xml2cbuf_recurse(cbuf             *cb,
  * @see  clixon_xml2file  to file, which is faster
  */
 int
-clixon_xml2cbuf1(cbuf                *cb,
+clixon_xml2cbuf1(cbuf             *cb,
-                 cxobj               *xn,
+                 cxobj            *xn,
-                 int                  level,
+                 int               level,
-                 int                  pretty,
+                 int               pretty,
-                 char                *prefix,
+                 char             *prefix,
-                 int32_t              depth,
+                 int32_t           depth,
-                 int                  skiptop,
+                 int               skiptop,
-                 withdefaults_type    wdef)
+                 withdefaults_type wdef)
 {
     int    retval = -1;
     cxobj *xc;

test/test_datastore_system_only.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Datastore system only config test
 # see https://github.com/clicon/clixon/pull/534 and extension system-only-config
-# Test uses a "standard" yang and a "local" yang which augmanets the standard
+# Test uses a "standard" yang and a "local" yang which augments the standard
 
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@@ -21,9 +21,6 @@ test -d $CFD || mkdir -p $CFD
 
 AUTOCLI=$(autocli_config clixon-\* kw-nokey false)
 
-# Well-known digest of mount-point xpath
-subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml
-
 cat <<EOF > $cfg
 <clixon-config xmlns="http://clicon.org/config">
   <CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@@ -39,6 +36,7 @@ cat <<EOF > $cfg
   <CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
   <CLICON_BACKEND_PIDFILE>/usr/local/var/run/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
   <CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
+  <CLICON_XMLDB_SYSTEM_ONLY_CONFIG>true</CLICON_XMLDB_SYSTEM_ONLY_CONFIG>
   <CLICON_NETCONF_MONITORING>true</CLICON_NETCONF_MONITORING>
   <CLICON_VALIDATE_STATE_XML>true</CLICON_VALIDATE_STATE_XML>
   <CLICON_STREAM_DISCOVERY_RFC5277>true</CLICON_STREAM_DISCOVERY_RFC5277>
@@ -142,16 +140,8 @@ show("Show a particular state of the system"){
 }
 EOF
 
-# Check content of db
+# Two reference files: What is expected in the datastore
-# Args:
+cat <<EOF > $dir/x_db
-# 0: dbname
-function check_db()
-{
-    dbname=$1
-
-    sudo chmod 755 $dir/${dbname}_db
-    sudo rm -f $dir/x_db
-    cat <<EOF > $dir/x_db
 <config>
    <store xmlns="urn:example:std">
       <keys>
@@ -162,13 +152,49 @@ function check_db()
    </store>
 </config>
 EOF
-    new "Check ${dbname}_db"
+
-    #    ret=$(diff $dir/x_db $dir/${dbname}_db)
+# What is expected in the system-only-config file (simulated system)
+cat <<EOF > $dir/y_db
+<store xmlns="urn:example:std">
+   <keys>
+      <key>
+         <name>a</name>
+         <system-only-data>mydata</system-only-data>
+      </key>
+   </keys>
+</store>
+EOF
+
+
+# Check content of db
+# Args:
+# 0: dbname
+# 1: system  true/false check in system or not (only after commit)
+function check_db()
+{
+    dbname=$1
+    system=$2
+
+    sudo chmod 755 $dir/${dbname}_db
+
+    new "Check not in ${dbname}_db"
     ret=$(diff $dir/x_db $dir/${dbname}_db)
     if [ $? -ne 0 ]; then
-        #        err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
         err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
     fi
+
+    if $system; then
+        new "Check $dir/system-only.xml"
+        ret=$(diff $dir/y_db $dir/system-only.xml)
+        if [ $? -ne 0 ]; then
+            err "$(cat $dir/y_db)" "$(cat $dir/system-only.xml)"
+        fi
+    else
+        new "Check no $dir/system-only.xml"
+        if [ -s $dir/system-only.xml ]; then
+            err "No file" "$(cat $dir/system-only.xml)"
+        fi
+    fi
 }
 
 new "test params: -f $cfg"
@@ -179,31 +205,89 @@ if [ $BE -ne 0 ]; then
     if [ $? -ne 0 ]; then
         err
     fi
-    new "start backend -s init -f $cfg"
+    new "start backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml"
-    start_backend -s init -f $cfg
+    start_backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml
 fi
 
-new "wait backend"
+new "wait backend 1"
 wait_backend
 
+sudo rm -f $dir/system-only.xml
+
 new "Add mydata"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></config></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
 
-new "Check mydata not in candidate"
+new "Check mydata present, but not in candidate datastore"
-check_db candidate
+check_db candidate false
 
 new "Get mydata from candidate"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
 
-new "Commit"
+new "Commit 1"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
 
-new "Check mydata not in running"
+new "Check mydata present, but not in running datastore"
-check_db running
+check_db running true
 
 new "Get mydata from running"
 expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
 
+new "Remove mydata"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data nc:operation=\"delete\" xmlns:nc=\"${BASENS}\">mydata</system-only-data></key></keys></store></config><default-operation>none</default-operation></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Check mydata present, but not in candidate datastore"
+check_db candidate true
+
+new "Commit 2"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Get mydata from running, expected not"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name></key></keys></store></data></rpc-reply>"
+
+new "Check mydata not present, but not in running datastore"
+check_db running false
+
+new "Add mydata again"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></config></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Commit 3"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Restart"
+if [ $BE -ne 0 ]; then
+    new "kill old backend"
+    sudo clixon_backend -zf $cfg
+    if [ $? -ne 0 ]; then
+        err
+    fi
+fi
+
+if [ $BE -ne 0 ]; then
+    new "start backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml"
+    start_backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml
+fi
+
+new "wait backend 2"
+wait_backend
+
+new "Check mydata present, but not in running datastore"
+check_db running true
+
+new "Get mydata from running"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
+
+new "Remove mydata"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data nc:operation=\"delete\" xmlns:nc=\"${BASENS}\">mydata</system-only-data></key></keys></store></config><default-operation>none</default-operation></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Commit 4"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
+
+new "Get mydata from running, expected not"
+expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name></key></keys></store></data></rpc-reply>"
+
+new "Check mydata not present, but not in running datastore"
+check_db running false
+
 if [ $BE -ne 0 ]; then
     new "Kill backend"
     # Check if premature kill

yang/clixon/clixon-config@2024-08-01.yang

@@ -54,6 +54,7 @@ module clixon-config {
             "Added options:
                 CLICON_YANG_DOMAIN_DIR
                 CLICON_YANG_USE_ORIGINAL
+                CLICON_XMLDB_SYSTEM_ONLY_CONFIG (tentative)
              Released in Clixon 7.2";
     }
     revision 2024-04-01 {
@@ -1194,6 +1195,15 @@ module clixon-config {
                  May not work together with CLICON_BACKEND_PRIVILEGES=drop and root, since
                  new files need to be created in XMLDB_DIR";
         }
+        leaf CLICON_XMLDB_SYSTEM_ONLY_CONFIG {
+            type boolean;
+            default true;
+            description
+                "If set, some fields in the configuration tree are not stored to datastore.
+                 Instead, the application must provide a mechanism to save the system-only-config
+                 in the system via commit/system-only-config callbacks.
+                 See also extension system-only-config in clixon-lib.yang";
+        }
         leaf CLICON_XML_CHANGELOG {
             type boolean;
             default false;

yang/clixon/clixon-lib@2024-08-01.yang

@@ -72,7 +72,7 @@ module clixon-lib {
     revision 2024-08-01 {
         description
             "Added: list-pagination-partial-state
-             Added: system-only-config extension
+             Added: system-only-config extension (tentative)
              Released in Clixon 7.2";
     }
     revision 2024-04-01 {
@@ -356,8 +356,11 @@ module clixon-lib {
         description
             "This extension marks which fields in the configuration tree should not be
              saved to datastore and be removed from memory after commit.
-             Instead, the application provides a mechanism to save the system-only-config
+             Instead, the application must provide a mechanism to save the system-only-config
-             in the system.
+             in the system:
+               1. Mark system-only config data in YANG with this extension
+               2. Write a commit callback for data write
+               2. Write a system-only-config callback for data read
              Note that the XML with these values will be remove from the datastore. The remaining XML
              still needs to be valid XML wrt YANG.
              An example of an invalid marking would be a list key. Because if the list keys are