From 3a656fac070d2dc69cdf59be9c353130d8885e50 Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Fri, 11 Oct 2024 11:59:49 +0200 Subject: [PATCH] System-only config New `ca_system_only` backend callback for reading system-only data New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option API: Added `system_only` parameter to clixon_xml2file1() Cleared running on commit and inited candidate on startup with system-only data Added callback code in main example --- CHANGELOG.md | 6 + apps/backend/backend_commit.c | 5 +- apps/backend/backend_main.c | 7 + example/main/example_backend.c | 172 +++++++++++++++++++++- lib/clixon/clixon_datastore.h | 1 + lib/clixon/clixon_plugin.h | 24 ++- lib/clixon/clixon_xml_io.h | 2 +- lib/src/clixon_datastore.c | 37 +++++ lib/src/clixon_datastore_read.c | 5 + lib/src/clixon_datastore_write.c | 6 +- lib/src/clixon_plugin.c | 165 +++++++++++++++++++++ lib/src/clixon_xml_io.c | 75 +++++----- test/test_datastore_system_only.sh | 134 +++++++++++++---- yang/clixon/clixon-config@2024-08-01.yang | 10 ++ yang/clixon/clixon-lib@2024-08-01.yang | 9 +- 15 files changed, 580 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6218c127..740a3c60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,12 @@ Expected: January 2025 * New: [feature request: support xpath functions for strings](https://github.com/clicon/clixon/issues/556) * Added: re-match, substring, string, string-length, translate, substring-before, substring-after, starts-with +* Added support for system-only-config data + * A mechanism to not store sensitive data in the datastore, instead use application callbacks to store the data in system state. + * New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option + * New `system-only-config` extension + * New `ca_system_only` backend callback for reading system-only data + ### Corrected Bugs * Fixed: [string length validation doesn't work for the entry "" in case it has default value specified](https://github.com/clicon/clixon/issues/563) diff --git a/apps/backend/backend_commit.c b/apps/backend/backend_commit.c index 15dd15dd..913c76cc 100644 --- a/apps/backend/backend_commit.c +++ b/apps/backend/backend_commit.c @@ -720,6 +720,10 @@ candidate_commit(clixon_handle h, */ if (xmldb_copy(h, db, "running") < 0) goto done; + /* Remove system-only-config data from destination */ + if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){ + xmldb_clear(h, "running"); + } xmldb_modified_set(h, db, 0); /* reset dirty bit */ /* Here pointers to old (source) tree are obsolete */ if (td->td_dvec){ @@ -731,7 +735,6 @@ candidate_commit(clixon_handle h, free(td->td_scvec); td->td_scvec = NULL; } - /* 9. Call plugin transaction end callbacks */ plugin_transaction_end_all(h, td); retval = 1; diff --git a/apps/backend/backend_main.c b/apps/backend/backend_main.c index c83e7b1a..6245b706 100644 --- a/apps/backend/backend_main.c +++ b/apps/backend/backend_main.c @@ -1030,6 +1030,13 @@ main(int argc, /* Initiate the shared candidate. */ if (xmldb_copy(h, "running", "candidate") < 0) goto done; + /* Add system-only config to candidate */ + if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){ + cxobj *x; + if ((x = xmldb_cache_get(h, "candidate")) != NULL) + if (xmldb_system_only_config(h, "/", NULL, &x) < 0) + goto done; + } if (xmldb_modified_set(h, "candidate", 0) <0) goto done; diff --git a/example/main/example_backend.c b/example/main/example_backend.c index 48671ead..d853280f 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -42,11 +42,14 @@ * -r enable the reset function * -s enable the state function * -S read state data from file, otherwise construct it programmatically (requires -s) + * -o System-only-config of xpath saved in mem + * -O read/write system-only-config to/from this file * -i read state file on init not by request for optimization (requires -sS ) * -u enable upgrade function - auto-upgrade testing * -U general-purpose upgrade * -t enable transaction logging (call syslog for every transaction) * -V Failing validate and commit if is present (synthetic error) + */ #include #include @@ -70,7 +73,7 @@ #include /* Command line options to be passed to getopt(3) */ -#define BACKEND_EXAMPLE_OPTS "a:m:M:n:rsS:x:iuUtV:" +#define BACKEND_EXAMPLE_OPTS "a:m:M:n:o:O:rsS:x:iuUtV:" /* Enabling this improves performance in tests, but there may trigger the "double XPath" * problem. @@ -103,6 +106,20 @@ static char *_mount_namespace = NULL; */ static int _notification_stream_s = 0; +/*! System-only config xpath + * + * Start backend with -o + * Combined with -O to write to file + */ +static char *_system_only_xpath = NULL; + +/*! System-only config file + * + * Start backend with -O + * Combined with -o to write to file + */ +static char *_system_only_file = NULL; + /*! Variable to control if reset code is run. * * The reset code inserts "extra XML" which assumes ietf-interfaces is @@ -185,6 +202,7 @@ static int _validate_fail_toggle = 0; /* fail at validate and commit */ /* forward */ static int example_stream_timer_setup(clixon_handle h, int sec); +static int main_system_only_commit(clixon_handle h, transaction_data td); int main_begin(clixon_handle h, @@ -224,17 +242,29 @@ main_complete(clixon_handle h, } /*! This is called on commit. Identify modifications and adjust machine state + * + * Somewhat complex due to the different test-cases + * @param[in] h Clixon handle + * @param[in] td Transaction data + * @retval 0 OK + * @retval -1 Error */ int main_commit(clixon_handle h, transaction_data td) { + int retval = -1; cxobj *target = transaction_target(td); /* wanted XML tree */ cxobj **vec = NULL; int i; size_t len; cvec *nsc = NULL; + if (_system_only_xpath != NULL){ + if (main_system_only_commit(h, td) < 0) + goto done; + goto ok; + } if (_transaction_log) transaction_log(h, td, LOG_NOTICE, __FUNCTION__); if (_validate_fail_xpath){ @@ -242,7 +272,7 @@ main_commit(clixon_handle h, xpath_first(transaction_target(td), NULL, "%s", _validate_fail_xpath)){ _validate_fail_toggle = 0; /* toggle if triggered */ clixon_err(OE_XML, 0, "User error"); - return -1; /* induce fail */ + goto done; /* simulate fail */ } } @@ -256,12 +286,14 @@ main_commit(clixon_handle h, if (clixon_debug_get()) for (i=0; i on entry. + * @retval 0 OK + * @retval -1 Error + * @see example_statedata + * @see main_system_only_commit where data is written + */ +int +main_system_only_callback(clixon_handle h, + cvec *nsc, + char *xpath, + cxobj *xconfig) +{ + int retval = -1; + char *file; + FILE *fp = NULL; + + if ((file = _system_only_file) == NULL) + goto ok; + if ((fp = fopen(file, "r")) == NULL) + goto ok; + if (clixon_xml_parse_file(fp, YB_NONE, NULL, &xconfig, NULL) < 0) + goto done; + ok: + retval = 0; + done: + if (fp) + fclose(fp); + return retval; +} + /*! Example of state pagination callback and how to use pagination_data * * @param[in] h Generic handler @@ -1411,6 +1568,7 @@ static clixon_plugin_api api = { .ca_daemon=example_daemon, /* daemon */ .ca_reset=example_reset, /* reset */ .ca_statedata=example_statedata, /* statedata : Note fn is switched if -sS */ + .ca_system_only=main_system_only_callback, /* System-only-config callback */ .ca_lockdb=example_lockdb, /* Database lock changed state */ .ca_trans_begin=main_begin, /* trans begin */ .ca_trans_validate=main_validate, /* trans validate */ @@ -1461,6 +1619,12 @@ clixon_plugin_init(clixon_handle h) case 'n': _notification_stream_s = atoi(optarg); break; + case 'o': + _system_only_xpath = optarg; + break; + case 'O': + _system_only_file = optarg; + break; case 'r': _reset = 1; break; diff --git a/lib/clixon/clixon_datastore.h b/lib/clixon/clixon_datastore.h index 5df21bf4..ea770359 100644 --- a/lib/clixon/clixon_datastore.h +++ b/lib/clixon/clixon_datastore.h @@ -104,5 +104,6 @@ int xmldb_print(clixon_handle h, FILE *f); int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix); int xmldb_populate(clixon_handle h, const char *db); int xmldb_multi_upgrade(clixon_handle h, const char *db); +int xmldb_system_only_config(clixon_handle h, const char *xpath, cvec *nsc, cxobj **xret); #endif /* _CLIXON_DATASTORE_H */ diff --git a/lib/clixon/clixon_plugin.h b/lib/clixon/clixon_plugin.h index d8480591..5cffa5f0 100644 --- a/lib/clixon/clixon_plugin.h +++ b/lib/clixon/clixon_plugin.h @@ -214,12 +214,12 @@ typedef int (plgreset_t)(clixon_handle h, const char *db); * XXX: This callback may be replaced with a "dispatcher" type API in the future where the * XPath binding is stricter, similar to the pagination API. * - * @param[in] h Clixon handle - * @param[in] xpath Part of state requested - * @param[in] nsc XPath namespace context. - * @param[out] xtop XML tree where statedata is added - * @retval 0 OK - * @retval -1 Fatal error + * @param[in] h Clixon handle + * @param[in] xpath Part of state requested + * @param[in] nsc XPath namespace context. + * @param[out] xtop XML tree where data is added + * @retval 0 OK + * @retval -1 Fatal error * * @note The system will make an xpath check and filter out non-matching trees * @note The system does not validate the xml, unless CLICON_VALIDATE_STATE_XML is set @@ -251,7 +251,13 @@ typedef int (plglockdb_t)(clixon_handle h, char *db, int lock, int id); */ typedef void *transaction_data; -/* Transaction callback */ +/* Transaction callback + * + * @param[in] h Clixon handle + * @param[in] td Transaction data + * @retval 0 OK + * @retval -1 Error + */ typedef int (trans_cb_t)(clixon_handle h, transaction_data td); /*! Hook to override default prompt with explicit function @@ -396,6 +402,7 @@ struct clixon_plugin_api{ plgdaemon_t *cb_daemon; /* Plugin daemonized (always called) */ plgreset_t *cb_reset; /* Reset system status */ plgstatedata_t *cb_statedata; /* Provide state data XML from plugin */ + plgstatedata_t *cb_system_only; /* Provide system-only config XML from plugin */ plglockdb_t *cb_lockdb; /* Database lock changed state */ trans_cb_t *cb_trans_begin; /* Transaction start */ trans_cb_t *cb_trans_validate; /* Transaction validation */ @@ -419,6 +426,7 @@ struct clixon_plugin_api{ #define ca_daemon u.cau_backend.cb_daemon #define ca_reset u.cau_backend.cb_reset #define ca_statedata u.cau_backend.cb_statedata +#define ca_system_only u.cau_backend.cb_system_only #define ca_lockdb u.cau_backend.cb_lockdb #define ca_trans_begin u.cau_backend.cb_trans_begin #define ca_trans_validate u.cau_backend.cb_trans_validate @@ -508,6 +516,8 @@ int clixon_plugin_datastore_upgrade_all(clixon_handle h, const char *db, cxobj * int clixon_plugin_yang_mount_one(clixon_plugin_t *cp, clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib); int clixon_plugin_yang_mount_all(clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib); +int clixon_plugin_system_only_all(clixon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath, cxobj **xtop); + int clixon_plugin_yang_patch_one(clixon_plugin_t *cp, clixon_handle h, yang_stmt *ymod); int clixon_plugin_yang_patch_all(clixon_handle h, yang_stmt *ymod); diff --git a/lib/clixon/clixon_xml_io.h b/lib/clixon/clixon_xml_io.h index fba224c3..0e36dc25 100644 --- a/lib/clixon/clixon_xml_io.h +++ b/lib/clixon/clixon_xml_io.h @@ -45,7 +45,7 @@ */ int clixon_xml2file1(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef, - int multi); + int multi, int system_only); int clixon_xml2file(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext); int clixon_xml2file_multi(clixon_handle h, const char *db, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext, diff --git a/lib/src/clixon_datastore.c b/lib/src/clixon_datastore.c index d8880ba8..ad6e06c4 100644 --- a/lib/src/clixon_datastore.c +++ b/lib/src/clixon_datastore.c @@ -1021,3 +1021,40 @@ xmldb_multi_upgrade(clixon_handle h, free(tofile); return retval; } + +/*! Get system-only config data by calling user callback + * + * @param[in] h Clixon handle + * @param[in] xpath XPath selection, may be used to filter early + * @param[in] nsc XML Namespace context for xpath + * @param[in,out] xret Existing XML tree, merge x into this, or rpc-error + * @retval 1 OK + * @retval 0 Statedata callback failed (error in xret) + * @retval -1 Error (fatal) + */ +int +xmldb_system_only_config(clixon_handle h, + const char *xpath, + cvec *nsc, + cxobj **xret) +{ + int retval = -1; + yang_stmt *yspec; + int ret; + + clixon_debug(CLIXON_DBG_BACKEND, ""); + if ((yspec = clicon_dbspec_yang(h)) == NULL){ + clixon_err(OE_YANG, ENOENT, "No yang spec"); + goto done; + } + if ((ret = clixon_plugin_system_only_all(h, yspec, nsc, (char*)xpath, xret)) < 0) + goto done; + if (ret == 0) + goto fail; + retval = 1; /* OK */ + done: + return retval; + fail: + retval = 0; + goto done; +} diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index d728b5ae..51239373 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -886,6 +886,11 @@ xmldb_get_cache(clixon_handle h, if (xml_apply(x1t, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0) goto done; } + if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG") && + strcmp(db, "running") == 0){ + if (xmldb_system_only_config(h, xpath?xpath:"/", nsc, &x1t) < 0) + goto done; + } /* If empty NACM config, then disable NACM if loaded */ if (clicon_option_bool(h, "CLICON_NACM_DISABLED_ON_EMPTY")){ diff --git a/lib/src/clixon_datastore_write.c b/lib/src/clixon_datastore_write.c index 0e19994e..42a221e3 100644 --- a/lib/src/clixon_datastore_write.c +++ b/lib/src/clixon_datastore_write.c @@ -1560,7 +1560,7 @@ xmldb_multi_write_applyfn(cxobj *x, goto done; } /* Dont recurse multi-file yet */ - if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0) < 0) + if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0, 0) < 0) goto done; } retval = 2; /* Locally abort */ @@ -1627,7 +1627,9 @@ xmldb_dump(clixon_handle h, } switch (format){ case FORMAT_XML: - if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi) < 0) + if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi, + clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG") + ) < 0) goto done; if (multi){ mw.mw_h = h; diff --git a/lib/src/clixon_plugin.c b/lib/src/clixon_plugin.c index 68cb46b1..f7f0e7fe 100644 --- a/lib/src/clixon_plugin.c +++ b/lib/src/clixon_plugin.c @@ -68,6 +68,11 @@ #include "clixon_options.h" #include "clixon_xml_nsctx.h" #include "clixon_xml_map.h" +#include "clixon_xml_bind.h" +#include "clixon_xml_sort.h" +#include "clixon_xml_default.h" +#include "clixon_xpath_ctx.h" +#include "clixon_xpath.h" #include "clixon_yang_module.h" #include "clixon_netconf_lib.h" #include "clixon_validate.h" @@ -927,6 +932,166 @@ clixon_plugin_yang_mount_all(clixon_handle h, return retval; } +/*! Call single backend statedata callback + * + * Create an xml tree (xret) for one callback only on the form: + * ..., + * call a user supplied function (ca_system_only) which can do two things: + * - Fill in config XML in the tree and return 0 + * - Call cli_error() and return -1 + * In the former case, this function returns the config XML tree to the caller (which + * typically merges the tree with other config trees). + * In the latter error case, this function returns 0 (invalid) to the caller with no tree + * If a fatal error occurs in this function, -1 is returned. + * + * @param[in] cp Plugin handle + * @param[in] h clicon handle + * @param[in] nsc namespace context for xpath + * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[out] xp If retval=1, tree created and returned: ... + * @retval 1 OK if callback found (and called) xret is set + * @retval 0 Callback failed. no XML tree returned + * @retval -1 Fatal error + */ +static int +clixon_plugin_system_only_one(clixon_plugin_t *cp, + clixon_handle h, + cvec *nsc, + char *xpath, + cxobj **xp) +{ + int retval = -1; + plgstatedata_t *fn; /* Plugin statedata fn */ + cxobj *x = NULL; + void *wh = NULL; + + if ((fn = clixon_plugin_api_get(cp)->ca_system_only) != NULL){ + if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL) + goto done; + wh = NULL; + if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0) + goto done; + if (fn(h, nsc, xpath, x) < 0){ + if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0) + goto done; + if (clixon_err_category() < 0) + clixon_log(h, LOG_WARNING, "%s: Internal error: Stateonly callback in plugin: %s returned -1 but did not make a clixon_err call", + __FUNCTION__, clixon_plugin_name_get(cp)); + goto fail; /* Dont quit here on user callbacks */ + } + if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0) + goto done; + } + if (xp && x){ + *xp = x; + x = NULL; + } + retval = 1; + done: + if (x) + xml_free(x); + return retval; + fail: + retval = 0; + goto done; +} + +/*! Go through all backend system-only callbacks and collect config data + * + * @param[in] h clicon handle + * @param[in] yspec Yang spec + * @param[in] nsc Namespace context + * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[in,out] xret Config XML tree is merged with existing tree. + * @retval 1 OK + * @retval 0 Callback failed (xret set with netconf-error) + * @retval -1 Error + * @note xret can be replaced in this function + */ +int +clixon_plugin_system_only_all(clixon_handle h, + yang_stmt *yspec, + cvec *nsc, + char *xpath, + cxobj **xret) +{ + int retval = -1; + int ret; + cxobj *x = NULL; + clixon_plugin_t *cp = NULL; + cbuf *cberr = NULL; + cxobj *xerr = NULL; + + clixon_debug(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, ""); + while ((cp = clixon_plugin_each(h, cp)) != NULL) { + if ((ret = clixon_plugin_system_only_one(cp, h, nsc, xpath, &x)) < 0) + goto done; + if (ret == 0){ + if ((cberr = cbuf_new()) == NULL){ + clixon_err(OE_UNIX, errno, "cbuf_new"); + goto done; + } + /* error reason should be in clixon_err_reason */ + cprintf(cberr, "Internal error, system-only callback in plugin %s returned invalid XML: %s", + clixon_plugin_name_get(cp), clixon_err_reason()); + if (netconf_operation_failed_xml(&xerr, "application", cbuf_get(cberr)) < 0) + goto done; + xml_free(*xret); + *xret = xerr; + xerr = NULL; + goto fail; + } + if (x == NULL) + continue; + if (xml_child_nr(x) == 0){ + xml_free(x); + x = NULL; + continue; + } + clixon_debug_xml(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, x, "%s SYSTEM-ONLY:", clixon_plugin_name_get(cp)); + /* XXX: ret == 0 invalid yang binding should be handled as internal error */ + if ((ret = xml_bind_yang(h, x, YB_MODULE, yspec, &xerr)) < 0) + goto done; + if (ret == 0){ + if (clixon_netconf_internal_error(xerr, + ". Internal error, system-only callback returned invalid XML from plugin: ", + clixon_plugin_name_get(cp)) < 0) + goto done; + xml_free(*xret); + *xret = xerr; + xerr = NULL; + goto fail; + } + if (xml_sort_recurse(x) < 0) + goto done; + /* Remove global defaults and empty non-presence containers */ + if (xml_default_nopresence(x, 2, 0) < 0) + goto done; + if (xpath_first(x, nsc, "%s", xpath) != NULL){ + if ((ret = netconf_trymerge(x, yspec, xret)) < 0) + goto done; + if (ret == 0) + goto fail; + } + if (x){ + xml_free(x); + x = NULL; + } + } /* while plugin */ + retval = 1; + done: + if (xerr) + xml_free(xerr); + if (cberr) + cbuf_free(cberr); + if (x) + xml_free(x); + return retval; + fail: + retval = 0; + goto done; +} + /*! Call plugin YANG schema patch * * @param[in] cp Plugin handle diff --git a/lib/src/clixon_xml_io.c b/lib/src/clixon_xml_io.c index 2d7d1ba0..e4382b58 100644 --- a/lib/src/clixon_xml_io.c +++ b/lib/src/clixon_xml_io.c @@ -203,6 +203,7 @@ xml2output_wdef(cxobj *x, * @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow * @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL * @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI + * @param[in] system_only Enable checks for system-only-config extension * @retval 0 OK * @retval -1 Error * One can use clixon_xml2cbuf to get common code, but using fprintf is @@ -223,30 +224,32 @@ xml2file_recurse(FILE *f, clicon_output_cb *fn, int autocliext, withdefaults_type wdef, - int multi) + int multi, + int system_only) { - int retval = -1; - char *name; - char *namespace; - cxobj *xc; - int hasbody; - int haselement; - char *val; - char *encstr = NULL; /* xml encoded string */ - int exist; - yang_stmt *y; - int level1; - int tag = 0; - int ret; - int subfile = 0; /* File is split into subfile */ - char *xpath = NULL; - char *hexstr = NULL; + int retval = -1; + char *name; + char *namespace; + cxobj *xc; + int hasbody; + int haselement; + char *val; + char *encstr = NULL; /* xml encoded string */ + int exist; + yang_stmt *y; + int level1; + int tag = 0; + int subfile = 0; /* File is split into subfile */ + char *xpath = NULL; + char *hexstr = NULL; + int ret; if (x == NULL) goto ok; y = xml_spec(x); - /* Check if system-only, then do not write to datastore */ - if (y != NULL) { + /* Check if system-only, then do not write to datastore + */ + if (y != NULL && system_only){ exist = 0; if (yang_extension_value(y, "system-only-config", CLIXON_LIB_NS, &exist, NULL) < 0) goto done; @@ -302,7 +305,7 @@ xml2file_recurse(FILE *f, while ((xc = xml_child_each(x, xc, -1)) != NULL) { switch (xml_type(xc)){ case CX_ATTR: - if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi) < 0) + if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0) goto done; break; case CX_BODY: @@ -360,7 +363,7 @@ xml2file_recurse(FILE *f, } if (xml_type(xc) != CX_ATTR && !subfile) if (xml2file_recurse(f, xc, level+1, pretty, prefix, - fn, autocliext, wdef, multi) <0) + fn, autocliext, wdef, multi, system_only) <0) goto done; if (xa){ if (xml_purge(xa) < 0) @@ -412,6 +415,7 @@ xml2file_recurse(FILE *f, * @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow * @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL * @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI + * @param[in] system_only Enable checks for system-only-config extension * @retval 0 OK * @retval -1 Error * @see clixon_xml2cbuf print to a cbuf string @@ -428,7 +432,8 @@ clixon_xml2file1(FILE *f, int skiptop, int autocliext, withdefaults_type wdef, - int multi) + int multi, + int system_only) { int retval = 1; cxobj *xc; @@ -438,11 +443,11 @@ clixon_xml2file1(FILE *f, if (skiptop){ xc = NULL; while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL) - if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi) < 0) + if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0) goto done; } else { - if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi) < 0) + if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0) goto done; } retval = 0; @@ -478,7 +483,7 @@ clixon_xml2file(FILE *f, int skiptop, int autocliext) { - return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0); + return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0, 0); } /*! Print an XML tree structure to an output stream @@ -493,7 +498,7 @@ int xml_print(FILE *f, cxobj *x) { - return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0); + return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0, 0); } /*! Dump cxobj structure with pointers and flags for debugging, internal function @@ -541,7 +546,7 @@ xml_dump(FILE *f, return xml_dump1(f, x, 0); } -/*! Internal: print XML tree structure to a cligen buffer and encode chars "<>&" +/*! Internal: print XML tree structure to a cligen buffer and encode chars "<>&" * * @param[in,out] cb Cligen buffer to write to * @param[in] xn Clixon xml tree @@ -724,14 +729,14 @@ xml2cbuf_recurse(cbuf *cb, * @see clixon_xml2file to file, which is faster */ int -clixon_xml2cbuf1(cbuf *cb, - cxobj *xn, - int level, - int pretty, - char *prefix, - int32_t depth, - int skiptop, - withdefaults_type wdef) +clixon_xml2cbuf1(cbuf *cb, + cxobj *xn, + int level, + int pretty, + char *prefix, + int32_t depth, + int skiptop, + withdefaults_type wdef) { int retval = -1; cxobj *xc; diff --git a/test/test_datastore_system_only.sh b/test/test_datastore_system_only.sh index 944dbb48..b789d0a5 100755 --- a/test/test_datastore_system_only.sh +++ b/test/test_datastore_system_only.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Datastore system only config test # see https://github.com/clicon/clixon/pull/534 and extension system-only-config -# Test uses a "standard" yang and a "local" yang which augmanets the standard +# Test uses a "standard" yang and a "local" yang which augments the standard # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi @@ -21,9 +21,6 @@ test -d $CFD || mkdir -p $CFD AUTOCLI=$(autocli_config clixon-\* kw-nokey false) -# Well-known digest of mount-point xpath -subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml - cat < $cfg $cfg @@ -39,6 +36,7 @@ cat < $cfg /usr/local/lib/$APPNAME/backend /usr/local/var/run/$APPNAME.pidfile $dir + true true true true @@ -142,16 +140,8 @@ show("Show a particular state of the system"){ } EOF -# Check content of db -# Args: -# 0: dbname -function check_db() -{ - dbname=$1 - - sudo chmod 755 $dir/${dbname}_db - sudo rm -f $dir/x_db - cat < $dir/x_db +# Two reference files: What is expected in the datastore +cat < $dir/x_db @@ -162,13 +152,49 @@ function check_db() EOF - new "Check ${dbname}_db" - # ret=$(diff $dir/x_db $dir/${dbname}_db) + +# What is expected in the system-only-config file (simulated system) +cat < $dir/y_db + + + + a + mydata + + + +EOF + + +# Check content of db +# Args: +# 0: dbname +# 1: system true/false check in system or not (only after commit) +function check_db() +{ + dbname=$1 + system=$2 + + sudo chmod 755 $dir/${dbname}_db + + new "Check not in ${dbname}_db" ret=$(diff $dir/x_db $dir/${dbname}_db) if [ $? -ne 0 ]; then - # err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)" err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)" fi + + if $system; then + new "Check $dir/system-only.xml" + ret=$(diff $dir/y_db $dir/system-only.xml) + if [ $? -ne 0 ]; then + err "$(cat $dir/y_db)" "$(cat $dir/system-only.xml)" + fi + else + new "Check no $dir/system-only.xml" + if [ -s $dir/system-only.xml ]; then + err "No file" "$(cat $dir/system-only.xml)" + fi + fi } new "test params: -f $cfg" @@ -179,31 +205,89 @@ if [ $BE -ne 0 ]; then if [ $? -ne 0 ]; then err fi - new "start backend -s init -f $cfg" - start_backend -s init -f $cfg + new "start backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml" + start_backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml fi -new "wait backend" +new "wait backend 1" wait_backend +sudo rm -f $dir/system-only.xml + new "Add mydata" expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "amydata" "" -new "Check mydata not in candidate" -check_db candidate +new "Check mydata present, but not in candidate datastore" +check_db candidate false new "Get mydata from candidate" expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "amydata" -new "Commit" +new "Commit 1" expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" -new "Check mydata not in running" -check_db running +new "Check mydata present, but not in running datastore" +check_db running true new "Get mydata from running" expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "amydata" +new "Remove mydata" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "amydatanone" "" + +new "Check mydata present, but not in candidate datastore" +check_db candidate true + +new "Commit 2" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" + +new "Get mydata from running, expected not" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "a" + +new "Check mydata not present, but not in running datastore" +check_db running false + +new "Add mydata again" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "amydata" "" + +new "Commit 3" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" + +new "Restart" +if [ $BE -ne 0 ]; then + new "kill old backend" + sudo clixon_backend -zf $cfg + if [ $? -ne 0 ]; then + err + fi +fi + +if [ $BE -ne 0 ]; then + new "start backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml" + start_backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml +fi + +new "wait backend 2" +wait_backend + +new "Check mydata present, but not in running datastore" +check_db running true + +new "Get mydata from running" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "amydata" + +new "Remove mydata" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "amydatanone" "" + +new "Commit 4" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "" + +new "Get mydata from running, expected not" +expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "" "a" + +new "Check mydata not present, but not in running datastore" +check_db running false + if [ $BE -ne 0 ]; then new "Kill backend" # Check if premature kill diff --git a/yang/clixon/clixon-config@2024-08-01.yang b/yang/clixon/clixon-config@2024-08-01.yang index 2d8d980d..cb16f779 100644 --- a/yang/clixon/clixon-config@2024-08-01.yang +++ b/yang/clixon/clixon-config@2024-08-01.yang @@ -54,6 +54,7 @@ module clixon-config { "Added options: CLICON_YANG_DOMAIN_DIR CLICON_YANG_USE_ORIGINAL + CLICON_XMLDB_SYSTEM_ONLY_CONFIG (tentative) Released in Clixon 7.2"; } revision 2024-04-01 { @@ -1194,6 +1195,15 @@ module clixon-config { May not work together with CLICON_BACKEND_PRIVILEGES=drop and root, since new files need to be created in XMLDB_DIR"; } + leaf CLICON_XMLDB_SYSTEM_ONLY_CONFIG { + type boolean; + default true; + description + "If set, some fields in the configuration tree are not stored to datastore. + Instead, the application must provide a mechanism to save the system-only-config + in the system via commit/system-only-config callbacks. + See also extension system-only-config in clixon-lib.yang"; + } leaf CLICON_XML_CHANGELOG { type boolean; default false; diff --git a/yang/clixon/clixon-lib@2024-08-01.yang b/yang/clixon/clixon-lib@2024-08-01.yang index c97adc61..8c3cc4d7 100644 --- a/yang/clixon/clixon-lib@2024-08-01.yang +++ b/yang/clixon/clixon-lib@2024-08-01.yang @@ -72,7 +72,7 @@ module clixon-lib { revision 2024-08-01 { description "Added: list-pagination-partial-state - Added: system-only-config extension + Added: system-only-config extension (tentative) Released in Clixon 7.2"; } revision 2024-04-01 { @@ -356,8 +356,11 @@ module clixon-lib { description "This extension marks which fields in the configuration tree should not be saved to datastore and be removed from memory after commit. - Instead, the application provides a mechanism to save the system-only-config - in the system. + Instead, the application must provide a mechanism to save the system-only-config + in the system: + 1. Mark system-only config data in YANG with this extension + 2. Write a commit callback for data write + 2. Write a system-only-config callback for data read Note that the XML with these values will be remove from the datastore. The remaining XML still needs to be valid XML wrt YANG. An example of an invalid marking would be a list key. Because if the list keys are