experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

System-only config

Browse source 
New `ca_system_only` backend callback for reading system-only data
New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option
API: Added `system_only` parameter to clixon_xml2file1()
Cleared running on commit and inited candidate on startup with system-only data
Added callback code in main example
This commit is contained in:
Olof hagsand 2024-10-11 11:59:49 +02:00
parent aec0a5fc3f
commit 3a656fac07
15 changed files with 580 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

6
CHANGELOG.md
View file

@ -19,6 +19,12 @@ Expected: January 2025
* New: [feature request: support xpath functions for strings](https://github.com/clicon/clixon/issues/556)
* Added: re-match, substring, string, string-length, translate, substring-before, substring-after, starts-with
* Added support for system-only-config data
* A mechanism to not store sensitive data in the datastore, instead use application callbacks to store the data in system state.
* New `CLICON_XMLDB_SYSTEM_ONLY_CONFIG` configuration option
* New `system-only-config` extension
* New `ca_system_only` backend callback for reading system-only data
### Corrected Bugs
* Fixed: [string length validation doesn't work for the entry "" in case it has default value specified](https://github.com/clicon/clixon/issues/563)

5
apps/backend/backend_commit.c
View file

@ -720,6 +720,10 @@ candidate_commit(clixon_handle h,
*/
if (xmldb_copy(h, db, "running") < 0)
goto done;
/* Remove system-only-config data from destination */
if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){
xmldb_clear(h, "running");
}
xmldb_modified_set(h, db, 0); /* reset dirty bit */
/* Here pointers to old (source) tree are obsolete */
if (td->td_dvec){
@ -731,7 +735,6 @@ candidate_commit(clixon_handle h,
free(td->td_scvec);
td->td_scvec = NULL;
}
/* 9. Call plugin transaction end callbacks */
plugin_transaction_end_all(h, td);
retval = 1;

7
apps/backend/backend_main.c
View file

@ -1030,6 +1030,13 @@ main(int argc,
/* Initiate the shared candidate. */
if (xmldb_copy(h, "running", "candidate") < 0)
goto done;
/* Add system-only config to candidate */
if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")){
cxobj *x;
if ((x = xmldb_cache_get(h, "candidate")) != NULL)
if (xmldb_system_only_config(h, "/", NULL, &x) < 0)
goto done;
}
if (xmldb_modified_set(h, "candidate", 0) <0)
goto done;

172
example/main/example_backend.c
View file

@ -42,11 +42,14 @@
* -r enable the reset function
* -s enable the state function
* -S <file> read state data from file, otherwise construct it programmatically (requires -s)
* -o <xpath> System-only-config of xpath saved in mem
* -O <file> read/write system-only-config to/from this file
* -i read state file on init not by request for optimization (requires -sS <file>)
* -u enable upgrade function - auto-upgrade testing
* -U general-purpose upgrade
* -t enable transaction logging (call syslog for every transaction)
* -V <xpath> Failing validate and commit if <xpath> is present (synthetic error)
*/
#include <stdio.h>
#include <stdlib.h>
@ -70,7 +73,7 @@
#include <clixon/clixon_backend.h>
/* Command line options to be passed to getopt(3) */
#define BACKEND_EXAMPLE_OPTS "a:m:M:n:rsS:x:iuUtV:"
#define BACKEND_EXAMPLE_OPTS "a:m:M:n:o:O:rsS:x:iuUtV:"
/* Enabling this improves performance in tests, but there may trigger the "double XPath"
* problem.
@ -103,6 +106,20 @@ static char *_mount_namespace = NULL;
*/
static int _notification_stream_s = 0;
/*! System-only config xpath
*
* Start backend with -o <xpath>
* Combined with -O <file> to write to file
*/
static char *_system_only_xpath = NULL;
/*! System-only config file
*
* Start backend with -O <file>
* Combined with -o <file> to write to file
*/
static char *_system_only_file = NULL;
/*! Variable to control if reset code is run.
*
* The reset code inserts "extra XML" which assumes ietf-interfaces is
@ -185,6 +202,7 @@ static int _validate_fail_toggle = 0; /* fail at validate and commit */
/* forward */
static int example_stream_timer_setup(clixon_handle h, int sec);
static int main_system_only_commit(clixon_handle h, transaction_data td);
int
main_begin(clixon_handle h,
@ -224,17 +242,29 @@ main_complete(clixon_handle h,
}
/*! This is called on commit. Identify modifications and adjust machine state
*
* Somewhat complex due to the different test-cases
* @param[in] h Clixon handle
* @param[in] td Transaction data
* @retval 0 OK
* @retval -1 Error
*/
int
main_commit(clixon_handle h,
transaction_data td)
{
int retval = -1;
cxobj *target = transaction_target(td); /* wanted XML tree */
cxobj **vec = NULL;
int i;
size_t len;
cvec *nsc = NULL;
if (_system_only_xpath != NULL){
if (main_system_only_commit(h, td) < 0)
goto done;
goto ok;
}
if (_transaction_log)
transaction_log(h, td, LOG_NOTICE, __FUNCTION__);
if (_validate_fail_xpath){
@ -242,7 +272,7 @@ main_commit(clixon_handle h,
xpath_first(transaction_target(td), NULL, "%s", _validate_fail_xpath)){
_validate_fail_toggle = 0; /* toggle if triggered */
clixon_err(OE_XML, 0, "User error");
return -1; /* induce fail */
goto done; /* simulate fail */
}
}
@ -256,12 +286,14 @@ main_commit(clixon_handle h,
if (clixon_debug_get())
for (i=0; i<len; i++) /* Loop over added i/fs */
xml_print(stdout, vec[i]); /* Print the added interface */
done:
ok:
retval = 0;
done:
if (nsc)
xml_nsctx_free(nsc);
if (vec)
free(vec);
return 0;
return retval;
}
int
@ -637,6 +669,131 @@ example_statefile(clixon_handle h,
return retval;
}
/*! System-only config commit data
*
* @param[in] h Clixon handle
* @param[in] td Transaction data
* @retval 0 OK
* @retval -1 Error
*
* System-only config data as defined by _ is not written to datastore.
* Instead, in this ocmmit action, it is written to file _state_file
* @see main_system_only_commit callback for reading data
* @note Only single system-only config data supported
*/
static int
main_system_only_commit(clixon_handle h,
transaction_data td)
{
int retval = -1;
cxobj *src;
cxobj *target;
cvec *nsc = NULL;
cxobj **vec0 = NULL;
size_t veclen0;
cxobj **vec1 = NULL;
cxobj *x0t;
cxobj *x1t = NULL;
size_t veclen1;
cxobj *x;
int i;
char *xpath;
char *file;
FILE *fp = NULL;
clixon_debug(CLIXON_DBG_DEFAULT, "");
xpath = _system_only_xpath;
file = _system_only_file;
if (xpath == NULL || file == NULL){
clixon_err(OE_PLUGIN, EINVAL, "Both -o and -O must be given system-only config");
goto done;
}
src = transaction_src(td); /* existing XML tree */
target = transaction_target(td); /* wanted XML tree */
if (xpath_vec_flag(target, nsc, "%s", XML_FLAG_ADD | XML_FLAG_CHANGE,
&vec0, &veclen0, xpath) < 0)
goto done;
for (i=0; i<veclen0; i++){
x = vec0[i];
if (fp == NULL &&
(fp = fopen(file, "w")) == NULL){
clixon_err(OE_UNIX, errno, "open(%s)", file);
goto done;
}
xml_flag_set(x, XML_FLAG_MARK);
x0t = xml_root(x);
if ((x1t = xml_new(xml_name(x0t), NULL, CX_ELMNT)) == NULL)
goto done;
xml_apply_ancestor(x, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
if (xml_copy_marked(x0t, x1t) < 0) /* config */
goto done;
if (xml_apply(x0t, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
goto done;
if (clixon_xml2file(fp, x1t, 0, 1, NULL, fprintf, 1, 0) < 0)
goto done;
xml_flag_reset(x, XML_FLAG_MARK);
break; // XXX only single data
}
if (xpath_vec_flag(src, nsc, "%s", XML_FLAG_DEL,
&vec1, &veclen1, xpath) < 0)
goto done;
for (i=0; i<veclen1; i++){
x = vec1[i];
if (fp == NULL &&
(fp = fopen(file, "w")) == NULL){
clixon_err(OE_UNIX, errno, "open(%s)", file);
goto done;
}
break; // XXX only single data
}
retval = 0;
done:
if (fp)
fclose(fp);
if (vec0)
free(vec0);
if (vec1)
free(vec1);
if (x1t)
xml_free(x1t);
return retval;
}
/*! Called to get system-only config data
*
* @param[in] h Clixon handle
* @param[in] nsc External XML namespace context, or NULL
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[out] xstate XML tree, <config/> on entry.
* @retval 0 OK
* @retval -1 Error
* @see example_statedata
* @see main_system_only_commit where data is written
*/
int
main_system_only_callback(clixon_handle h,
cvec *nsc,
char *xpath,
cxobj *xconfig)
{
int retval = -1;
char *file;
FILE *fp = NULL;
if ((file = _system_only_file) == NULL)
goto ok;
if ((fp = fopen(file, "r")) == NULL)
goto ok;
if (clixon_xml_parse_file(fp, YB_NONE, NULL, &xconfig, NULL) < 0)
goto done;
ok:
retval = 0;
done:
if (fp)
fclose(fp);
return retval;
}
/*! Example of state pagination callback and how to use pagination_data
*
* @param[in] h Generic handler
@ -1411,6 +1568,7 @@ static clixon_plugin_api api = {
.ca_daemon=example_daemon, /* daemon */
.ca_reset=example_reset, /* reset */
.ca_statedata=example_statedata, /* statedata : Note fn is switched if -sS <file> */
.ca_system_only=main_system_only_callback, /* System-only-config callback */
.ca_lockdb=example_lockdb, /* Database lock changed state */
.ca_trans_begin=main_begin, /* trans begin */
.ca_trans_validate=main_validate, /* trans validate */
@ -1461,6 +1619,12 @@ clixon_plugin_init(clixon_handle h)
case 'n':
_notification_stream_s = atoi(optarg);
break;
case 'o':
_system_only_xpath = optarg;
break;
case 'O':
_system_only_file = optarg;
break;
case 'r':
_reset = 1;
break;

1
lib/clixon/clixon_datastore.h
View file

@ -104,5 +104,6 @@ int xmldb_print(clixon_handle h, FILE *f);
int xmldb_rename(clixon_handle h, const char *db, const char *newdb, const char *suffix);
int xmldb_populate(clixon_handle h, const char *db);
int xmldb_multi_upgrade(clixon_handle h, const char *db);
int xmldb_system_only_config(clixon_handle h, const char *xpath, cvec *nsc, cxobj **xret);
#endif /* _CLIXON_DATASTORE_H */

24
lib/clixon/clixon_plugin.h
View file

@ -214,12 +214,12 @@ typedef int (plgreset_t)(clixon_handle h, const char *db);
* XXX: This callback may be replaced with a "dispatcher" type API in the future where the
* XPath binding is stricter, similar to the pagination API.
*
* @param[in] h Clixon handle
* @param[in] xpath Part of state requested
* @param[in] nsc XPath namespace context.
* @param[out] xtop XML tree where statedata is added
* @retval 0 OK
* @retval -1 Fatal error
* @param[in] h Clixon handle
* @param[in] xpath Part of state requested
* @param[in] nsc XPath namespace context.
* @param[out] xtop XML tree where data is added
* @retval 0 OK
* @retval -1 Fatal error
*
* @note The system will make an xpath check and filter out non-matching trees
* @note The system does not validate the xml, unless CLICON_VALIDATE_STATE_XML is set
@ -251,7 +251,13 @@ typedef int (plglockdb_t)(clixon_handle h, char *db, int lock, int id);
*/
typedef void *transaction_data;
/* Transaction callback */
/* Transaction callback
*
* @param[in] h Clixon handle
* @param[in] td Transaction data
* @retval 0 OK
* @retval -1 Error
*/
typedef int (trans_cb_t)(clixon_handle h, transaction_data td);
/*! Hook to override default prompt with explicit function
@ -396,6 +402,7 @@ struct clixon_plugin_api{
plgdaemon_t *cb_daemon; /* Plugin daemonized (always called) */
plgreset_t *cb_reset; /* Reset system status */
plgstatedata_t *cb_statedata; /* Provide state data XML from plugin */
plgstatedata_t *cb_system_only; /* Provide system-only config XML from plugin */
plglockdb_t *cb_lockdb; /* Database lock changed state */
trans_cb_t *cb_trans_begin; /* Transaction start */
trans_cb_t *cb_trans_validate; /* Transaction validation */
@ -419,6 +426,7 @@ struct clixon_plugin_api{
#define ca_daemon u.cau_backend.cb_daemon
#define ca_reset u.cau_backend.cb_reset
#define ca_statedata u.cau_backend.cb_statedata
#define ca_system_only u.cau_backend.cb_system_only
#define ca_lockdb u.cau_backend.cb_lockdb
#define ca_trans_begin u.cau_backend.cb_trans_begin
#define ca_trans_validate u.cau_backend.cb_trans_validate
@ -508,6 +516,8 @@ int clixon_plugin_datastore_upgrade_all(clixon_handle h, const char *db, cxobj *
int clixon_plugin_yang_mount_one(clixon_plugin_t *cp, clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib);
int clixon_plugin_yang_mount_all(clixon_handle h, cxobj *xt, int *config, validate_level *vl, cxobj **yanglib);
int clixon_plugin_system_only_all(clixon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath, cxobj **xtop);
int clixon_plugin_yang_patch_one(clixon_plugin_t *cp, clixon_handle h, yang_stmt *ymod);
int clixon_plugin_yang_patch_all(clixon_handle h, yang_stmt *ymod);

2
lib/clixon/clixon_xml_io.h
View file

@ -45,7 +45,7 @@
*/
int clixon_xml2file1(FILE *f, cxobj *xn, int level, int pretty, char *prefix,
clicon_output_cb *fn, int skiptop, int autocliext, withdefaults_type wdef,
int multi);
int multi, int system_only);
int clixon_xml2file(FILE *f, cxobj *xn, int level, int pretty, char *prefix, clicon_output_cb *fn, int skiptop, int autocliext);
int clixon_xml2file_multi(clixon_handle h, const char *db, cxobj *xn, int level, int pretty,
char *prefix, clicon_output_cb *fn, int skiptop, int autocliext,

37
lib/src/clixon_datastore.c
View file

@ -1021,3 +1021,40 @@ xmldb_multi_upgrade(clixon_handle h,
free(tofile);
return retval;
}
/*! Get system-only config data by calling user callback
*
* @param[in] h Clixon handle
* @param[in] xpath XPath selection, may be used to filter early
* @param[in] nsc XML Namespace context for xpath
* @param[in,out] xret Existing XML tree, merge x into this, or rpc-error
* @retval 1 OK
* @retval 0 Statedata callback failed (error in xret)
* @retval -1 Error (fatal)
*/
int
xmldb_system_only_config(clixon_handle h,
const char *xpath,
cvec *nsc,
cxobj **xret)
{
int retval = -1;
yang_stmt *yspec;
int ret;
clixon_debug(CLIXON_DBG_BACKEND, "");
if ((yspec = clicon_dbspec_yang(h)) == NULL){
clixon_err(OE_YANG, ENOENT, "No yang spec");
goto done;
}
if ((ret = clixon_plugin_system_only_all(h, yspec, nsc, (char*)xpath, xret)) < 0)
goto done;
if (ret == 0)
goto fail;
retval = 1; /* OK */
done:
return retval;
fail:
retval = 0;
goto done;
}

5
lib/src/clixon_datastore_read.c
View file

@ -886,6 +886,11 @@ xmldb_get_cache(clixon_handle h,
if (xml_apply(x1t, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
goto done;
}
if (clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG") &&
strcmp(db, "running") == 0){
if (xmldb_system_only_config(h, xpath?xpath:"/", nsc, &x1t) < 0)
goto done;
}
/* If empty NACM config, then disable NACM if loaded
*/
if (clicon_option_bool(h, "CLICON_NACM_DISABLED_ON_EMPTY")){

6
lib/src/clixon_datastore_write.c
View file

@ -1560,7 +1560,7 @@ xmldb_multi_write_applyfn(cxobj *x,
goto done;
}
/* Dont recurse multi-file yet */
if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0) < 0)
if (clixon_xml2file1(fsub, x, 0, mw->mw_pretty, NULL, fprintf, 1, 0, mw->mw_wdef, 0, 0) < 0)
goto done;
}
retval = 2; /* Locally abort */
@ -1627,7 +1627,9 @@ xmldb_dump(clixon_handle h,
}
switch (format){
case FORMAT_XML:
if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi) < 0)
if (clixon_xml2file1(f, xt, 0, pretty, NULL, fprintf, 0, 0, wdef, multi,
clicon_option_bool(h, "CLICON_XMLDB_SYSTEM_ONLY_CONFIG")
) < 0)
goto done;
if (multi){
mw.mw_h = h;

165
lib/src/clixon_plugin.c
View file

@ -68,6 +68,11 @@
#include "clixon_options.h"
#include "clixon_xml_nsctx.h"
#include "clixon_xml_map.h"
#include "clixon_xml_bind.h"
#include "clixon_xml_sort.h"
#include "clixon_xml_default.h"
#include "clixon_xpath_ctx.h"
#include "clixon_xpath.h"
#include "clixon_yang_module.h"
#include "clixon_netconf_lib.h"
#include "clixon_validate.h"
@ -927,6 +932,166 @@ clixon_plugin_yang_mount_all(clixon_handle h,
return retval;
}
/*! Call single backend statedata callback
*
* Create an xml tree (xret) for one callback only on the form:
* <config>...</config>,
* call a user supplied function (ca_system_only) which can do two things:
* - Fill in config XML in the tree and return 0
* - Call cli_error() and return -1
* In the former case, this function returns the config XML tree to the caller (which
* typically merges the tree with other config trees).
* In the latter error case, this function returns 0 (invalid) to the caller with no tree
* If a fatal error occurs in this function, -1 is returned.
*
* @param[in] cp Plugin handle
* @param[in] h clicon handle
* @param[in] nsc namespace context for xpath
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[out] xp If retval=1, tree created and returned: <config>...
* @retval 1 OK if callback found (and called) xret is set
* @retval 0 Callback failed. no XML tree returned
* @retval -1 Fatal error
*/
static int
clixon_plugin_system_only_one(clixon_plugin_t *cp,
clixon_handle h,
cvec *nsc,
char *xpath,
cxobj **xp)
{
int retval = -1;
plgstatedata_t *fn; /* Plugin statedata fn */
cxobj *x = NULL;
void *wh = NULL;
if ((fn = clixon_plugin_api_get(cp)->ca_system_only) != NULL){
if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
goto done;
wh = NULL;
if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (fn(h, nsc, xpath, x) < 0){
if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
if (clixon_err_category() < 0)
clixon_log(h, LOG_WARNING, "%s: Internal error: Stateonly callback in plugin: %s returned -1 but did not make a clixon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
goto fail; /* Dont quit here on user callbacks */
}
if (clixon_resource_check(h, &wh, clixon_plugin_name_get(cp), __FUNCTION__) < 0)
goto done;
}
if (xp && x){
*xp = x;
x = NULL;
}
retval = 1;
done:
if (x)
xml_free(x);
return retval;
fail:
retval = 0;
goto done;
}
/*! Go through all backend system-only callbacks and collect config data
*
* @param[in] h clicon handle
* @param[in] yspec Yang spec
* @param[in] nsc Namespace context
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in,out] xret Config XML tree is merged with existing tree.
* @retval 1 OK
* @retval 0 Callback failed (xret set with netconf-error)
* @retval -1 Error
* @note xret can be replaced in this function
*/
int
clixon_plugin_system_only_all(clixon_handle h,
yang_stmt *yspec,
cvec *nsc,
char *xpath,
cxobj **xret)
{
int retval = -1;
int ret;
cxobj *x = NULL;
clixon_plugin_t *cp = NULL;
cbuf *cberr = NULL;
cxobj *xerr = NULL;
clixon_debug(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, "");
while ((cp = clixon_plugin_each(h, cp)) != NULL) {
if ((ret = clixon_plugin_system_only_one(cp, h, nsc, xpath, &x)) < 0)
goto done;
if (ret == 0){
if ((cberr = cbuf_new()) == NULL){
clixon_err(OE_UNIX, errno, "cbuf_new");
goto done;
}
/* error reason should be in clixon_err_reason */
cprintf(cberr, "Internal error, system-only callback in plugin %s returned invalid XML: %s",
clixon_plugin_name_get(cp), clixon_err_reason());
if (netconf_operation_failed_xml(&xerr, "application", cbuf_get(cberr)) < 0)
goto done;
xml_free(*xret);
*xret = xerr;
xerr = NULL;
goto fail;
}
if (x == NULL)
continue;
if (xml_child_nr(x) == 0){
xml_free(x);
x = NULL;
continue;
}
clixon_debug_xml(CLIXON_DBG_BACKEND | CLIXON_DBG_DETAIL, x, "%s SYSTEM-ONLY:", clixon_plugin_name_get(cp));
/* XXX: ret == 0 invalid yang binding should be handled as internal error */
if ((ret = xml_bind_yang(h, x, YB_MODULE, yspec, &xerr)) < 0)
goto done;
if (ret == 0){
if (clixon_netconf_internal_error(xerr,
". Internal error, system-only callback returned invalid XML from plugin: ",
clixon_plugin_name_get(cp)) < 0)
goto done;
xml_free(*xret);
*xret = xerr;
xerr = NULL;
goto fail;
}
if (xml_sort_recurse(x) < 0)
goto done;
/* Remove global defaults and empty non-presence containers */
if (xml_default_nopresence(x, 2, 0) < 0)
goto done;
if (xpath_first(x, nsc, "%s", xpath) != NULL){
if ((ret = netconf_trymerge(x, yspec, xret)) < 0)
goto done;
if (ret == 0)
goto fail;
}
if (x){
xml_free(x);
x = NULL;
}
} /* while plugin */
retval = 1;
done:
if (xerr)
xml_free(xerr);
if (cberr)
cbuf_free(cberr);
if (x)
xml_free(x);
return retval;
fail:
retval = 0;
goto done;
}
/*! Call plugin YANG schema patch
*
* @param[in] cp Plugin handle

75
lib/src/clixon_xml_io.c
View file

@ -203,6 +203,7 @@ xml2output_wdef(cxobj *x,
* @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow
* @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
* @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI
* @param[in] system_only Enable checks for system-only-config extension
* @retval 0 OK
* @retval -1 Error
* One can use clixon_xml2cbuf to get common code, but using fprintf is
@ -223,30 +224,32 @@ xml2file_recurse(FILE *f,
clicon_output_cb *fn,
int autocliext,
withdefaults_type wdef,
int multi)
int multi,
int system_only)
{
int retval = -1;
char *name;
char *namespace;
cxobj *xc;
int hasbody;
int haselement;
char *val;
char *encstr = NULL; /* xml encoded string */
int exist;
yang_stmt *y;
int level1;
int tag = 0;
int ret;
int subfile = 0; /* File is split into subfile */
char *xpath = NULL;
char *hexstr = NULL;
int retval = -1;
char *name;
char *namespace;
cxobj *xc;
int hasbody;
int haselement;
char *val;
char *encstr = NULL; /* xml encoded string */
int exist;
yang_stmt *y;
int level1;
int tag = 0;
int subfile = 0; /* File is split into subfile */
char *xpath = NULL;
char *hexstr = NULL;
int ret;
if (x == NULL)
goto ok;
y = xml_spec(x);
/* Check if system-only, then do not write to datastore */
if (y != NULL) {
/* Check if system-only, then do not write to datastore
*/
if (y != NULL && system_only){
exist = 0;
if (yang_extension_value(y, "system-only-config", CLIXON_LIB_NS, &exist, NULL) < 0)
goto done;
@ -302,7 +305,7 @@ xml2file_recurse(FILE *f,
while ((xc = xml_child_each(x, xc, -1)) != NULL) {
switch (xml_type(xc)){
case CX_ATTR:
if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi) < 0)
if (xml2file_recurse(f, xc, level+1, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
goto done;
break;
case CX_BODY:
@ -360,7 +363,7 @@ xml2file_recurse(FILE *f,
}
if (xml_type(xc) != CX_ATTR && !subfile)
if (xml2file_recurse(f, xc, level+1, pretty, prefix,
fn, autocliext, wdef, multi) <0)
fn, autocliext, wdef, multi, system_only) <0)
goto done;
if (xa){
if (xml_purge(xa) < 0)
@ -412,6 +415,7 @@ xml2file_recurse(FILE *f,
* @param[in] autocliext How to handle autocli extensions: 0: ignore 1: follow
* @param[in] wdef With-defaults parameter, default is WITHDEFAULTS_REPORT_ALL
* @param[in] multi Multi-file split datastore, see CLICON_XMLDB_MULTI
* @param[in] system_only Enable checks for system-only-config extension
* @retval 0 OK
* @retval -1 Error
* @see clixon_xml2cbuf print to a cbuf string
@ -428,7 +432,8 @@ clixon_xml2file1(FILE *f,
int skiptop,
int autocliext,
withdefaults_type wdef,
int multi)
int multi,
int system_only)
{
int retval = 1;
cxobj *xc;
@ -438,11 +443,11 @@ clixon_xml2file1(FILE *f,
if (skiptop){
xc = NULL;
while ((xc = xml_child_each(xn, xc, CX_ELMNT)) != NULL)
if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
if (xml2file_recurse(f, xc, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
goto done;
}
else {
if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi) < 0)
if (xml2file_recurse(f, xn, level, pretty, prefix, fn, autocliext, wdef, multi, system_only) < 0)
goto done;
}
retval = 0;
@ -478,7 +483,7 @@ clixon_xml2file(FILE *f,
int skiptop,
int autocliext)
{
return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0);
return clixon_xml2file1(f, xn, level, pretty, prefix, fn, skiptop, autocliext, 0, 0, 0);
}
/*! Print an XML tree structure to an output stream
@ -493,7 +498,7 @@ int
xml_print(FILE *f,
cxobj *x)
{
return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0);
return xml2file_recurse(f, x, 0, 1, NULL, fprintf, 0, WITHDEFAULTS_REPORT_ALL, 0, 0);
}
/*! Dump cxobj structure with pointers and flags for debugging, internal function
@ -541,7 +546,7 @@ xml_dump(FILE *f,
return xml_dump1(f, x, 0);
}
/*! Internal: print XML tree structure to a cligen buffer and encode chars "<>&"
/*! Internal: print XML tree structure to a cligen buffer and encode chars "<>&"
*
* @param[in,out] cb Cligen buffer to write to
* @param[in] xn Clixon xml tree
@ -724,14 +729,14 @@ xml2cbuf_recurse(cbuf *cb,
* @see clixon_xml2file to file, which is faster
*/
int
clixon_xml2cbuf1(cbuf *cb,
cxobj *xn,
int level,
int pretty,
char *prefix,
int32_t depth,
int skiptop,
withdefaults_type wdef)
clixon_xml2cbuf1(cbuf *cb,
cxobj *xn,
int level,
int pretty,
char *prefix,
int32_t depth,
int skiptop,
withdefaults_type wdef)
{
int retval = -1;
cxobj *xc;

134
test/test_datastore_system_only.sh
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
# Datastore system only config test
# see https://github.com/clicon/clixon/pull/534 and extension system-only-config
# Test uses a "standard" yang and a "local" yang which augmanets the standard
# Test uses a "standard" yang and a "local" yang which augments the standard
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
@ -21,9 +21,6 @@ test -d $CFD || mkdir -p $CFD
AUTOCLI=$(autocli_config clixon-\* kw-nokey false)
# Well-known digest of mount-point xpath
subfilename=9121a04a6f67ca5ac2184286236d42f3b7301e97.xml
cat <<EOF > $cfg
<clixon-config xmlns="http://clicon.org/config">
<CLICON_CONFIGFILE>$cfg</CLICON_CONFIGFILE>
@ -39,6 +36,7 @@ cat <<EOF > $cfg
<CLICON_BACKEND_DIR>/usr/local/lib/$APPNAME/backend</CLICON_BACKEND_DIR>
<CLICON_BACKEND_PIDFILE>/usr/local/var/run/$APPNAME.pidfile</CLICON_BACKEND_PIDFILE>
<CLICON_XMLDB_DIR>$dir</CLICON_XMLDB_DIR>
<CLICON_XMLDB_SYSTEM_ONLY_CONFIG>true</CLICON_XMLDB_SYSTEM_ONLY_CONFIG>
<CLICON_NETCONF_MONITORING>true</CLICON_NETCONF_MONITORING>
<CLICON_VALIDATE_STATE_XML>true</CLICON_VALIDATE_STATE_XML>
<CLICON_STREAM_DISCOVERY_RFC5277>true</CLICON_STREAM_DISCOVERY_RFC5277>
@ -142,16 +140,8 @@ show("Show a particular state of the system"){
}
EOF
# Check content of db
# Args:
# 0: dbname
function check_db()
{
dbname=$1
sudo chmod 755 $dir/${dbname}_db
sudo rm -f $dir/x_db
cat <<EOF > $dir/x_db
# Two reference files: What is expected in the datastore
cat <<EOF > $dir/x_db
<config>
<store xmlns="urn:example:std">
<keys>
@ -162,13 +152,49 @@ function check_db()
</store>
</config>
EOF
new "Check ${dbname}_db"
# ret=$(diff $dir/x_db $dir/${dbname}_db)
# What is expected in the system-only-config file (simulated system)
cat <<EOF > $dir/y_db
<store xmlns="urn:example:std">
<keys>
<key>
<name>a</name>
<system-only-data>mydata</system-only-data>
</key>
</keys>
</store>
EOF
# Check content of db
# Args:
# 0: dbname
# 1: system true/false check in system or not (only after commit)
function check_db()
{
dbname=$1
system=$2
sudo chmod 755 $dir/${dbname}_db
new "Check not in ${dbname}_db"
ret=$(diff $dir/x_db $dir/${dbname}_db)
if [ $? -ne 0 ]; then
# err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
err "$(cat $dir/x_db)" "$(cat $dir/${dbname}_db)"
fi
if $system; then
new "Check $dir/system-only.xml"
ret=$(diff $dir/y_db $dir/system-only.xml)
if [ $? -ne 0 ]; then
err "$(cat $dir/y_db)" "$(cat $dir/system-only.xml)"
fi
else
new "Check no $dir/system-only.xml"
if [ -s $dir/system-only.xml ]; then
err "No file" "$(cat $dir/system-only.xml)"
fi
fi
}
new "test params: -f $cfg"
@ -179,31 +205,89 @@ if [ $BE -ne 0 ]; then
if [ $? -ne 0 ]; then
err
fi
new "start backend -s init -f $cfg"
start_backend -s init -f $cfg
new "start backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml"
start_backend -s init -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml
fi
new "wait backend"
new "wait backend 1"
wait_backend
sudo rm -f $dir/system-only.xml
new "Add mydata"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></config></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check mydata not in candidate"
check_db candidate
new "Check mydata present, but not in candidate datastore"
check_db candidate false
new "Get mydata from candidate"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><candidate/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
new "Commit"
new "Commit 1"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check mydata not in running"
check_db running
new "Check mydata present, but not in running datastore"
check_db running true
new "Get mydata from running"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
new "Remove mydata"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data nc:operation=\"delete\" xmlns:nc=\"${BASENS}\">mydata</system-only-data></key></keys></store></config><default-operation>none</default-operation></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Check mydata present, but not in candidate datastore"
check_db candidate true
new "Commit 2"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Get mydata from running, expected not"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name></key></keys></store></data></rpc-reply>"
new "Check mydata not present, but not in running datastore"
check_db running false
new "Add mydata again"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></config></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Commit 3"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Restart"
if [ $BE -ne 0 ]; then
new "kill old backend"
sudo clixon_backend -zf $cfg
if [ $? -ne 0 ]; then
err
fi
fi
if [ $BE -ne 0 ]; then
new "start backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml"
start_backend -s running -f $cfg -- -o store/keys/key/system-only-data -O $dir/system-only.xml
fi
new "wait backend 2"
wait_backend
new "Check mydata present, but not in running datastore"
check_db running true
new "Get mydata from running"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data>mydata</system-only-data></key></keys></store></data></rpc-reply>"
new "Remove mydata"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><edit-config><target><candidate/></target><config><store xmlns=\"urn:example:std\"><keys><key><name>a</name><system-only-data nc:operation=\"delete\" xmlns:nc=\"${BASENS}\">mydata</system-only-data></key></keys></store></config><default-operation>none</default-operation></edit-config></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Commit 4"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><commit/></rpc>" "<rpc-reply $DEFAULTNS><ok/></rpc-reply>"
new "Get mydata from running, expected not"
expecteof_netconf "$clixon_netconf -qf $cfg" 0 "$DEFAULTHELLO" "<rpc $DEFAULTNS><get-config><source><running/></source></get-config></rpc>" "<rpc-reply $DEFAULTNS><data><store xmlns=\"urn:example:std\"><keys><key><name>a</name></key></keys></store></data></rpc-reply>"
new "Check mydata not present, but not in running datastore"
check_db running false
if [ $BE -ne 0 ]; then
new "Kill backend"
# Check if premature kill

10
yang/clixon/clixon-config@2024-08-01.yang
View file

@ -54,6 +54,7 @@ module clixon-config {
"Added options:
CLICON_YANG_DOMAIN_DIR
CLICON_YANG_USE_ORIGINAL
CLICON_XMLDB_SYSTEM_ONLY_CONFIG (tentative)
Released in Clixon 7.2";
}
revision 2024-04-01 {
@ -1194,6 +1195,15 @@ module clixon-config {
May not work together with CLICON_BACKEND_PRIVILEGES=drop and root, since
new files need to be created in XMLDB_DIR";
}
leaf CLICON_XMLDB_SYSTEM_ONLY_CONFIG {
type boolean;
default true;
description
"If set, some fields in the configuration tree are not stored to datastore.
Instead, the application must provide a mechanism to save the system-only-config
in the system via commit/system-only-config callbacks.
See also extension system-only-config in clixon-lib.yang";
}
leaf CLICON_XML_CHANGELOG {
type boolean;
default false;

9
yang/clixon/clixon-lib@2024-08-01.yang
View file

@ -72,7 +72,7 @@ module clixon-lib {
revision 2024-08-01 {
description
"Added: list-pagination-partial-state
Added: system-only-config extension
Added: system-only-config extension (tentative)
Released in Clixon 7.2";
}
revision 2024-04-01 {
@ -356,8 +356,11 @@ module clixon-lib {
description
"This extension marks which fields in the configuration tree should not be
saved to datastore and be removed from memory after commit.
Instead, the application provides a mechanism to save the system-only-config
in the system.
Instead, the application must provide a mechanism to save the system-only-config
in the system:
1. Mark system-only config data in YANG with this extension
2. Write a commit callback for data write
2. Write a system-only-config callback for data read
Note that the XML with these values will be remove from the datastore. The remaining XML
still needs to be valid XML wrt YANG.
An example of an invalid marking would be a list key. Because if the list keys are