From caabfd464e202964551d20b20c89bb8e57c8c51a Mon Sep 17 00:00:00 2001 From: Olof hagsand Date: Sat, 4 Sep 2021 11:35:27 +0200 Subject: [PATCH] * New state callback signature (ca_statedata2) * The new callback contains parameters for paging * Goal is to replace ca_statedata callback * New plugin callback when lock/unlock occurs * Add `ca_lockdb` tro plugin init to use it. * Fixed: Typing 'q' in CLI more paging did not properly quit output * Output continued but was not shown, for a very large file this could cause considerable delay * Fixed: Lock was broken in first get get access * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access. --- CHANGELOG.md | 18 + apps/backend/backend_client.c | 60 ++- apps/backend/backend_get.c | 68 ++-- apps/backend/backend_plugin.c | 85 ++++- apps/backend/clixon_backend_plugin.h | 3 + apps/cli/cli_common.c | 2 + apps/cli/cli_show.c | 10 +- example/main/example_backend.c | 347 +++++++++++------- lib/clixon/clixon_data.h | 2 +- lib/clixon/clixon_plugin.h | 44 ++- lib/src/clixon_datastore.c | 17 +- lib/src/clixon_datastore_read.c | 4 + ...nation_config.sh => test_paging_config.sh} | 0 ...gination_draft.sh => test_paging_draft.sh} | 0 ...gination_state.sh => test_paging_state.sh} | 60 +-- 15 files changed, 487 insertions(+), 233 deletions(-) rename test/{test_pagination_config.sh => test_paging_config.sh} (100%) rename test/{test_pagination_draft.sh => test_paging_draft.sh} (100%) rename test/{test_pagination_state.sh => test_paging_state.sh} (64%) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6772a75..c7320e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,10 @@ Expected: September, 2021 * ietf-origin@2018-02-14.yang * ietf-yang-metadata@2016-08-05.yang * ietf-netconf-with-defaults@2011-06-01.yang + * New state callback signature (ca_statedata2) + * The new callback contains parameters for paging + * Goal is to replace ca_statedata callback + * YANG Leafref feature update * Closer adherence to RFC 7950. Some of this is changed behavior, some is new feature. * Essentially instead of looking at the referring leaf, context is referred(target) node @@ -66,6 +70,11 @@ Expected: September, 2021 * See draft-wwlh-netconf-list-pagination-00.txt * New http media: application/yang-collection+xml/json +* New state callback signature (ca_statedata2) + * The new callback contains parameters for paging + * Goal is to replace ca_statedata callback + + ### API changes on existing protocol/config features Users may have to change how they access the system @@ -82,6 +91,11 @@ Users may have to change how they access the system * Removed default of `CLICON_RESTCONF_INSTALLDIR` * The default behaviour is changed to use the config $(sbindir) to locate `clixon_restconf` when starting restconf internally +### C/CLI-API changes on existing features + +Developers may need to change their code + + ### Minor features * JSON errors are now labelled with JSON and not XML @@ -106,6 +120,10 @@ Users may have to change how they access the system * In this case, eg "uses", single quotes can now be used, but not `qstring + qstring` in this case * Fixed: [Performance issue when parsing large JSON param](https://github.com/clicon/clixon/issues/266) * Fixed: [Duplicate lines emitted by cli_show_config (cli output style) when yang list element has composite key](https://github.com/clicon/clixon/issues/258) +* Fixed: Typing 'q' in CLI more paging did not properly quit output + * Output continued but was not shown, for a very large file this could cause considerable delay +* Fixed: Lock was broken in first get get access + * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access. * Fixed: [JSON leaf-list output single element leaf-list does not use array](https://github.com/clicon/clixon/issues/261) * Fixed: Netconf diff callback did not work with choice and same value replace * Eg if YANG is `choice c { leaf x; leaf y }` and XML changed from `42` to `42` the datastrore changed, but was not detected by diff algorithms and provided to validate callbacks. diff --git a/apps/backend/backend_client.c b/apps/backend/backend_client.c index 45dd0c3a..f9882c76 100644 --- a/apps/backend/backend_client.c +++ b/apps/backend/backend_client.c @@ -122,6 +122,37 @@ ce_event_cb(clicon_handle h, return 0; } +/*! Unlock all db:s of a client and call user unlock calback + * @see xmldb_unlock_all unlocks, but does not call user callbacks which is a backend thing + */ +static int +release_all_dbs(clicon_handle h, + uint32_t id) +{ + int retval = -1; + char **keys = NULL; + size_t klen; + int i; + db_elmnt *de; + + /* get all db:s */ + if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0) + goto done; + /* Identify the ones locked by client id */ + for (i = 0; i < klen; i++) { + if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL && + de->de_id == id){ + de->de_id = 0; /* unlock */ + clicon_db_elmnt_set(h, keys[i], de); + if (clixon_plugin_lockdb_all(h, keys[i], 0, id) < 0) + goto done; + } + } + retval = 0; + done: + return retval; +} + /*! Remove client entry state * Close down everything wrt clients (eg sockets, subscriptions) * Finally actually remove client struct in handle @@ -148,7 +179,8 @@ backend_client_rm(clicon_handle h, clixon_event_unreg_fd(ce->ce_s, from_client); close(ce->ce_s); ce->ce_s = 0; - xmldb_unlock_all(h, ce->ce_id); + if (release_all_dbs(h, ce->ce_id) < 0) + return -1; } break; } @@ -606,6 +638,9 @@ from_client_lock(clicon_handle h, } if (xmldb_lock(h, db, id) < 0) goto done; + /* user callback */ + if (clixon_plugin_lockdb_all(h, db, 1, id) < 0) + goto done; cprintf(cbret, "", NETCONF_BASE_NAMESPACE); ok: retval = 0; @@ -677,6 +712,9 @@ from_client_unlock(clicon_handle h, } else{ xmldb_unlock(h, db); + /* user callback */ + if (clixon_plugin_lockdb_all(h, db, 0, id) < 0) + goto done; if (cprintf(cbret, "", NETCONF_BASE_NAMESPACE) < 0) goto done; } @@ -699,15 +737,16 @@ from_client_unlock(clicon_handle h, */ static int from_client_close_session(clicon_handle h, - cxobj *xe, - cbuf *cbret, - void *arg, - void *regarg) + cxobj *xe, + cbuf *cbret, + void *arg, + void *regarg) { struct client_entry *ce = (struct client_entry *)arg; uint32_t id = ce->ce_id; - xmldb_unlock_all(h, id); + if (release_all_dbs(h, id) < 0) + return -1; stream_ss_delete_all(h, ce_event_cb, (void*)ce); cprintf(cbret, "", NETCONF_BASE_NAMESPACE); return 0; @@ -751,11 +790,16 @@ from_client_kill_session(clicon_handle h, goto ok; /* may or may not be in active client list, probably not */ if ((ce = ce_find_byid(backend_client_list(h), id)) != NULL){ - xmldb_unlock_all(h, id); /* Removes locks on all databases */ + if (release_all_dbs(h, id) < 0) + goto done; backend_client_rm(h, ce); /* Removes client struct */ } - if (xmldb_islocked(h, db) == id) + if (xmldb_islocked(h, db) == id){ xmldb_unlock(h, db); + /* user callback */ + if (clixon_plugin_lockdb_all(h, db, 0, id) < 0) + goto done; + } cprintf(cbret, "", NETCONF_BASE_NAMESPACE); ok: retval = 0; diff --git a/apps/backend/backend_get.c b/apps/backend/backend_get.c index a13ab980..78346f61 100644 --- a/apps/backend/backend_get.c +++ b/apps/backend/backend_get.c @@ -69,6 +69,7 @@ #include "clixon_backend_handle.h" #include "clixon_backend_plugin.h" #include "clixon_backend_commit.h" +#include "backend_client.h" #include "backend_handle.h" #include "backend_get.h" @@ -260,7 +261,7 @@ get_client_statedata(clicon_handle h, goto fail; } /* Use plugin state callbacks */ - if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, 0, 0, xret)) < 0) + if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, PAGING_NONE, 0, 0, xret)) < 0) goto done; if (ret == 0) goto fail; @@ -313,6 +314,7 @@ element2value(clicon_handle h, * It is specialized enough to have its own function. Specifically, extra attributes as well * as the list-paginaiton API * @param[in] h Clicon handle + * @param[in] ce Client entry, for locking * @param[in] xe Request: * @param[in] content Get config/state/both * @param[in] db Database name @@ -326,16 +328,17 @@ element2value(clicon_handle h, * XXX Lots of this code (in particular at the end) is copy of get_common */ static int -get_list_pagination(clicon_handle h, - cxobj *xe, - netconf_content content, - char *db, - int32_t depth, - yang_stmt *yspec, - char *xpath, - cvec *nsc, - char *username, - cbuf *cbret +get_list_pagination(clicon_handle h, + struct client_entry *ce, + cxobj *xe, + netconf_content content, + char *db, + int32_t depth, + yang_stmt *yspec, + char *xpath, + cvec *nsc, + char *username, + cbuf *cbret ) { int retval = -1; @@ -359,6 +362,8 @@ get_list_pagination(clicon_handle h, int ret; int i; cxobj *xnacm = NULL; + uint32_t iddb; /* DBs lock, if any */ + enum paging_status pagingstatus; #ifdef REMAINING cxobj *xcache = NULL; uint32_t total = 0; @@ -495,11 +500,16 @@ get_list_pagination(clicon_handle h, break; }/* switch content */ if (!list_config){ - /* Look if registered, - * get all state list and filter using cbpath - */ + /* Check if running locked (by this session) */ + if ((iddb = xmldb_islocked(h, "running")) != 0 && + iddb == ce->ce_id) + pagingstatus = PAGING_LOCK; + else + pagingstatus = PAGING_STATELESS; /* Use plugin state callbacks */ - if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, offset, limit, &xret)) < 0) + if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, + pagingstatus, + offset, limit, &xret)) < 0) goto done; } /* Code complex to filter out anything that is outside of xpath @@ -603,6 +613,7 @@ get_list_pagination(clicon_handle h, /*! Common get/get-config code for retrieving configuration and state information. * * @param[in] h Clicon handle + * @param[in] ce Client entry, for locking * @param[in] xe Request: * @param[in] content Get config/state/both * @param[in] db Database name @@ -613,11 +624,12 @@ get_list_pagination(clicon_handle h, * @see from_client_get_config */ static int -get_common(clicon_handle h, - cxobj *xe, - netconf_content content, - char *db, - cbuf *cbret +get_common(clicon_handle h, + struct client_entry *ce, + cxobj *xe, + netconf_content content, + char *db, + cbuf *cbret ) { int retval = -1; @@ -696,7 +708,7 @@ get_common(clicon_handle h, * check config/state */ if (list_pagination){ - if (get_list_pagination(h, xe, content, db, + if (get_list_pagination(h, ce, xe, content, db, depth, yspec, xpath, nsc, username, cbret) < 0) goto done; @@ -901,14 +913,15 @@ from_client_get_config(clicon_handle h, void *arg, void *regarg) { - int retval = -1; - char *db; + int retval = -1; + char *db; + struct client_entry *ce = (struct client_entry *)arg; if ((db = netconf_db_find(xe, "source")) == NULL){ clicon_err(OE_XML, 0, "db not found"); goto done; } - retval = get_common(h, xe, CONTENT_CONFIG, db, cbret); + retval = get_common(h, ce, xe, CONTENT_CONFIG, db, cbret); done: return retval; } @@ -932,11 +945,12 @@ from_client_get(clicon_handle h, void *arg, void *regarg) { - netconf_content content = CONTENT_ALL; - char *attr; + netconf_content content = CONTENT_ALL; + char *attr; + struct client_entry *ce = (struct client_entry *)arg; /* Clixon extensions: content */ if ((attr = xml_find_value(xe, "content")) != NULL) content = netconf_content_str2int(attr); - return get_common(h, xe, content, "running", cbret); + return get_common(h, ce, xe, content, "running", cbret); } diff --git a/apps/backend/backend_plugin.c b/apps/backend/backend_plugin.c index 7ee0be58..8cbf20f4 100644 --- a/apps/backend/backend_plugin.c +++ b/apps/backend/backend_plugin.c @@ -239,6 +239,7 @@ clixon_plugin_daemon_all(clicon_handle h) * @param[in] h clicon handle * @param[in] nsc namespace context for xpath * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[in] pagingstatus List pagination status * @param[in] offset Offset, for list pagination * @param[in] limit Limit, for list pagination * @param[out] xp If retval=1, state tree created and returned: ... @@ -251,6 +252,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp, clicon_handle h, cvec *nsc, char *xpath, + enum paging_status pagingstatus, uint32_t offset, uint32_t limit, cxobj **xp) @@ -262,9 +264,9 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp, clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp)); if ((fn2 = clixon_plugin_api_get(cp)->ca_statedata2) != NULL){ - if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL) + if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL) goto done; - if (fn2(h, nsc, xpath, offset, limit, x) < 0){ + if (fn2(h, nsc, xpath, pagingstatus, offset, limit, x) < 0){ if (clicon_errno < 0) clicon_log(LOG_WARNING, "%s: Internal error: State callback in plugin: %s returned -1 but did not make a clicon_err call", __FUNCTION__, clixon_plugin_name_get(cp)); @@ -272,7 +274,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp, } } else if ((fn = clixon_plugin_api_get(cp)->ca_statedata) != NULL){ - if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL) + if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL) goto done; if (fn(h, nsc, xpath, x) < 0){ if (clicon_errno < 0) @@ -298,6 +300,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp, * @param[in] yspec Yang spec * @param[in] nsc Namespace context * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[in] pagination List pagination * @param[in] offset Offset, for list pagination * @param[in] limit Limit, for list pagination * @param[in,out] xret State XML tree is merged with existing tree. @@ -307,13 +310,14 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp, * @note xret can be replaced in this function */ int -clixon_plugin_statedata_all(clicon_handle h, - yang_stmt *yspec, - cvec *nsc, - char *xpath, - uint32_t offset, - uint32_t limit, - cxobj **xret) +clixon_plugin_statedata_all(clicon_handle h, + yang_stmt *yspec, + cvec *nsc, + char *xpath, + enum paging_status pagingstatus, + uint32_t offset, + uint32_t limit, + cxobj **xret) { int retval = -1; int ret; @@ -324,7 +328,8 @@ clixon_plugin_statedata_all(clicon_handle h, clicon_debug(1, "%s", __FUNCTION__); while ((cp = clixon_plugin_each(h, cp)) != NULL) { - if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, offset, limit, &x)) < 0) + if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, pagingstatus, + offset, limit, &x)) < 0) goto done; if (ret == 0){ if ((cberr = cbuf_new()) == NULL){ @@ -399,6 +404,64 @@ clixon_plugin_statedata_all(clicon_handle h, goto done; } +/*! Lock database status has changed status + * @param[in] cp Plugin handle + * @param[in] h Clixon handle + * @param[in] db Database name (eg "running") + * @param[in] lock Lock status: 0: unlocked, 1: locked + * @param[in] id Session id (of locker/unlocker) + * @retval -1 Fatal error + * @retval 0 OK + */ +static int +clixon_plugin_lockdb_one(clixon_plugin_t *cp, + clicon_handle h, + char *db, + int lock, + int id) +{ + int retval = -1; + plglockdb_t *fn; /* Plugin statedata fn */ + + clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp)); + if ((fn = clixon_plugin_api_get(cp)->ca_lockdb) != NULL){ + if (fn(h, db, lock, id) < 0) + goto done; + } + retval = 0; + done: + return retval; +} + +/*! Lock database status has changed status + * @param[in] h Clixon handle + * @param[in] db Database name (eg "running") + * @param[in] lock Lock status: 0: unlocked, 1: locked + * @param[in] id Session id (of locker/unlocker) + * @retval -1 Fatal error + * @retval 0 OK +*/ +int +clixon_plugin_lockdb_all(clicon_handle h, + char *db, + int lock, + int id + ) + +{ + int retval = -1; + clixon_plugin_t *cp = NULL; + + clicon_debug(1, "%s", __FUNCTION__); + while ((cp = clixon_plugin_each(h, cp)) != NULL) { + if (clixon_plugin_lockdb_one(cp, h, db, lock, id) < 0) + goto done; + } + retval = 0; + done: + return retval; +} + /*! Create and initialize a validate/commit transaction * @retval td New alloced transaction, * @retval NULL Error diff --git a/apps/backend/clixon_backend_plugin.h b/apps/backend/clixon_backend_plugin.h index a0c90320..1178f66d 100644 --- a/apps/backend/clixon_backend_plugin.h +++ b/apps/backend/clixon_backend_plugin.h @@ -76,7 +76,10 @@ int clixon_plugin_pre_daemon_all(clicon_handle h); int clixon_plugin_daemon_all(clicon_handle h); int clixon_plugin_statedata_all(clicon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath, + enum paging_status pagingstatus, uint32_t offset, uint32_t limit, cxobj **xtop); +int clixon_plugin_lockdb_all(clicon_handle h, char *db, int lock, int id); + transaction_data_t * transaction_new(void); int transaction_free(transaction_data_t *); diff --git a/apps/cli/cli_common.c b/apps/cli/cli_common.c index fc75c9a1..270e833b 100644 --- a/apps/cli/cli_common.c +++ b/apps/cli/cli_common.c @@ -1043,6 +1043,8 @@ cli_notification_cb(int s, default: break; } + if (cli_output_status() < 0) + break; } } retval = 0; diff --git a/apps/cli/cli_show.c b/apps/cli/cli_show.c index 40472166..8e65a160 100644 --- a/apps/cli/cli_show.c +++ b/apps/cli/cli_show.c @@ -950,6 +950,8 @@ cli_pagination(clicon_handle h, } if ((nsc = xml_nsctx_init(prefix, namespace)) == NULL) goto done; + if (clicon_rpc_lock(h, "running") < 0) + goto done; for (i = 0;; i++){ if (clicon_rpc_get_pageable_list(h, "running", xpath, nsc, CONTENT_ALL, @@ -984,7 +986,11 @@ cli_pagination(clicon_handle h, default: break; } + if (cli_output_status() < 0) + break; } + if (cli_output_status() < 0) + break; if (xlen != window) /* Break if fewer elements than requested */ break; if (xret){ @@ -995,7 +1001,9 @@ cli_pagination(clicon_handle h, free(xvec); xvec = NULL; } - } + } /* for */ + if (clicon_rpc_unlock(h, "running") < 0) + goto done; retval = 0; done: if (xvec) diff --git a/example/main/example_backend.c b/example/main/example_backend.c index 6282a64b..2d515262 100644 --- a/example/main/example_backend.c +++ b/example/main/example_backend.c @@ -91,8 +91,13 @@ static char *_state_file = NULL; * Primarily for testing * Start backend with -- -siS */ -static int _state_file_init = 0; -static cxobj *_state_xstate = NULL; +static int _state_file_cached = 0; + +/*! Cache control of read state file paging example, + * keep xml tree cache as long as db is locked + */ +static cxobj *_state_xml_cache = NULL; /* XML cache */ +static int _state_file_transaction = 0; /*! Variable to control module-specific upgrade callbacks. * If set, call test-case for upgrading ietf-interfaces, otherwise call @@ -343,13 +348,17 @@ example_copy_extra(clicon_handle h, /* Clicon handle */ return retval; } -/*! Called to get state data from plugin - * @param[in] h Clicon handle - * @param[in] nsc External XML namespace context, or NULL - * @param[in] xpath String with XPATH syntax. or NULL for all - * @param[in] xstate XML tree, on entry. - * @retval 0 OK - * @retval -1 Error +/*! Called to get state data from plugin by programmatically adding state + * + * @param[in] h Clicon handle + * @param[in] nsc External XML namespace context, or NULL + * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[in] paging List pagination (not uses here) + * @param[in] offset Offset, for list pagination + * @param[in] limit Limit, for list pagination + * @param[in] xstate XML tree, on entry. + * @retval 0 OK + * @retval -1 Error * @see xmldb_get * @note this example code returns requires this yang snippet: container state { @@ -360,11 +369,14 @@ example_copy_extra(clicon_handle h, /* Clicon handle */ } } * This yang snippet is present in clixon-example.yang for example. + * XXX paging for lock + * @see example_statefile where state is read from file and also paging */ int example_statedata(clicon_handle h, cvec *nsc, char *xpath, + enum paging_status paging, uint32_t offset, uint32_t limit, cxobj *xstate) @@ -377,127 +389,62 @@ example_statedata(clicon_handle h, cxobj *xt = NULL; char *name; cvec *nsc1 = NULL; - cvec *nsc2 = NULL; yang_stmt *yspec = NULL; - FILE *fp = NULL; - cxobj *x1; - uint32_t upper; if (!_state) goto ok; yspec = clicon_dbspec_yang(h); - - /* If -S is set, then read state data from file, otherwise construct it programmatically */ - if (_state_file){ - if (_state_file_init){ -#if 0 /* This is just for a zero-copy version (only works once) */ - { - cxobj *xx = NULL; - while (xml_child_nr(_state_xstate)){ - xx = xml_child_i(_state_xstate,0); - if (xml_addsub(xstate, xx) < 0) - goto done; - } - } -#else - if (xml_copy(_state_xstate, xstate) < 0) - goto done; -#endif - } - else{ - if ((fp = fopen(_state_file, "r")) == NULL){ - clicon_err(OE_UNIX, errno, "open(%s)", _state_file); - goto done; - } - if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL) - goto done; - /* Note, does not care about xpath / list-pagination */ - if (clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL) < 0) - goto done; - if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) - goto done; - if (limit == 0) - upper = xlen; - else{ - if ((upper = offset+limit)>xlen) - upper = xlen; - } - for (i=offset; i"); + for (i=0; i%sex:ethup", name); + cprintf(cb, "42foo"); + cprintf(cb, ""); } + cprintf(cb, ""); + if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0) + goto done; } - else { - /* Example of statedata, in this case merging state data with - * state information. In this case adding dummy interface operation state - * to configured interfaces. - * Get config according to xpath */ - if ((nsc1 = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:yang:ietf-interfaces")) == NULL) + /* State in test_yang.sh , test_restconf.sh and test_order.sh */ + if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){ + if (clixon_xml_parse_string("" + "42" + "41" + "43" /* should not be ordered */ + "", + YB_NONE, + NULL, &xstate, NULL) < 0) + goto done; /* For the case when urn:example:clixon is not loaded */ + } + /* Event state from RFC8040 Appendix B.3.1 + * Note: (1) order is by-system so is different, + * (2) event-count is XOR on name, so is not 42 and 4 + */ + if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){ + cbuf_reset(cb); + cprintf(cb, ""); + cprintf(cb, "interface-down90"); + cprintf(cb, "interface-up77"); + cprintf(cb, ""); + if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0) goto done; - if (xmldb_get0(h, "running", YB_MODULE, nsc1, "/interfaces/interface/name", 1, &xt, NULL, NULL) < 0) - goto done; - if (xpath_vec(xt, nsc1, "/interfaces/interface/name", &xvec, &xlen) < 0) - goto done; - if (xlen){ - cprintf(cb, ""); - for (i=0; i%sex:ethup", name); - cprintf(cb, "42foo"); - cprintf(cb, ""); - } - cprintf(cb, ""); - if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0) - goto done; - } - /* State in test_yang.sh , test_restconf.sh and test_order.sh */ - if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){ - if (clixon_xml_parse_string("" - "42" - "41" - "43" /* should not be ordered */ - "", - YB_NONE, - NULL, &xstate, NULL) < 0) - goto done; /* For the case when urn:example:clixon is not loaded */ - } - /* Event state from RFC8040 Appendix B.3.1 - * Note: (1) order is by-system so is different, - * (2) event-count is XOR on name, so is not 42 and 4 - */ - if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){ - cbuf_reset(cb); - cprintf(cb, ""); - cprintf(cb, "interface-down90"); - cprintf(cb, "interface-up77"); - cprintf(cb, ""); - if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0) - goto done; - } } ok: retval = 0; done: - if (fp) - fclose(fp); if (nsc1) xml_nsctx_free(nsc1); - if (nsc2) - xml_nsctx_free(nsc2); if (xt) xml_free(xt); if (cb) @@ -507,6 +454,157 @@ example_statedata(clicon_handle h, return retval; } +/*! Called to get state data from plugin by reading a file, also paging + * + * The example shows how to read and parse a state XML file, (which is cached in the -i case). + * Return the requested xpath / paging xstate by copying from the parsed state XML file + * @param[in] h Clicon handle + * @param[in] nsc External XML namespace context, or NULL + * @param[in] xpath String with XPATH syntax. or NULL for all + * @param[in] paging List pagination + * @param[in] offset Offset, for list pagination + * @param[in] limit Limit, for list pagination + * @param[in] xstate XML tree, on entry. Copy to this + * @retval 0 OK + * @retval -1 Error + * @see xmldb_get + * @see example_statefile where state is programmatically added + */ +int +example_statefile(clicon_handle h, + cvec *nsc, + char *xpath, + enum paging_status paging, + uint32_t offset, + uint32_t limit, + cxobj *xstate) +{ + int retval = -1; + cxobj **xvec = NULL; + size_t xlen = 0; + int i; + cxobj *xt = NULL; + yang_stmt *yspec = NULL; + FILE *fp = NULL; + cxobj *x1; + uint32_t lower; + uint32_t upper; + int ret; + + /* If -S is set, then read state data from file */ + if (!_state || !_state_file) + goto ok; + yspec = clicon_dbspec_yang(h); + /* Read state file if either not cached, or the cache is NULL */ + if (_state_file_cached == 0 || + _state_xml_cache == NULL){ + if ((fp = fopen(_state_file, "r")) == NULL){ + clicon_err(OE_UNIX, errno, "open(%s)", _state_file); + goto done; + } + if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL) + goto done; + if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL)) < 0) + goto done; +#if 0 + if (ret == 0){ + if (clixon_netconf_internal_error(xstate, + ". Internal error, state callback returned invalid XML", + NULL) < 0) + goto done; + goto ok; + } +#endif + if (_state_file_cached) + _state_xml_cache = xt; + } + if (_state_file_cached) + xt = _state_xml_cache; + if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) + goto done; + switch (paging){ + case PAGING_NONE: + lower = 0; + upper = xlen; + break; + case PAGING_STATELESS: + case PAGING_LOCK: + lower = offset; + if (limit == 0) + upper = xlen; + else{ + if ((upper = offset+limit)>xlen) + upper = xlen; + } + break; + } + /* Mark elements to copy: + * For every node found in x0, mark the tree as changed + */ + for (i=lower; i */ + .ca_lockdb=example_lockdb, /* Database lock changed state */ .ca_trans_begin=main_begin, /* trans begin */ .ca_trans_validate=main_validate, /* trans validate */ .ca_trans_complete=main_complete, /* trans complete */ @@ -1076,9 +1170,10 @@ clixon_plugin_init(clicon_handle h) break; case 'S': /* state file (requires -s) */ _state_file = optarg; + api.ca_statedata2 = example_statefile; /* Switch state data callback */ break; case 'i': /* read state file on init not by request (requires -sS */ - _state_file_init = 1; + _state_file_cached = 1; break; case 'u': /* module-specific upgrade */ _module_upgrade = 1; diff --git a/lib/clixon/clixon_data.h b/lib/clixon/clixon_data.h index e5c577ef..d256f399 100644 --- a/lib/clixon/clixon_data.h +++ b/lib/clixon/clixon_data.h @@ -52,7 +52,7 @@ * modified, and these changes have not been committed or rolled back. */ typedef struct { - uint32_t de_id; /* session id */ + uint32_t de_id; /* session id keeps lock */ cxobj *de_xml; /* cache */ int de_modified; /* Dirty since loaded/copied/committed/etc XXX:nocache? */ int de_empty; /* Empty on read from file, xmldb_readfile and xmldb_put sets it */ diff --git a/lib/clixon/clixon_plugin.h b/lib/clixon/clixon_plugin.h index 502f8e67..d1c7962c 100644 --- a/lib/clixon/clixon_plugin.h +++ b/lib/clixon/clixon_plugin.h @@ -206,11 +206,51 @@ typedef int (plgreset_t)(clicon_handle h, const char *db); */ typedef int (plgstatedata_t)(clicon_handle h, cvec *nsc, char *xpath, cxobj *xtop); -/*! Temporary new statedata callback */ +/*! List paging status in the plugin state data callback + * + * List paging is either enabled or not. + * If paging is enabled, the xpath addresses a list/ leaf-list and the plugin should return + * entries according to the values of offset and limit. + * Paging can use a lock/transaction mechanism + * If locking is not used, the plugin cannot expect more paging calls, and no state or caching + * should be used + * If locking is used, the paging is part of a session transaction and the plugin may cache + * state (such as a cache) and can expect more paging calls until the running db-lock is released, + * (see ca_lockdb) + * The transaction is the regular lock/unlock db of running-db of a specific session. + */ +enum paging_status{ + PAGING_NONE, /* No list paging: limit/offset are no-ops */ + PAGING_STATELESS, /* Stateless list paging, dont expect more paging calls */ + PAGING_LOCK /* Transactional list paging, can expect more paging until lock release */ +}; + +/* Plugin statedata + * @param[in] Clicon handle + * @param[in] xpath Part of state requested + * @param[in] nsc XPATH namespace context. + * @param[in] pagination List pagination: 0: No, 1: begin/next, 2: end + * @param[in] offset Offset, for list pagination + * @param[in] limit Limit, for list pagination + * @param[out] xtop XML tree where statedata is added + * @retval -1 Fatal error + * @retval 0 OK + */ typedef int (plgstatedata2_t)(clicon_handle h, cvec *nsc, char *xpath, + enum paging_status pagination, uint32_t offset, uint32_t limit, cxobj *xtop); +/*! Lock databse status has changed status + * @param[in] h Clixon handle + * @param[in] db Database name (eg "running") + * @param[in] lock Lock status: 0: unlocked, 1: locked + * @param[in] id Session id (of locker/unlocker) + * @retval -1 Fatal error + * @retval 0 OK +*/ +typedef int (plglockdb_t)(clicon_handle h, char *db, int lock, int id); + /* Transaction-data type * @see clixon_backend_transaction.h for full transaction API */ @@ -283,6 +323,7 @@ struct clixon_plugin_api{ plgreset_t *cb_reset; /* Reset system status */ plgstatedata_t *cb_statedata; /* Get state data from plugin (backend only) */ plgstatedata2_t *cb_statedata2; /* Get state data from plugin (backend only) */ + plglockdb_t *cb_lockdb; /* Database lock changed state */ trans_cb_t *cb_trans_begin; /* Transaction start */ trans_cb_t *cb_trans_validate; /* Transaction validation */ trans_cb_t *cb_trans_complete; /* Transaction validation complete */ @@ -305,6 +346,7 @@ struct clixon_plugin_api{ #define ca_reset u.cau_backend.cb_reset #define ca_statedata u.cau_backend.cb_statedata #define ca_statedata2 u.cau_backend.cb_statedata2 +#define ca_lockdb u.cau_backend.cb_lockdb #define ca_trans_begin u.cau_backend.cb_trans_begin #define ca_trans_validate u.cau_backend.cb_trans_validate #define ca_trans_complete u.cau_backend.cb_trans_complete diff --git a/lib/src/clixon_datastore.c b/lib/src/clixon_datastore.c index 583c3206..093fa5b0 100644 --- a/lib/src/clixon_datastore.c +++ b/lib/src/clixon_datastore.c @@ -302,26 +302,29 @@ xmldb_unlock(clicon_handle h, * @param[in] h Clicon handle * @param[in] id Session id * @retval -1 Error - * @retval 0 OK + * @retval 0 OK */ int xmldb_unlock_all(clicon_handle h, uint32_t id) { - int retval = -1; - char **keys = NULL; - size_t klen; - int i; - db_elmnt *de; + int retval = -1; + char **keys = NULL; + size_t klen; + int i; + db_elmnt *de; + /* get all db:s */ if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0) goto done; - for (i = 0; i < klen; i++) + /* Identify the ones locked by client id */ + for (i = 0; i < klen; i++) { if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL && de->de_id == id){ de->de_id = 0; clicon_db_elmnt_set(h, keys[i], de); } + } retval = 0; done: if (keys) diff --git a/lib/src/clixon_datastore_read.c b/lib/src/clixon_datastore_read.c index c4cf0293..fd5f960c 100644 --- a/lib/src/clixon_datastore_read.c +++ b/lib/src/clixon_datastore_read.c @@ -794,6 +794,8 @@ xmldb_get_cache(clicon_handle h, * No, argument against: we may want to have a semantically wrong file and wish to edit? */ de0.de_xml = x0t; + if (de) + de0.de_id = de->de_id; clicon_db_elmnt_set(h, db, &de0); /* Content is copied */ } /* x0t == NULL */ else @@ -955,6 +957,8 @@ xmldb_get_zerocopy(clicon_handle h, * No, argument against: we may want to have a semantically wrong file and wish to edit? */ de0.de_xml = x0t; + if (de) + de0.de_id = de->de_id; clicon_db_elmnt_set(h, db, &de0); } /* x0t == NULL */ else diff --git a/test/test_pagination_config.sh b/test/test_paging_config.sh similarity index 100% rename from test/test_pagination_config.sh rename to test/test_paging_config.sh diff --git a/test/test_pagination_draft.sh b/test/test_paging_draft.sh similarity index 100% rename from test/test_pagination_draft.sh rename to test/test_paging_draft.sh diff --git a/test/test_pagination_state.sh b/test/test_paging_state.sh similarity index 64% rename from test/test_pagination_state.sh rename to test/test_paging_state.sh index 90e683b8..4b51387e 100755 --- a/test/test_pagination_state.sh +++ b/test/test_paging_state.sh @@ -7,8 +7,8 @@ # Magic line must be first in script (see README.md) s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi -echo "...skipped: Must run interactvely" -if [ "$s" = $0 ]; then exit 0; else return 0; fi +#echo "...skipped: Must run interactvely" +#if [ "$s" = $0 ]; then exit 0; else return 0; fi APPNAME=example @@ -50,51 +50,11 @@ EOF # See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log) # XXX members not currently used, only audit-logs as generated below cat< $fstate - - - alice - - 2020-07-08T12:38:32Z - admin - 2021-04-01T02:51:11Z - - - - bob - - 2020-08-14T03:30:00Z - standard - 2020-08-14T03:34:30Z - - - - eric - - 2020-09-17T19:38:32Z - pro - 2020-09-17T18:02:04Z - - - - lin - - 2020-07-09T12:38:32Z - standard - 2021-04-01T02:51:11Z - - - - joe - - 2020-10-08T12:38:32Z - pro - 2021-04-01T02:51:11Z - - - EOF # Append generated state data to $fstate file +# Generation of random timestamps (not used) +# and succesive bob$i member-ids new "generate state with $perfnr list entries" echo "" >> $fstate for (( i=0; i<$perfnr; i++ )); do @@ -103,17 +63,15 @@ for (( i=0; i<$perfnr; i++ )); do day=$(( ( RANDOM % 10 ) )) hour=$(( ( RANDOM % 10 ) )) echo " 2020-0$mon-0$dayT0$hour:48:11Z" >> $fstate - echo " bob" >> $fstate - ip1=$(( ( RANDOM % 255 ) )) - ip2=$(( ( RANDOM % 255 ) )) - echo " 192.168.$ip1.$ip2" >> $fstate + echo " bob$i" >> $fstate + echo " 192.168.1.32" >> $fstate echo " POST" >> $fstate echo " true" >> $fstate echo " " >> $fstate done echo -n "" >> $fstate # No CR -new "test params: -f $cfg -s init -- -sS $fstate" +new "test params: -f $cfg -s init -- -siS $fstate" if [ $BE -ne 0 ]; then new "kill old backend" @@ -123,8 +81,8 @@ if [ $BE -ne 0 ]; then fi sudo pkill -f clixon_backend # to be sure - new "start backend -s init -f $cfg -- -sS $fstate" - start_backend -s init -f $cfg -- -sS $fstate + new "start backend -s init -f $cfg -- -siS $fstate" + start_backend -s init -f $cfg -- -siS $fstate fi new "wait backend"