experiments/clixon
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

* New state callback signature (ca_statedata2)

Browse source 
* The new callback contains parameters for paging
   * Goal is to replace ca_statedata callback
* New plugin callback when lock/unlock occurs
  * Add `ca_lockdb` tro plugin init to use it.
* Fixed: Typing 'q' in CLI more paging did not properly quit output
  * Output continued but was not shown, for a very large file this could cause considerable delay
* Fixed: Lock was broken in first get get access
  * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access.
This commit is contained in:
Olof hagsand 2021-09-04 11:35:27 +02:00
parent aaf9a89183
commit caabfd464e
15 changed files with 487 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

18
CHANGELOG.md
View file

@ -47,6 +47,10 @@ Expected: September, 2021
* ietf-origin@2018-02-14.yang
* ietf-yang-metadata@2016-08-05.yang
* ietf-netconf-with-defaults@2011-06-01.yang
* New state callback signature (ca_statedata2)
* The new callback contains parameters for paging
* Goal is to replace ca_statedata callback
* YANG Leafref feature update
* Closer adherence to RFC 7950. Some of this is changed behavior, some is new feature.
* Essentially instead of looking at the referring leaf, context is referred(target) node
@ -66,6 +70,11 @@ Expected: September, 2021
* See draft-wwlh-netconf-list-pagination-00.txt
* New http media: application/yang-collection+xml/json
* New state callback signature (ca_statedata2)
* The new callback contains parameters for paging
* Goal is to replace ca_statedata callback
### API changes on existing protocol/config features
Users may have to change how they access the system
@ -82,6 +91,11 @@ Users may have to change how they access the system
* Removed default of `CLICON_RESTCONF_INSTALLDIR`
* The default behaviour is changed to use the config $(sbindir) to locate `clixon_restconf` when starting restconf internally
### C/CLI-API changes on existing features
Developers may need to change their code
### Minor features
* JSON errors are now labelled with JSON and not XML
@ -106,6 +120,10 @@ Users may have to change how they access the system
* In this case, eg "uses", single quotes can now be used, but not `qstring + qstring` in this case
* Fixed: [Performance issue when parsing large JSON param](https://github.com/clicon/clixon/issues/266)
* Fixed: [Duplicate lines emitted by cli_show_config (cli output style) when yang list element has composite key](https://github.com/clicon/clixon/issues/258)
* Fixed: Typing 'q' in CLI more paging did not properly quit output
* Output continued but was not shown, for a very large file this could cause considerable delay
* Fixed: Lock was broken in first get get access
* if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access.
* Fixed: [JSON leaf-list output single element leaf-list does not use array](https://github.com/clicon/clixon/issues/261)
* Fixed: Netconf diff callback did not work with choice and same value replace
* Eg if YANG is `choice c { leaf x; leaf y }` and XML changed from `<x>42</x>` to `<y>42</y>` the datastrore changed, but was not detected by diff algorithms and provided to validate callbacks.

60
apps/backend/backend_client.c
View file

@ -122,6 +122,37 @@ ce_event_cb(clicon_handle h,
return 0;
}
/*! Unlock all db:s of a client and call user unlock calback
* @see xmldb_unlock_all unlocks, but does not call user callbacks which is a backend thing
*/
static int
release_all_dbs(clicon_handle h,
uint32_t id)
{
int retval = -1;
char **keys = NULL;
size_t klen;
int i;
db_elmnt *de;
/* get all db:s */
if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0)
goto done;
/* Identify the ones locked by client id */
for (i = 0; i < klen; i++) {
if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL &&
de->de_id == id){
de->de_id = 0; /* unlock */
clicon_db_elmnt_set(h, keys[i], de);
if (clixon_plugin_lockdb_all(h, keys[i], 0, id) < 0)
goto done;
}
}
retval = 0;
done:
return retval;
}
/*! Remove client entry state
* Close down everything wrt clients (eg sockets, subscriptions)
* Finally actually remove client struct in handle
@ -148,7 +179,8 @@ backend_client_rm(clicon_handle h,
clixon_event_unreg_fd(ce->ce_s, from_client);
close(ce->ce_s);
ce->ce_s = 0;
xmldb_unlock_all(h, ce->ce_id);
if (release_all_dbs(h, ce->ce_id) < 0)
return -1;
}
break;
}
@ -606,6 +638,9 @@ from_client_lock(clicon_handle h,
}
if (xmldb_lock(h, db, id) < 0)
goto done;
/* user callback */
if (clixon_plugin_lockdb_all(h, db, 1, id) < 0)
goto done;
cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
ok:
retval = 0;
@ -677,6 +712,9 @@ from_client_unlock(clicon_handle h,
}
else{
xmldb_unlock(h, db);
/* user callback */
if (clixon_plugin_lockdb_all(h, db, 0, id) < 0)
goto done;
if (cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE) < 0)
goto done;
}
@ -699,15 +737,16 @@ from_client_unlock(clicon_handle h,
*/
static int
from_client_close_session(clicon_handle h,
cxobj *xe,
cbuf *cbret,
void *arg,
void *regarg)
cxobj *xe,
cbuf *cbret,
void *arg,
void *regarg)
{
struct client_entry *ce = (struct client_entry *)arg;
uint32_t id = ce->ce_id;
xmldb_unlock_all(h, id);
if (release_all_dbs(h, id) < 0)
return -1;
stream_ss_delete_all(h, ce_event_cb, (void*)ce);
cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
return 0;
@ -751,11 +790,16 @@ from_client_kill_session(clicon_handle h,
goto ok;
/* may or may not be in active client list, probably not */
if ((ce = ce_find_byid(backend_client_list(h), id)) != NULL){
xmldb_unlock_all(h, id); /* Removes locks on all databases */
if (release_all_dbs(h, id) < 0)
goto done;
backend_client_rm(h, ce); /* Removes client struct */
}
if (xmldb_islocked(h, db) == id)
if (xmldb_islocked(h, db) == id){
xmldb_unlock(h, db);
/* user callback */
if (clixon_plugin_lockdb_all(h, db, 0, id) < 0)
goto done;
}
cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
ok:
retval = 0;

68
apps/backend/backend_get.c
View file

@ -69,6 +69,7 @@
#include "clixon_backend_handle.h"
#include "clixon_backend_plugin.h"
#include "clixon_backend_commit.h"
#include "backend_client.h"
#include "backend_handle.h"
#include "backend_get.h"
@ -260,7 +261,7 @@ get_client_statedata(clicon_handle h,
goto fail;
}
/* Use plugin state callbacks */
if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, 0, 0, xret)) < 0)
if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, PAGING_NONE, 0, 0, xret)) < 0)
goto done;
if (ret == 0)
goto fail;
@ -313,6 +314,7 @@ element2value(clicon_handle h,
* It is specialized enough to have its own function. Specifically, extra attributes as well
* as the list-paginaiton API
* @param[in] h Clicon handle
* @param[in] ce Client entry, for locking
* @param[in] xe Request: <rpc><xn></rpc>
* @param[in] content Get config/state/both
* @param[in] db Database name
@ -326,16 +328,17 @@ element2value(clicon_handle h,
* XXX Lots of this code (in particular at the end) is copy of get_common
*/
static int
get_list_pagination(clicon_handle h,
cxobj *xe,
netconf_content content,
char *db,
int32_t depth,
yang_stmt *yspec,
char *xpath,
cvec *nsc,
char *username,
cbuf *cbret
get_list_pagination(clicon_handle h,
struct client_entry *ce,
cxobj *xe,
netconf_content content,
char *db,
int32_t depth,
yang_stmt *yspec,
char *xpath,
cvec *nsc,
char *username,
cbuf *cbret
)
{
int retval = -1;
@ -359,6 +362,8 @@ get_list_pagination(clicon_handle h,
int ret;
int i;
cxobj *xnacm = NULL;
uint32_t iddb; /* DBs lock, if any */
enum paging_status pagingstatus;
#ifdef REMAINING
cxobj *xcache = NULL;
uint32_t total = 0;
@ -495,11 +500,16 @@ get_list_pagination(clicon_handle h,
break;
}/* switch content */
if (!list_config){
/* Look if registered,
* get all state list and filter using cbpath
*/
/* Check if running locked (by this session) */
if ((iddb = xmldb_islocked(h, "running")) != 0 &&
iddb == ce->ce_id)
pagingstatus = PAGING_LOCK;
else
pagingstatus = PAGING_STATELESS;
/* Use plugin state callbacks */
if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, offset, limit, &xret)) < 0)
if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath,
pagingstatus,
offset, limit, &xret)) < 0)
goto done;
}
/* Code complex to filter out anything that is outside of xpath
@ -603,6 +613,7 @@ get_list_pagination(clicon_handle h,
/*! Common get/get-config code for retrieving configuration and state information.
*
* @param[in] h Clicon handle
* @param[in] ce Client entry, for locking
* @param[in] xe Request: <rpc><xn></rpc>
* @param[in] content Get config/state/both
* @param[in] db Database name
@ -613,11 +624,12 @@ get_list_pagination(clicon_handle h,
* @see from_client_get_config
*/
static int
get_common(clicon_handle h,
cxobj *xe,
netconf_content content,
char *db,
cbuf *cbret
get_common(clicon_handle h,
struct client_entry *ce,
cxobj *xe,
netconf_content content,
char *db,
cbuf *cbret
)
{
int retval = -1;
@ -696,7 +708,7 @@ get_common(clicon_handle h,
* check config/state
*/
if (list_pagination){
if (get_list_pagination(h, xe, content, db,
if (get_list_pagination(h, ce, xe, content, db,
depth, yspec, xpath, nsc, username,
cbret) < 0)
goto done;
@ -901,14 +913,15 @@ from_client_get_config(clicon_handle h,
void *arg,
void *regarg)
{
int retval = -1;
char *db;
int retval = -1;
char *db;
struct client_entry *ce = (struct client_entry *)arg;
if ((db = netconf_db_find(xe, "source")) == NULL){
clicon_err(OE_XML, 0, "db not found");
goto done;
}
retval = get_common(h, xe, CONTENT_CONFIG, db, cbret);
retval = get_common(h, ce, xe, CONTENT_CONFIG, db, cbret);
done:
return retval;
}
@ -932,11 +945,12 @@ from_client_get(clicon_handle h,
void *arg,
void *regarg)
{
netconf_content content = CONTENT_ALL;
char *attr;
netconf_content content = CONTENT_ALL;
char *attr;
struct client_entry *ce = (struct client_entry *)arg;
/* Clixon extensions: content */
if ((attr = xml_find_value(xe, "content")) != NULL)
content = netconf_content_str2int(attr);
return get_common(h, xe, content, "running", cbret);
return get_common(h, ce, xe, content, "running", cbret);
}

85
apps/backend/backend_plugin.c
View file

@ -239,6 +239,7 @@ clixon_plugin_daemon_all(clicon_handle h)
* @param[in] h clicon handle
* @param[in] nsc namespace context for xpath
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] pagingstatus List pagination status
* @param[in] offset Offset, for list pagination
* @param[in] limit Limit, for list pagination
* @param[out] xp If retval=1, state tree created and returned: <config>...
@ -251,6 +252,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
clicon_handle h,
cvec *nsc,
char *xpath,
enum paging_status pagingstatus,
uint32_t offset,
uint32_t limit,
cxobj **xp)
@ -262,9 +264,9 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp));
if ((fn2 = clixon_plugin_api_get(cp)->ca_statedata2) != NULL){
if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
goto done;
if (fn2(h, nsc, xpath, offset, limit, x) < 0){
if (fn2(h, nsc, xpath, pagingstatus, offset, limit, x) < 0){
if (clicon_errno < 0)
clicon_log(LOG_WARNING, "%s: Internal error: State callback in plugin: %s returned -1 but did not make a clicon_err call",
__FUNCTION__, clixon_plugin_name_get(cp));
@ -272,7 +274,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
}
}
else if ((fn = clixon_plugin_api_get(cp)->ca_statedata) != NULL){
if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
goto done;
if (fn(h, nsc, xpath, x) < 0){
if (clicon_errno < 0)
@ -298,6 +300,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
* @param[in] yspec Yang spec
* @param[in] nsc Namespace context
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] pagination List pagination
* @param[in] offset Offset, for list pagination
* @param[in] limit Limit, for list pagination
* @param[in,out] xret State XML tree is merged with existing tree.
@ -307,13 +310,14 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
* @note xret can be replaced in this function
*/
int
clixon_plugin_statedata_all(clicon_handle h,
yang_stmt *yspec,
cvec *nsc,
char *xpath,
uint32_t offset,
uint32_t limit,
cxobj **xret)
clixon_plugin_statedata_all(clicon_handle h,
yang_stmt *yspec,
cvec *nsc,
char *xpath,
enum paging_status pagingstatus,
uint32_t offset,
uint32_t limit,
cxobj **xret)
{
int retval = -1;
int ret;
@ -324,7 +328,8 @@ clixon_plugin_statedata_all(clicon_handle h,
clicon_debug(1, "%s", __FUNCTION__);
while ((cp = clixon_plugin_each(h, cp)) != NULL) {
if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, offset, limit, &x)) < 0)
if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, pagingstatus,
offset, limit, &x)) < 0)
goto done;
if (ret == 0){
if ((cberr = cbuf_new()) == NULL){
@ -399,6 +404,64 @@ clixon_plugin_statedata_all(clicon_handle h,
goto done;
}
/*! Lock database status has changed status
* @param[in] cp Plugin handle
* @param[in] h Clixon handle
* @param[in] db Database name (eg "running")
* @param[in] lock Lock status: 0: unlocked, 1: locked
* @param[in] id Session id (of locker/unlocker)
* @retval -1 Fatal error
* @retval 0 OK
*/
static int
clixon_plugin_lockdb_one(clixon_plugin_t *cp,
clicon_handle h,
char *db,
int lock,
int id)
{
int retval = -1;
plglockdb_t *fn; /* Plugin statedata fn */
clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp));
if ((fn = clixon_plugin_api_get(cp)->ca_lockdb) != NULL){
if (fn(h, db, lock, id) < 0)
goto done;
}
retval = 0;
done:
return retval;
}
/*! Lock database status has changed status
* @param[in] h Clixon handle
* @param[in] db Database name (eg "running")
* @param[in] lock Lock status: 0: unlocked, 1: locked
* @param[in] id Session id (of locker/unlocker)
* @retval -1 Fatal error
* @retval 0 OK
*/
int
clixon_plugin_lockdb_all(clicon_handle h,
char *db,
int lock,
int id
)
{
int retval = -1;
clixon_plugin_t *cp = NULL;
clicon_debug(1, "%s", __FUNCTION__);
while ((cp = clixon_plugin_each(h, cp)) != NULL) {
if (clixon_plugin_lockdb_one(cp, h, db, lock, id) < 0)
goto done;
}
retval = 0;
done:
return retval;
}
/*! Create and initialize a validate/commit transaction
* @retval td New alloced transaction,
* @retval NULL Error

3
apps/backend/clixon_backend_plugin.h
View file

@ -76,7 +76,10 @@ int clixon_plugin_pre_daemon_all(clicon_handle h);
int clixon_plugin_daemon_all(clicon_handle h);
int clixon_plugin_statedata_all(clicon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath,
enum paging_status pagingstatus,
uint32_t offset, uint32_t limit, cxobj **xtop);
int clixon_plugin_lockdb_all(clicon_handle h, char *db, int lock, int id);
transaction_data_t * transaction_new(void);
int transaction_free(transaction_data_t *);

2
apps/cli/cli_common.c
View file

@ -1043,6 +1043,8 @@ cli_notification_cb(int s,
default:
break;
}
if (cli_output_status() < 0)
break;
}
}
retval = 0;

10
apps/cli/cli_show.c
View file

@ -950,6 +950,8 @@ cli_pagination(clicon_handle h,
}
if ((nsc = xml_nsctx_init(prefix, namespace)) == NULL)
goto done;
if (clicon_rpc_lock(h, "running") < 0)
goto done;
for (i = 0;; i++){
if (clicon_rpc_get_pageable_list(h, "running", xpath, nsc,
CONTENT_ALL,
@ -984,7 +986,11 @@ cli_pagination(clicon_handle h,
default:
break;
}
if (cli_output_status() < 0)
break;
}
if (cli_output_status() < 0)
break;
if (xlen != window) /* Break if fewer elements than requested */
break;
if (xret){
@ -995,7 +1001,9 @@ cli_pagination(clicon_handle h,
free(xvec);
xvec = NULL;
}
}
} /* for */
if (clicon_rpc_unlock(h, "running") < 0)
goto done;
retval = 0;
done:
if (xvec)

347
example/main/example_backend.c
View file

@ -91,8 +91,13 @@ static char *_state_file = NULL;
* Primarily for testing
* Start backend with -- -siS <file>
*/
static int _state_file_init = 0;
static cxobj *_state_xstate = NULL;
static int _state_file_cached = 0;
/*! Cache control of read state file paging example,
* keep xml tree cache as long as db is locked
*/
static cxobj *_state_xml_cache = NULL; /* XML cache */
static int _state_file_transaction = 0;
/*! Variable to control module-specific upgrade callbacks.
* If set, call test-case for upgrading ietf-interfaces, otherwise call
@ -343,13 +348,17 @@ example_copy_extra(clicon_handle h, /* Clicon handle */
return retval;
}
/*! Called to get state data from plugin
* @param[in] h Clicon handle
* @param[in] nsc External XML namespace context, or NULL
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] xstate XML tree, <config/> on entry.
* @retval 0 OK
* @retval -1 Error
/*! Called to get state data from plugin by programmatically adding state
*
* @param[in] h Clicon handle
* @param[in] nsc External XML namespace context, or NULL
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] paging List pagination (not uses here)
* @param[in] offset Offset, for list pagination
* @param[in] limit Limit, for list pagination
* @param[in] xstate XML tree, <config/> on entry.
* @retval 0 OK
* @retval -1 Error
* @see xmldb_get
* @note this example code returns requires this yang snippet:
container state {
@ -360,11 +369,14 @@ example_copy_extra(clicon_handle h, /* Clicon handle */
}
}
* This yang snippet is present in clixon-example.yang for example.
* XXX paging for lock
* @see example_statefile where state is read from file and also paging
*/
int
example_statedata(clicon_handle h,
cvec *nsc,
char *xpath,
enum paging_status paging,
uint32_t offset,
uint32_t limit,
cxobj *xstate)
@ -377,127 +389,62 @@ example_statedata(clicon_handle h,
cxobj *xt = NULL;
char *name;
cvec *nsc1 = NULL;
cvec *nsc2 = NULL;
yang_stmt *yspec = NULL;
FILE *fp = NULL;
cxobj *x1;
uint32_t upper;
if (!_state)
goto ok;
yspec = clicon_dbspec_yang(h);
/* If -S is set, then read state data from file, otherwise construct it programmatically */
if (_state_file){
if (_state_file_init){
#if 0 /* This is just for a zero-copy version (only works once) */
{
cxobj *xx = NULL;
while (xml_child_nr(_state_xstate)){
xx = xml_child_i(_state_xstate,0);
if (xml_addsub(xstate, xx) < 0)
goto done;
}
}
#else
if (xml_copy(_state_xstate, xstate) < 0)
goto done;
#endif
}
else{
if ((fp = fopen(_state_file, "r")) == NULL){
clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
goto done;
}
if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL)
goto done;
/* Note, does not care about xpath / list-pagination */
if (clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL) < 0)
goto done;
if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0)
goto done;
if (limit == 0)
upper = xlen;
else{
if ((upper = offset+limit)>xlen)
upper = xlen;
}
for (i=offset; i<upper; i++){
if ((x1 = xvec[i]) == NULL)
break;
xml_flag_set(x1, XML_FLAG_MARK);
}
/* Remove everything that is not marked */
if (xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL) < 0)
goto done;
if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)XML_FLAG_MARK) < 0)
goto done;
if (xml_copy(xt, xstate) < 0)
goto done;
if (xvec){
free(xvec);
xvec = 0;
xlen = 0;
}
/* Example of statedata, in this case merging state data with
* state information. In this case adding dummy interface operation state
* to configured interfaces.
* Get config according to xpath */
if ((nsc1 = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:yang:ietf-interfaces")) == NULL)
goto done;
if (xmldb_get0(h, "running", YB_MODULE, nsc1, "/interfaces/interface/name", 1, &xt, NULL, NULL) < 0)
goto done;
if (xpath_vec(xt, nsc1, "/interfaces/interface/name", &xvec, &xlen) < 0)
goto done;
if (xlen){
cprintf(cb, "<interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\">");
for (i=0; i<xlen; i++){
name = xml_body(xvec[i]);
cprintf(cb, "<interface xmlns:ex=\"urn:example:clixon\"><name>%s</name><type>ex:eth</type><oper-status>up</oper-status>", name);
cprintf(cb, "<ex:my-status><ex:int>42</ex:int><ex:str>foo</ex:str></ex:my-status>");
cprintf(cb, "</interface>");
}
cprintf(cb, "</interfaces>");
if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0)
goto done;
}
else {
/* Example of statedata, in this case merging state data with
* state information. In this case adding dummy interface operation state
* to configured interfaces.
* Get config according to xpath */
if ((nsc1 = xml_nsctx_init(NULL, "urn:ietf:params:xml:ns:yang:ietf-interfaces")) == NULL)
/* State in test_yang.sh , test_restconf.sh and test_order.sh */
if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){
if (clixon_xml_parse_string("<state xmlns=\"urn:example:clixon\">"
"<op>42</op>"
"<op>41</op>"
"<op>43</op>" /* should not be ordered */
"</state>",
YB_NONE,
NULL, &xstate, NULL) < 0)
goto done; /* For the case when urn:example:clixon is not loaded */
}
/* Event state from RFC8040 Appendix B.3.1
* Note: (1) order is by-system so is different,
* (2) event-count is XOR on name, so is not 42 and 4
*/
if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){
cbuf_reset(cb);
cprintf(cb, "<events xmlns=\"urn:example:events\">");
cprintf(cb, "<event><name>interface-down</name><event-count>90</event-count></event>");
cprintf(cb, "<event><name>interface-up</name><event-count>77</event-count></event>");
cprintf(cb, "</events>");
if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0)
goto done;
if (xmldb_get0(h, "running", YB_MODULE, nsc1, "/interfaces/interface/name", 1, &xt, NULL, NULL) < 0)
goto done;
if (xpath_vec(xt, nsc1, "/interfaces/interface/name", &xvec, &xlen) < 0)
goto done;
if (xlen){
cprintf(cb, "<interfaces xmlns=\"urn:ietf:params:xml:ns:yang:ietf-interfaces\">");
for (i=0; i<xlen; i++){
name = xml_body(xvec[i]);
cprintf(cb, "<interface xmlns:ex=\"urn:example:clixon\"><name>%s</name><type>ex:eth</type><oper-status>up</oper-status>", name);
cprintf(cb, "<ex:my-status><ex:int>42</ex:int><ex:str>foo</ex:str></ex:my-status>");
cprintf(cb, "</interface>");
}
cprintf(cb, "</interfaces>");
if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0)
goto done;
}
/* State in test_yang.sh , test_restconf.sh and test_order.sh */
if (yang_find_module_by_namespace(yspec, "urn:example:clixon") != NULL){
if (clixon_xml_parse_string("<state xmlns=\"urn:example:clixon\">"
"<op>42</op>"
"<op>41</op>"
"<op>43</op>" /* should not be ordered */
"</state>",
YB_NONE,
NULL, &xstate, NULL) < 0)
goto done; /* For the case when urn:example:clixon is not loaded */
}
/* Event state from RFC8040 Appendix B.3.1
* Note: (1) order is by-system so is different,
* (2) event-count is XOR on name, so is not 42 and 4
*/
if (yang_find_module_by_namespace(yspec, "urn:example:events") != NULL){
cbuf_reset(cb);
cprintf(cb, "<events xmlns=\"urn:example:events\">");
cprintf(cb, "<event><name>interface-down</name><event-count>90</event-count></event>");
cprintf(cb, "<event><name>interface-up</name><event-count>77</event-count></event>");
cprintf(cb, "</events>");
if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0)
goto done;
}
}
ok:
retval = 0;
done:
if (fp)
fclose(fp);
if (nsc1)
xml_nsctx_free(nsc1);
if (nsc2)
xml_nsctx_free(nsc2);
if (xt)
xml_free(xt);
if (cb)
@ -507,6 +454,157 @@ example_statedata(clicon_handle h,
return retval;
}
/*! Called to get state data from plugin by reading a file, also paging
*
* The example shows how to read and parse a state XML file, (which is cached in the -i case).
* Return the requested xpath / paging xstate by copying from the parsed state XML file
* @param[in] h Clicon handle
* @param[in] nsc External XML namespace context, or NULL
* @param[in] xpath String with XPATH syntax. or NULL for all
* @param[in] paging List pagination
* @param[in] offset Offset, for list pagination
* @param[in] limit Limit, for list pagination
* @param[in] xstate XML tree, <config/> on entry. Copy to this
* @retval 0 OK
* @retval -1 Error
* @see xmldb_get
* @see example_statefile where state is programmatically added
*/
int
example_statefile(clicon_handle h,
cvec *nsc,
char *xpath,
enum paging_status paging,
uint32_t offset,
uint32_t limit,
cxobj *xstate)
{
int retval = -1;
cxobj **xvec = NULL;
size_t xlen = 0;
int i;
cxobj *xt = NULL;
yang_stmt *yspec = NULL;
FILE *fp = NULL;
cxobj *x1;
uint32_t lower;
uint32_t upper;
int ret;
/* If -S is set, then read state data from file */
if (!_state || !_state_file)
goto ok;
yspec = clicon_dbspec_yang(h);
/* Read state file if either not cached, or the cache is NULL */
if (_state_file_cached == 0 ||
_state_xml_cache == NULL){
if ((fp = fopen(_state_file, "r")) == NULL){
clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
goto done;
}
if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL)
goto done;
if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL)) < 0)
goto done;
#if 0
if (ret == 0){
if (clixon_netconf_internal_error(xstate,
". Internal error, state callback returned invalid XML",
NULL) < 0)
goto done;
goto ok;
}
#endif
if (_state_file_cached)
_state_xml_cache = xt;
}
if (_state_file_cached)
xt = _state_xml_cache;
if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0)
goto done;
switch (paging){
case PAGING_NONE:
lower = 0;
upper = xlen;
break;
case PAGING_STATELESS:
case PAGING_LOCK:
lower = offset;
if (limit == 0)
upper = xlen;
else{
if ((upper = offset+limit)>xlen)
upper = xlen;
}
break;
}
/* Mark elements to copy:
* For every node found in x0, mark the tree as changed
*/
for (i=lower; i<upper; i++){
if ((x1 = xvec[i]) == NULL)
break;
xml_flag_set(x1, XML_FLAG_MARK);
xml_apply_ancestor(x1, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
}
if (xml_copy_marked(xt, xstate) < 0) /* Copy the marked elements */
goto done;
/* Unmark original tree */
if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
goto done;
/* Unmark returned state tree */
if (xml_apply(xstate, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
goto done;
if (_state_file_cached)
xt = NULL; /* ensure cache is not cleared */
if (paging == PAGING_LOCK)
_state_file_transaction++;
ok:
retval = 0;
done:
if (fp)
fclose(fp);
if (xt)
xml_free(xt);
if (xvec)
free(xvec);
return retval;
}
/*! Lock databse status has changed status
* @param[in] h Clixon handle
* @param[in] db Database name (eg "running")
* @param[in] lock Lock status: 0: unlocked, 1: locked
* @param[in] id Session id (of locker/unlocker)
* @retval -1 Fatal error
* @retval 0 OK
*/
int
example_lockdb(clicon_handle h,
char *db,
int lock,
int id)
{
int retval = -1;
clicon_debug(1, "%s Lock callback: db%s: locked:%d", __FUNCTION__, db, lock);
/* Part of cached paging example
*/
if (strcmp(db, "running") == 0 && lock == 0 &&
_state && _state_file && _state_file_cached && _state_file_transaction){
if (_state_xml_cache){
xml_free(_state_xml_cache);
_state_xml_cache = NULL;
}
_state_file_transaction = 0;
}
retval = 0;
// done:
return retval;
}
/*! Callback for yang extensions example:e4
*
* @param[in] h Clixon handle
@ -982,7 +1080,7 @@ example_start(clicon_handle h)
/*! Plugin daemon.
* @param[in] h Clicon handle
*
* plugin_daemon is called once after damonization has been made but before lowering of privileges
* plugin_daemon is called once after daemonization has been made but before lowering of privileges
* the main event loop is entered.
*/
int
@ -994,19 +1092,14 @@ example_daemon(clicon_handle h)
yang_stmt *yspec;
/* Read state file (or should this be in init/start?) */
if (_state && _state_file && _state_file_init){
if (_state && _state_file && _state_file_cached){
yspec = clicon_dbspec_yang(h);
if ((fp = fopen(_state_file, "r")) == NULL){
clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
goto done;
}
if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &_state_xstate, NULL)) < 0)
if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &_state_xml_cache, NULL)) < 1)
goto done;
if (ret == 0){
fprintf(stderr, "%s error\n", __FUNCTION__);
goto done;
}
fprintf(stderr, "%s done\n", __FUNCTION__);
}
retval = 0;
done:
@ -1032,7 +1125,8 @@ static clixon_plugin_api api = {
.ca_extension=example_extension, /* yang extensions */
.ca_daemon=example_daemon, /* daemon */
.ca_reset=example_reset, /* reset */
.ca_statedata2=example_statedata, /* statedata2 */
.ca_statedata2=example_statedata, /* statedata2 : Note fn is switched if -sS <file> */
.ca_lockdb=example_lockdb, /* Database lock changed state */
.ca_trans_begin=main_begin, /* trans begin */
.ca_trans_validate=main_validate, /* trans validate */
.ca_trans_complete=main_complete, /* trans complete */
@ -1076,9 +1170,10 @@ clixon_plugin_init(clicon_handle h)
break;
case 'S': /* state file (requires -s) */
_state_file = optarg;
api.ca_statedata2 = example_statefile; /* Switch state data callback */
break;
case 'i': /* read state file on init not by request (requires -sS <file> */
_state_file_init = 1;
_state_file_cached = 1;
break;
case 'u': /* module-specific upgrade */
_module_upgrade = 1;

2
lib/clixon/clixon_data.h
View file

@ -52,7 +52,7 @@
* modified, and these changes have not been committed or rolled back.
*/
typedef struct {
uint32_t de_id; /* session id */
uint32_t de_id; /* session id keeps lock */
cxobj *de_xml; /* cache */
int de_modified; /* Dirty since loaded/copied/committed/etc XXX:nocache? */
int de_empty; /* Empty on read from file, xmldb_readfile and xmldb_put sets it */

44
lib/clixon/clixon_plugin.h
View file

@ -206,11 +206,51 @@ typedef int (plgreset_t)(clicon_handle h, const char *db);
*/
typedef int (plgstatedata_t)(clicon_handle h, cvec *nsc, char *xpath, cxobj *xtop);
/*! Temporary new statedata callback */
/*! List paging status in the plugin state data callback
*
* List paging is either enabled or not.
* If paging is enabled, the xpath addresses a list/ leaf-list and the plugin should return
* entries according to the values of offset and limit.
* Paging can use a lock/transaction mechanism
* If locking is not used, the plugin cannot expect more paging calls, and no state or caching
* should be used
* If locking is used, the paging is part of a session transaction and the plugin may cache
* state (such as a cache) and can expect more paging calls until the running db-lock is released,
* (see ca_lockdb)
* The transaction is the regular lock/unlock db of running-db of a specific session.
*/
enum paging_status{
PAGING_NONE, /* No list paging: limit/offset are no-ops */
PAGING_STATELESS, /* Stateless list paging, dont expect more paging calls */
PAGING_LOCK /* Transactional list paging, can expect more paging until lock release */
};
/* Plugin statedata
* @param[in] Clicon handle
* @param[in] xpath Part of state requested
* @param[in] nsc XPATH namespace context.
* @param[in] pagination List pagination: 0: No, 1: begin/next, 2: end
* @param[in] offset Offset, for list pagination
* @param[in] limit Limit, for list pagination
* @param[out] xtop XML tree where statedata is added
* @retval -1 Fatal error
* @retval 0 OK
*/
typedef int (plgstatedata2_t)(clicon_handle h, cvec *nsc, char *xpath,
enum paging_status pagination,
uint32_t offset, uint32_t limit,
cxobj *xtop);
/*! Lock databse status has changed status
* @param[in] h Clixon handle
* @param[in] db Database name (eg "running")
* @param[in] lock Lock status: 0: unlocked, 1: locked
* @param[in] id Session id (of locker/unlocker)
* @retval -1 Fatal error
* @retval 0 OK
*/
typedef int (plglockdb_t)(clicon_handle h, char *db, int lock, int id);
/* Transaction-data type
* @see clixon_backend_transaction.h for full transaction API
*/
@ -283,6 +323,7 @@ struct clixon_plugin_api{
plgreset_t *cb_reset; /* Reset system status */
plgstatedata_t *cb_statedata; /* Get state data from plugin (backend only) */
plgstatedata2_t *cb_statedata2; /* Get state data from plugin (backend only) */
plglockdb_t *cb_lockdb; /* Database lock changed state */
trans_cb_t *cb_trans_begin; /* Transaction start */
trans_cb_t *cb_trans_validate; /* Transaction validation */
trans_cb_t *cb_trans_complete; /* Transaction validation complete */
@ -305,6 +346,7 @@ struct clixon_plugin_api{
#define ca_reset u.cau_backend.cb_reset
#define ca_statedata u.cau_backend.cb_statedata
#define ca_statedata2 u.cau_backend.cb_statedata2
#define ca_lockdb u.cau_backend.cb_lockdb
#define ca_trans_begin u.cau_backend.cb_trans_begin
#define ca_trans_validate u.cau_backend.cb_trans_validate
#define ca_trans_complete u.cau_backend.cb_trans_complete

17
lib/src/clixon_datastore.c
View file

@ -302,26 +302,29 @@ xmldb_unlock(clicon_handle h,
* @param[in] h Clicon handle
* @param[in] id Session id
* @retval -1 Error
* @retval 0 OK
* @retval 0 OK
*/
int
xmldb_unlock_all(clicon_handle h,
uint32_t id)
{
int retval = -1;
char **keys = NULL;
size_t klen;
int i;
db_elmnt *de;
int retval = -1;
char **keys = NULL;
size_t klen;
int i;
db_elmnt *de;
/* get all db:s */
if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0)
goto done;
for (i = 0; i < klen; i++)
/* Identify the ones locked by client id */
for (i = 0; i < klen; i++) {
if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL &&
de->de_id == id){
de->de_id = 0;
clicon_db_elmnt_set(h, keys[i], de);
}
}
retval = 0;
done:
if (keys)

4
lib/src/clixon_datastore_read.c
View file

@ -794,6 +794,8 @@ xmldb_get_cache(clicon_handle h,
* No, argument against: we may want to have a semantically wrong file and wish to edit?
*/
de0.de_xml = x0t;
if (de)
de0.de_id = de->de_id;
clicon_db_elmnt_set(h, db, &de0); /* Content is copied */
} /* x0t == NULL */
else
@ -955,6 +957,8 @@ xmldb_get_zerocopy(clicon_handle h,
* No, argument against: we may want to have a semantically wrong file and wish to edit?
*/
de0.de_xml = x0t;
if (de)
de0.de_id = de->de_id;
clicon_db_elmnt_set(h, db, &de0);
} /* x0t == NULL */
else

0
test/test_pagination_config.sh → test/test_paging_config.sh
View file

0
test/test_pagination_draft.sh → test/test_paging_draft.sh
View file

60
test/test_pagination_state.sh → test/test_paging_state.sh
View file

@ -7,8 +7,8 @@
# Magic line must be first in script (see README.md)
s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
echo "...skipped: Must run interactvely"
if [ "$s" = $0 ]; then exit 0; else return 0; fi
#echo "...skipped: Must run interactvely"
#if [ "$s" = $0 ]; then exit 0; else return 0; fi
APPNAME=example
@ -50,51 +50,11 @@ EOF
# See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log)
# XXX members not currently used, only audit-logs as generated below
cat<<EOF > $fstate
<members xmlns="http://example.com/ns/example-social">
<member>
<member-id>alice</member-id>
<stats>
<joined>2020-07-08T12:38:32Z</joined>
<membership-level>admin</membership-level>
<last-activity>2021-04-01T02:51:11Z</last-activity>
</stats>
</member>
<member>
<member-id>bob</member-id>
<stats>
<joined>2020-08-14T03:30:00Z</joined>
<membership-level>standard</membership-level>
<last-activity>2020-08-14T03:34:30Z</last-activity>
</stats>
</member>
<member>
<member-id>eric</member-id>
<stats>
<joined>2020-09-17T19:38:32Z</joined>
<membership-level>pro</membership-level>
<last-activity>2020-09-17T18:02:04Z</last-activity>
</stats>
</member>
<member>
<member-id>lin</member-id>
<stats>
<joined>2020-07-09T12:38:32Z</joined>
<membership-level>standard</membership-level>
<last-activity>2021-04-01T02:51:11Z</last-activity>
</stats>
</member>
<member>
<member-id>joe</member-id>
<stats>
<joined>2020-10-08T12:38:32Z</joined>
<membership-level>pro</membership-level>
<last-activity>2021-04-01T02:51:11Z</last-activity>
</stats>
</member>
</members>
EOF
# Append generated state data to $fstate file
# Generation of random timestamps (not used)
# and succesive bob$i member-ids
new "generate state with $perfnr list entries"
echo "<audit-logs xmlns=\"http://example.com/ns/example-social\">" >> $fstate
for (( i=0; i<$perfnr; i++ )); do
@ -103,17 +63,15 @@ for (( i=0; i<$perfnr; i++ )); do
day=$(( ( RANDOM % 10 ) ))
hour=$(( ( RANDOM % 10 ) ))
echo " <timestamp>2020-0$mon-0$dayT0$hour:48:11Z</timestamp>" >> $fstate
echo " <member-id>bob</member-id>" >> $fstate
ip1=$(( ( RANDOM % 255 ) ))
ip2=$(( ( RANDOM % 255 ) ))
echo " <source-ip>192.168.$ip1.$ip2</source-ip>" >> $fstate
echo " <member-id>bob$i</member-id>" >> $fstate
echo " <source-ip>192.168.1.32</source-ip>" >> $fstate
echo " <request>POST</request>" >> $fstate
echo " <outcome>true</outcome>" >> $fstate
echo " </audit-log>" >> $fstate
done
echo -n "</audit-logs>" >> $fstate # No CR
new "test params: -f $cfg -s init -- -sS $fstate"
new "test params: -f $cfg -s init -- -siS $fstate"
if [ $BE -ne 0 ]; then
new "kill old backend"
@ -123,8 +81,8 @@ if [ $BE -ne 0 ]; then
fi
sudo pkill -f clixon_backend # to be sure
new "start backend -s init -f $cfg -- -sS $fstate"
start_backend -s init -f $cfg -- -sS $fstate
new "start backend -s init -f $cfg -- -siS $fstate"
start_backend -s init -f $cfg -- -siS $fstate
fi
new "wait backend"