* New state callback signature (ca_statedata2)

* The new callback contains parameters for paging
   * Goal is to replace ca_statedata callback
* New plugin callback when lock/unlock occurs
  * Add `ca_lockdb` tro plugin init to use it.
* Fixed: Typing 'q' in CLI more paging did not properly quit output
  * Output continued but was not shown, for a very large file this could cause considerable delay
* Fixed: Lock was broken in first get get access
  * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access.

CHANGELOG.md

@@ -47,6 +47,10 @@ Expected: September, 2021
     * ietf-origin@2018-02-14.yang
     * ietf-yang-metadata@2016-08-05.yang
     * ietf-netconf-with-defaults@2011-06-01.yang
+  * New state callback signature (ca_statedata2)
+    * The new callback contains parameters for paging
+    * Goal is to replace ca_statedata callback
+
 * YANG Leafref feature update
   * Closer adherence to RFC 7950. Some of this is changed behavior, some is new feature.
   * Essentially instead of looking at the referring leaf, context is referred(target) node
@@ -66,6 +70,11 @@ Expected: September, 2021
   * See draft-wwlh-netconf-list-pagination-00.txt
   * New http media: application/yang-collection+xml/json
 
+* New state callback signature (ca_statedata2)
+   * The new callback contains parameters for paging
+   * Goal is to replace ca_statedata callback
+
+
 ### API changes on existing protocol/config features
 
 Users may have to change how they access the system
@@ -82,6 +91,11 @@ Users may have to change how they access the system
    * Removed default of `CLICON_RESTCONF_INSTALLDIR`
      * The default behaviour is changed to use the config $(sbindir) to locate `clixon_restconf` when starting restconf internally
 
+### C/CLI-API changes on existing features
+
+Developers may need to change their code
+
+
 ### Minor features
 
 * JSON errors are now labelled with JSON and not XML
@@ -106,6 +120,10 @@ Users may have to change how they access the system
   * In this case, eg "uses", single quotes can now be used, but not `qstring + qstring` in this case
 * Fixed: [Performance issue when parsing large JSON param](https://github.com/clicon/clixon/issues/266)
 * Fixed: [Duplicate lines emitted by cli_show_config (cli output style) when yang list element has composite key](https://github.com/clicon/clixon/issues/258)
+* Fixed: Typing 'q' in CLI more paging did not properly quit output
+  * Output continued but was not shown, for a very large file this could cause considerable delay
+* Fixed: Lock was broken in first get get access
+  * if the first netconf operation to a backend was lock;get;unlock, the lock was broken in the first get access.
 * Fixed: [JSON leaf-list output single element leaf-list does not use array](https://github.com/clicon/clixon/issues/261)
 * Fixed: Netconf diff callback did not work with choice and same value replace
   * Eg if YANG is `choice c { leaf x; leaf y }` and XML changed from `<x>42</x>` to `<y>42</y>` the datastrore changed, but was not detected by diff algorithms and provided to validate callbacks.

apps/backend/backend_client.c

@@ -122,6 +122,37 @@ ce_event_cb(clicon_handle h,
     return 0;
 }
 
+/*! Unlock all db:s of a client and call user unlock calback 
+ * @see xmldb_unlock_all  unlocks, but does not call user callbacks which is a backend thing
+ */
+static int
+release_all_dbs(clicon_handle h,
+		uint32_t      id)
+{
+    int       retval = -1;
+    char    **keys = NULL;
+    size_t    klen;
+    int       i;
+    db_elmnt *de;
+
+    /* get all db:s */
+    if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0)
+	goto done;
+    /* Identify the ones locked by client id */
+    for (i = 0; i < klen; i++) {
+	if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL &&
+	    de->de_id == id){
+	    de->de_id = 0; /* unlock */
+	    clicon_db_elmnt_set(h, keys[i], de);
+	    if (clixon_plugin_lockdb_all(h, keys[i], 0, id) < 0)
+		goto done;
+	}
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Remove client entry state
  * Close down everything wrt clients (eg sockets, subscriptions)
  * Finally actually remove client struct in handle
@@ -148,7 +179,8 @@ backend_client_rm(clicon_handle        h,
 		clixon_event_unreg_fd(ce->ce_s, from_client);
 		close(ce->ce_s);
 		ce->ce_s = 0;
-		xmldb_unlock_all(h, ce->ce_id);
+		if (release_all_dbs(h, ce->ce_id) < 0)
+		    return -1;
 	    }
 	    break;
 	}
@@ -606,6 +638,9 @@ from_client_lock(clicon_handle h,
     }
     if (xmldb_lock(h, db, id) < 0)
 	goto done;
+     /* user callback */
+    if (clixon_plugin_lockdb_all(h, db, 1, id) < 0)
+	goto done;
     cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
  ok:
     retval = 0;
@@ -677,6 +712,9 @@ from_client_unlock(clicon_handle h,
     }
     else{
 	xmldb_unlock(h, db);
+	/* user callback */
+	if (clixon_plugin_lockdb_all(h, db, 0, id) < 0)     
+	    goto done;
 	if (cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE) < 0)
 	    goto done;
     }
@@ -707,7 +745,8 @@ from_client_close_session(clicon_handle h,
     struct client_entry *ce = (struct client_entry *)arg;
     uint32_t             id = ce->ce_id;
 
-    xmldb_unlock_all(h, id);
+    if (release_all_dbs(h, id) < 0)
+	return -1;
     stream_ss_delete_all(h, ce_event_cb, (void*)ce);
     cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
     return 0;
@@ -751,11 +790,16 @@ from_client_kill_session(clicon_handle h,
 	goto ok;
     /* may or may not be in active client list, probably not */
     if ((ce = ce_find_byid(backend_client_list(h), id)) != NULL){
-	xmldb_unlock_all(h, id);  /* Removes locks on all databases */
+	if (release_all_dbs(h, id) < 0)
+	    goto done;
 	backend_client_rm(h, ce); /* Removes client struct */
     }
-    if (xmldb_islocked(h, db) == id)
+    if (xmldb_islocked(h, db) == id){
 	xmldb_unlock(h, db);
+	/* user callback */
+	if (clixon_plugin_lockdb_all(h, db, 0, id) < 0)     
+	    goto done;
+    }
     cprintf(cbret, "<rpc-reply xmlns=\"%s\"><ok/></rpc-reply>", NETCONF_BASE_NAMESPACE);
  ok:
     retval = 0;

apps/backend/backend_get.c

@@ -69,6 +69,7 @@
 #include "clixon_backend_handle.h"
 #include "clixon_backend_plugin.h"
 #include "clixon_backend_commit.h"
+#include "backend_client.h"
 #include "backend_handle.h"
 #include "backend_get.h"
 
@@ -260,7 +261,7 @@ get_client_statedata(clicon_handle h,
 	    goto fail;
     }
     /* Use plugin state callbacks */
-    if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, 0, 0, xret)) < 0)
+    if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, PAGING_NONE, 0, 0, xret)) < 0)
 	goto done;
     if (ret == 0)
 	goto fail;
@@ -313,6 +314,7 @@ element2value(clicon_handle  h,
  * It is specialized enough to have its own function. Specifically, extra attributes as well
  * as the list-paginaiton API
  * @param[in]  h       Clicon handle 
+ * @param[in]  ce      Client entry, for locking
  * @param[in]  xe      Request: <rpc><xn></rpc> 
  * @param[in]  content Get config/state/both
  * @param[in]  db      Database name
@@ -327,6 +329,7 @@ element2value(clicon_handle  h,
  */
 static int
 get_list_pagination(clicon_handle        h,
+		    struct client_entry *ce,
 		    cxobj               *xe,
 		    netconf_content      content,
 		    char                *db,
@@ -359,6 +362,8 @@ get_list_pagination(clicon_handle   h,
     int             ret;
     int             i;
     cxobj          *xnacm = NULL;
+    uint32_t        iddb; /* DBs lock, if any */
+    enum paging_status pagingstatus;
 #ifdef REMAINING
     cxobj          *xcache = NULL;
     uint32_t        total = 0;
@@ -495,11 +500,16 @@ get_list_pagination(clicon_handle   h,
 	break;
     }/* switch content */
     if (!list_config){
-	/* Look if registered, 
+	/* Check if running locked (by this session) */
-	 * get all state list and filter using cbpath
+	if ((iddb = xmldb_islocked(h, "running")) != 0 &&
-	 */
+	    iddb == ce->ce_id)
+	    pagingstatus = PAGING_LOCK;
+	else
+	    pagingstatus = PAGING_STATELESS;
 	/* Use plugin state callbacks */
-	if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath, offset, limit, &xret)) < 0)
+	if ((ret = clixon_plugin_statedata_all(h, yspec, nsc, xpath,
+					       pagingstatus,
+					       offset, limit, &xret)) < 0)
 	    goto done;
     }
     /* Code complex to filter out anything that is outside of xpath 
@@ -603,6 +613,7 @@ get_list_pagination(clicon_handle   h,
 /*! Common get/get-config code for retrieving  configuration and state information.
  *
  * @param[in]  h       Clicon handle 
+ * @param[in]  ce      Client entry, for locking
  * @param[in]  xe      Request: <rpc><xn></rpc> 
  * @param[in]  content Get config/state/both
  * @param[in]  db      Database name
@@ -614,6 +625,7 @@ get_list_pagination(clicon_handle   h,
  */
 static int
 get_common(clicon_handle        h,
+	   struct client_entry *ce,
 	   cxobj               *xe,
 	   netconf_content      content,
 	   char                *db,
@@ -696,7 +708,7 @@ get_common(clicon_handle   h,
      * check config/state
      */
     if (list_pagination){
-	if (get_list_pagination(h, xe, content, db,
+	if (get_list_pagination(h, ce, xe, content, db,
 				depth, yspec, xpath, nsc, username,
 				cbret) < 0)
 	    goto done;
@@ -903,12 +915,13 @@ from_client_get_config(clicon_handle h,
 {
     int                  retval = -1;
     char                *db;
+    struct client_entry *ce = (struct client_entry *)arg;
 
     if ((db = netconf_db_find(xe, "source")) == NULL){
 	clicon_err(OE_XML, 0, "db not found");
 	goto done;
     }
-    retval = get_common(h, xe, CONTENT_CONFIG, db, cbret);
+    retval = get_common(h, ce, xe, CONTENT_CONFIG, db, cbret);
  done:
     return retval;
 }
@@ -934,9 +947,10 @@ from_client_get(clicon_handle h,
 {
     netconf_content      content = CONTENT_ALL;
     char                *attr;
+    struct client_entry *ce = (struct client_entry *)arg;
     
     /* Clixon extensions: content */
     if ((attr = xml_find_value(xe, "content")) != NULL)
 	content = netconf_content_str2int(attr);
-    return get_common(h, xe, content, "running", cbret);
+    return get_common(h, ce, xe, content, "running", cbret);
 }

apps/backend/backend_plugin.c

@@ -239,6 +239,7 @@ clixon_plugin_daemon_all(clicon_handle h)
  * @param[in]  h       clicon handle
  * @param[in]  nsc     namespace context for xpath
  * @param[in]  xpath   String with XPATH syntax. or NULL for all
+ * @param[in]  pagingstatus List pagination status
  * @param[in]  offset  Offset, for list pagination
  * @param[in]  limit   Limit, for list pagination
  * @param[out] xp      If retval=1, state tree created and returned: <config>...
@@ -251,6 +252,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
 			    clicon_handle    h,
 			    cvec            *nsc,
 			    char            *xpath,
+			    enum paging_status pagingstatus,
 			    uint32_t         offset,
 			    uint32_t         limit,  
 			    cxobj          **xp)
@@ -262,9 +264,9 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
     
     clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp));
     if ((fn2 = clixon_plugin_api_get(cp)->ca_statedata2) != NULL){
-	if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
+	if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
 	    goto done;
-	if (fn2(h, nsc, xpath, offset, limit, x) < 0){
+	if (fn2(h, nsc, xpath, pagingstatus, offset, limit, x) < 0){
 	    if (clicon_errno < 0) 
 		clicon_log(LOG_WARNING, "%s: Internal error: State callback in plugin: %s returned -1 but did not make a clicon_err call",
 			   __FUNCTION__, clixon_plugin_name_get(cp));
@@ -272,7 +274,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
 	}
     }
     else if ((fn = clixon_plugin_api_get(cp)->ca_statedata) != NULL){
-	if ((x = xml_new(XML_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
+	if ((x = xml_new(DATASTORE_TOP_SYMBOL, NULL, CX_ELMNT)) == NULL)
 	    goto done;
 	if (fn(h, nsc, xpath, x) < 0){
 	    if (clicon_errno < 0) 
@@ -298,6 +300,7 @@ clixon_plugin_statedata_one(clixon_plugin_t *cp,
  * @param[in]     yspec   Yang spec
  * @param[in]     nsc     Namespace context
  * @param[in]     xpath   String with XPATH syntax. or NULL for all
+ * @param[in]     pagination List pagination
  * @param[in]     offset  Offset, for list pagination
  * @param[in]     limit   Limit, for list pagination
  * @param[in,out] xret    State XML tree is merged with existing tree.
@@ -311,6 +314,7 @@ clixon_plugin_statedata_all(clicon_handle h,
 			    yang_stmt         *yspec,
 			    cvec              *nsc,
 			    char              *xpath,
+			    enum paging_status pagingstatus,
 			    uint32_t           offset,
 			    uint32_t           limit,  
 			    cxobj            **xret)
@@ -324,7 +328,8 @@ clixon_plugin_statedata_all(clicon_handle h,
     
     clicon_debug(1, "%s", __FUNCTION__);
     while ((cp = clixon_plugin_each(h, cp)) != NULL) {
-	if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, offset, limit, &x)) < 0)
+	if ((ret = clixon_plugin_statedata_one(cp, h, nsc, xpath, pagingstatus,
+					       offset, limit, &x)) < 0)
 	    goto done;
 	if (ret == 0){
 	    if ((cberr = cbuf_new()) == NULL){
@@ -399,6 +404,64 @@ clixon_plugin_statedata_all(clicon_handle h,
     goto done;
 }
 
+/*! Lock database status has changed status
+ * @param[in]  cp      Plugin handle
+ * @param[in]  h    Clixon handle
+ * @param[in]  db   Database name (eg "running")
+ * @param[in]  lock Lock status: 0: unlocked, 1: locked
+ * @param[in]  id   Session id (of locker/unlocker)
+ * @retval    -1    Fatal error
+ * @retval     0    OK
+ */
+static int
+clixon_plugin_lockdb_one(clixon_plugin_t *cp,
+			 clicon_handle    h,
+			 char            *db,
+			 int              lock,
+			 int              id)
+{
+    int          retval = -1;
+    plglockdb_t *fn;          /* Plugin statedata fn */
+    
+    clicon_debug(1, "%s %s", __FUNCTION__, clixon_plugin_name_get(cp));
+    if ((fn = clixon_plugin_api_get(cp)->ca_lockdb) != NULL){
+	if (fn(h, db, lock, id) < 0)
+	    goto done;  
+    }
+    retval = 0;
+ done:
+    return retval;
+}
+
+/*! Lock database status has changed status
+ * @param[in]  h    Clixon handle
+ * @param[in]  db   Database name (eg "running")
+ * @param[in]  lock Lock status: 0: unlocked, 1: locked
+ * @param[in]  id   Session id (of locker/unlocker)
+ * @retval    -1    Fatal error
+ * @retval     0    OK
+*/
+int
+clixon_plugin_lockdb_all(clicon_handle h,
+			 char         *db,
+			 int           lock,
+			 int           id
+			 )
+
+{
+    int              retval = -1;
+    clixon_plugin_t *cp = NULL;
+    
+    clicon_debug(1, "%s", __FUNCTION__);
+    while ((cp = clixon_plugin_each(h, cp)) != NULL) {
+	if (clixon_plugin_lockdb_one(cp, h, db, lock, id) < 0)
+	    goto done;
+    } 
+    retval = 0;
+ done:
+    return retval;
+}
+
 /*! Create and initialize a validate/commit transaction 
  * @retval  td     New alloced transaction, 
  * @retval  NULL   Error

apps/backend/clixon_backend_plugin.h

@@ -76,7 +76,10 @@ int clixon_plugin_pre_daemon_all(clicon_handle h);
 int clixon_plugin_daemon_all(clicon_handle h);
 
 int clixon_plugin_statedata_all(clicon_handle h, yang_stmt *yspec, cvec *nsc, char *xpath,
+				enum paging_status pagingstatus,
 				uint32_t offset, uint32_t limit, cxobj **xtop);
+int clixon_plugin_lockdb_all(clicon_handle h, char *db, int lock, int id);
+
 transaction_data_t * transaction_new(void);
 int transaction_free(transaction_data_t *);
 

apps/cli/cli_common.c

@@ -1043,6 +1043,8 @@ cli_notification_cb(int   s,
 	    default:
 		break;
 	    }
+	    if (cli_output_status() < 0)
+		break;
 	}
     }
     retval = 0;

apps/cli/cli_show.c

@@ -950,6 +950,8 @@ cli_pagination(clicon_handle h,
     }
     if ((nsc = xml_nsctx_init(prefix, namespace)) == NULL)
 	goto done;
+    if (clicon_rpc_lock(h, "running") < 0)
+	goto done;
     for (i = 0;; i++){
 	if (clicon_rpc_get_pageable_list(h, "running", xpath, nsc,
 					 CONTENT_ALL,
@@ -984,7 +986,11 @@ cli_pagination(clicon_handle h,
 	    default:
 		break;
 	    }
+	    if (cli_output_status() < 0)
+		break;
 	}
+	if (cli_output_status() < 0)
+	    break;
 	if (xlen != window) /* Break if fewer elements than requested */
 	    break;
 	if (xret){
@@ -995,7 +1001,9 @@ cli_pagination(clicon_handle h,
 	    free(xvec);
 	    xvec = NULL;
 	}
-    }
+    } /* for */
+    if (clicon_rpc_unlock(h, "running") < 0)
+	goto done;
     retval = 0;
  done:
     if (xvec)

example/main/example_backend.c

@@ -91,8 +91,13 @@ static char *_state_file = NULL;
  * Primarily for testing
  * Start backend with -- -siS <file>
  */
-static int _state_file_init = 0;
+static int _state_file_cached = 0;
-static cxobj *_state_xstate = NULL;
+
+/*! Cache control of read state file paging example,
+ * keep xml tree cache as long as db is locked
+ */
+static cxobj *_state_xml_cache = NULL; /* XML cache */
+static int _state_file_transaction = 0;
 
 /*! Variable to control module-specific upgrade callbacks.
  * If set, call test-case for upgrading ietf-interfaces, otherwise call 
@@ -343,10 +348,14 @@ example_copy_extra(clicon_handle h,            /* Clicon handle */
     return retval;
 }
 
-/*! Called to get state data from plugin
+/*! Called to get state data from plugin by programmatically adding state
+ *
  * @param[in]    h        Clicon handle
  * @param[in]    nsc      External XML namespace context, or NULL
  * @param[in]    xpath    String with XPATH syntax. or NULL for all
+ * @param[in]    paging   List pagination (not uses here)
+ * @param[in]    offset   Offset, for list pagination
+ * @param[in]    limit    Limit, for list pagination
  * @param[in]    xstate   XML tree, <config/> on entry. 
  * @retval       0        OK
  * @retval      -1        Error
@@ -360,11 +369,14 @@ example_copy_extra(clicon_handle h,            /* Clicon handle */
          }
        }
  * This yang snippet is present in clixon-example.yang for example.
+ * XXX paging for lock
+ * @see example_statefile  where state is read from file and also paging
  */
 int 
 example_statedata(clicon_handle h, 
 		  cvec         *nsc,
 		  char         *xpath,
+		  enum paging_status paging,
 		  uint32_t      offset,
 		  uint32_t      limit,
 		  cxobj        *xstate)
@@ -377,71 +389,11 @@ example_statedata(clicon_handle h,
     cxobj     *xt = NULL;
     char      *name;
     cvec      *nsc1 = NULL;
-    cvec      *nsc2 = NULL;
     yang_stmt *yspec = NULL;
-    FILE      *fp = NULL;
-    cxobj     *x1;
-    uint32_t   upper;
 
     if (!_state)
 	goto ok;
     yspec = clicon_dbspec_yang(h);
-    
-    /* If -S is set, then read state data from file, otherwise construct it programmatically */
-    if (_state_file){
-	if (_state_file_init){
-#if 0 /* This is just for a zero-copy version (only works once) */
-	    {
-		cxobj *xx = NULL;
-		while (xml_child_nr(_state_xstate)){
-		    xx = xml_child_i(_state_xstate,0);
-		    if (xml_addsub(xstate, xx) < 0)
-			goto done;
-		}
-	    }
-#else
-	    if (xml_copy(_state_xstate, xstate) < 0)
-		goto done;
-#endif
-	}
-	else{
-	    if ((fp = fopen(_state_file, "r")) == NULL){
-		clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
-		goto done;
-	    }
-	    if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL)
-		goto done;
-	    /* Note, does not care about xpath / list-pagination */
-	    if (clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL) < 0)
-		goto done;
-	    if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) 
-		goto done;
-	    if (limit == 0)
-		upper = xlen;
-	    else{
-		if ((upper = offset+limit)>xlen)
-		    upper = xlen;
-	    }
-	    for (i=offset; i<upper; i++){
-		if ((x1 = xvec[i]) == NULL)
-		    break;
-		xml_flag_set(x1, XML_FLAG_MARK);
-	    }
-	    /* Remove everything that is not marked */
-	    if (xml_tree_prune_flagged_sub(xt, XML_FLAG_MARK, 1, NULL) < 0)
-		goto done;
-	    if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)XML_FLAG_MARK) < 0)
-		goto done;
-	    if (xml_copy(xt, xstate) < 0)
-		goto done;
-	    if (xvec){
-		free(xvec);
-		xvec = 0;
-		xlen = 0;
-	    }
-	}
-    }
-    else {
     /* Example of statedata, in this case merging state data with 
      * state information. In this case adding dummy interface operation state
      * to configured interfaces.
@@ -488,16 +440,11 @@ example_statedata(clicon_handle h,
 	if (clixon_xml_parse_string(cbuf_get(cb), YB_NONE, NULL, &xstate, NULL) < 0)
 	    goto done;
     }
-    }
  ok:
     retval = 0;
  done:
-    if (fp)
-	fclose(fp);
     if (nsc1)
 	xml_nsctx_free(nsc1);
-    if (nsc2)
-	xml_nsctx_free(nsc2);
     if (xt)
 	xml_free(xt);
     if (cb)
@@ -507,6 +454,157 @@ example_statedata(clicon_handle h,
     return retval;
 }
 
+/*! Called to get state data from plugin by reading a file, also paging
+ *
+ * The example shows how to read and parse a state XML file, (which is cached in the -i case).
+ * Return the requested xpath / paging xstate by copying from the parsed state XML file
+ * @param[in]    h        Clicon handle
+ * @param[in]    nsc      External XML namespace context, or NULL
+ * @param[in]    xpath    String with XPATH syntax. or NULL for all
+ * @param[in]    paging   List pagination
+ * @param[in]    offset   Offset, for list pagination
+ * @param[in]    limit    Limit, for list pagination
+ * @param[in]    xstate   XML tree, <config/> on entry. Copy to this
+ * @retval       0        OK
+ * @retval      -1        Error
+ * @see xmldb_get
+ * @see example_statefile  where state is programmatically added
+ */
+int 
+example_statefile(clicon_handle      h, 
+		  cvec              *nsc,
+		  char              *xpath,
+		  enum paging_status paging,
+		  uint32_t           offset,
+		  uint32_t           limit,
+		  cxobj             *xstate)
+{
+    int        retval = -1;
+    cxobj    **xvec = NULL;
+    size_t     xlen = 0;
+    int        i;
+    cxobj     *xt = NULL;
+    yang_stmt *yspec = NULL;
+    FILE      *fp = NULL;
+    cxobj     *x1;
+    uint32_t   lower;
+    uint32_t   upper;
+    int        ret;
+
+    /* If -S is set, then read state data from file */
+    if (!_state || !_state_file)
+	goto ok;
+    yspec = clicon_dbspec_yang(h);
+    /* Read state file if either not cached, or the cache is NULL */
+    if (_state_file_cached == 0 ||
+	_state_xml_cache == NULL){
+	if ((fp = fopen(_state_file, "r")) == NULL){
+	    clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
+	    goto done;
+	}
+	if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL)
+	    goto done;
+	if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &xt, NULL)) < 0)
+	    goto done;
+#if 0
+	if (ret == 0){
+	    if (clixon_netconf_internal_error(xstate,
+					      ". Internal error, state callback returned invalid XML",
+					      NULL) < 0)
+		goto done;
+	    goto ok;	    
+	}
+#endif
+	if (_state_file_cached)
+	    _state_xml_cache = xt;
+    }
+    if (_state_file_cached)
+	xt = _state_xml_cache;
+    if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) 
+	goto done;	
+    switch (paging){
+    case PAGING_NONE:
+	lower = 0;
+	upper = xlen;
+	break;
+    case PAGING_STATELESS:
+    case PAGING_LOCK:
+	lower = offset;
+	if (limit == 0)
+	    upper = xlen;
+	else{
+	    if ((upper = offset+limit)>xlen)
+		upper = xlen;
+	}
+	break;
+    }
+    /* Mark elements to copy:
+     * For every node found in x0, mark the tree as changed 
+     */
+    for (i=lower; i<upper; i++){
+	if ((x1 = xvec[i]) == NULL)
+	    break;
+	xml_flag_set(x1, XML_FLAG_MARK);
+	xml_apply_ancestor(x1, (xml_applyfn_t*)xml_flag_set, (void*)XML_FLAG_CHANGE);
+    }
+    if (xml_copy_marked(xt, xstate) < 0) /* Copy the marked elements */
+	goto done;
+    /* Unmark original tree */
+    if (xml_apply(xt, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
+	goto done;
+    /* Unmark returned state tree */
+    if (xml_apply(xstate, CX_ELMNT, (xml_applyfn_t*)xml_flag_reset, (void*)(XML_FLAG_MARK|XML_FLAG_CHANGE)) < 0)
+	goto done;
+    if (_state_file_cached)
+	xt = NULL; /* ensure cache is not cleared */
+    if (paging ==  PAGING_LOCK)
+	_state_file_transaction++;
+ ok:
+    retval = 0;
+ done:
+    if (fp)
+	fclose(fp);
+    if (xt)
+	xml_free(xt);
+    if (xvec)
+	free(xvec);
+    return retval;
+}
+
+/*! Lock databse status has changed status
+ * @param[in]  h    Clixon handle
+ * @param[in]  db   Database name (eg "running")
+ * @param[in]  lock Lock status: 0: unlocked, 1: locked
+ * @param[in]  id   Session id (of locker/unlocker)
+ * @retval    -1    Fatal error
+ * @retval     0    OK
+*/
+int
+example_lockdb(clicon_handle h,
+	       char         *db,
+	       int           lock,
+	       int           id)
+{
+    int retval = -1;
+
+    clicon_debug(1, "%s Lock callback: db%s: locked:%d", __FUNCTION__, db, lock);
+
+    /* Part of cached paging example
+     */
+    if (strcmp(db, "running") == 0 && lock == 0 &&
+	_state && _state_file && _state_file_cached && _state_file_transaction){
+	if (_state_xml_cache){
+	    xml_free(_state_xml_cache);
+	    _state_xml_cache = NULL;
+	}
+	_state_file_transaction = 0;
+    }
+
+    retval = 0;
+    // done:
+    return retval;
+}
+
 /*! Callback for yang extensions example:e4
  * 
  * @param[in] h    Clixon handle
@@ -982,7 +1080,7 @@ example_start(clicon_handle h)
 /*! Plugin daemon.
  * @param[in]  h     Clicon handle
  *
- * plugin_daemon is called once after damonization has been made but before lowering of privileges
+ * plugin_daemon is called once after daemonization has been made but before lowering of privileges
  * the main event loop is entered. 
  */
 int
@@ -994,19 +1092,14 @@ example_daemon(clicon_handle h)
     yang_stmt *yspec;
 
     /* Read state file (or should this be in init/start?) */
-    if (_state && _state_file && _state_file_init){
+    if (_state && _state_file && _state_file_cached){
 	yspec = clicon_dbspec_yang(h);
 	if ((fp = fopen(_state_file, "r")) == NULL){
 	    clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
 	    goto done;
 	}
-	if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &_state_xstate, NULL)) < 0)
+	if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, &_state_xml_cache, NULL)) < 1)
 	    goto done;
-	if (ret == 0){
-	    fprintf(stderr, "%s error\n", __FUNCTION__);
-	    goto done;
-	}
-	fprintf(stderr, "%s done\n", __FUNCTION__);
     }
     retval = 0;
  done:
@@ -1032,7 +1125,8 @@ static clixon_plugin_api api = {
     .ca_extension=example_extension,        /* yang extensions */
     .ca_daemon=example_daemon,              /* daemon */
     .ca_reset=example_reset,                /* reset */
-    .ca_statedata2=example_statedata,      /* statedata2 */
+    .ca_statedata2=example_statedata,       /* statedata2 : Note fn is switched if -sS <file> */
+    .ca_lockdb=example_lockdb,              /* Database lock changed state */
     .ca_trans_begin=main_begin,             /* trans begin */
     .ca_trans_validate=main_validate,       /* trans validate */
     .ca_trans_complete=main_complete,       /* trans complete */
@@ -1076,9 +1170,10 @@ clixon_plugin_init(clicon_handle h)
 	    break;
 	case 'S': /* state file (requires -s) */
 	    _state_file = optarg;
+	    api.ca_statedata2 = example_statefile; /* Switch state data callback */
 	    break;
 	case 'i': /* read state file on init not by request (requires -sS <file> */
-	    _state_file_init = 1;
+	    _state_file_cached = 1;
 	    break;
        case 'u': /* module-specific upgrade */
            _module_upgrade = 1;

lib/clixon/clixon_data.h

@@ -52,7 +52,7 @@
  *       modified, and these changes have not been committed or rolled back.
  */
 typedef struct {
-    uint32_t  de_id;       /* session id */
+    uint32_t  de_id;       /* session id keeps lock */
     cxobj    *de_xml;      /* cache */
     int       de_modified; /* Dirty since loaded/copied/committed/etc XXX:nocache? */
     int       de_empty;    /* Empty on read from file, xmldb_readfile and xmldb_put sets it */

lib/clixon/clixon_plugin.h

@@ -206,11 +206,51 @@ typedef int (plgreset_t)(clicon_handle h, const char *db);
  */
 typedef int (plgstatedata_t)(clicon_handle h, cvec *nsc, char *xpath, cxobj *xtop);
 
-/*! Temporary new statedata callback */
+/*! List paging status in the plugin state data callback
+ *
+ * List paging is either enabled or not.
+ * If paging is enabled, the xpath addresses a list/ leaf-list and the plugin should return 
+ * entries according to the values of offset and limit.
+ * Paging can use a lock/transaction mechanism 
+ * If locking is not used, the plugin cannot expect more paging calls, and no state or caching
+ * should be used
+ * If locking is used, the paging is part of a session transaction and the plugin may cache
+ * state (such as a cache) and can expect more paging calls until the running db-lock is released, 
+ * (see ca_lockdb)
+ * The transaction is the regular lock/unlock db of running-db of a specific session.
+ */
+enum paging_status{
+    PAGING_NONE,       /* No list paging: limit/offset are no-ops */
+    PAGING_STATELESS,  /* Stateless list paging, dont expect more paging calls */
+    PAGING_LOCK        /* Transactional list paging, can expect more paging until lock release */
+};
+
+/* Plugin statedata
+ * @param[in]  Clicon handle
+ * @param[in]  xpath      Part of state requested
+ * @param[in]  nsc        XPATH namespace context.
+ * @param[in]  pagination List pagination: 0: No, 1: begin/next, 2: end
+ * @param[in]  offset     Offset, for list pagination
+ * @param[in]  limit      Limit, for list pagination
+ * @param[out] xtop       XML tree where statedata is added
+ * @retval    -1          Fatal error
+ * @retval     0          OK
+ */
 typedef int (plgstatedata2_t)(clicon_handle h, cvec *nsc, char *xpath,
+			      enum paging_status pagination,
 			      uint32_t offset, uint32_t limit, 
 			      cxobj *xtop);
 
+/*! Lock databse status has changed status
+ * @param[in]  h    Clixon handle
+ * @param[in]  db   Database name (eg "running")
+ * @param[in]  lock Lock status: 0: unlocked, 1: locked
+ * @param[in]  id   Session id (of locker/unlocker)
+ * @retval    -1   Fatal error
+ * @retval     0   OK
+*/
+typedef int (plglockdb_t)(clicon_handle h, char *db, int lock, int id);
+
 /* Transaction-data type
  * @see clixon_backend_transaction.h for full transaction API 
  */
@@ -283,6 +323,7 @@ struct clixon_plugin_api{
 	    plgreset_t       *cb_reset;          /* Reset system status */
 	    plgstatedata_t   *cb_statedata;      /* Get state data from plugin (backend only) */
 	    plgstatedata2_t  *cb_statedata2;     /* Get state data from plugin (backend only) */
+	    plglockdb_t      *cb_lockdb;         /* Database lock changed state */
 	    trans_cb_t       *cb_trans_begin;	 /* Transaction start */
 	    trans_cb_t       *cb_trans_validate; /* Transaction validation */
 	    trans_cb_t       *cb_trans_complete; /* Transaction validation complete */
@@ -305,6 +346,7 @@ struct clixon_plugin_api{
 #define ca_reset          u.cau_backend.cb_reset
 #define ca_statedata      u.cau_backend.cb_statedata
 #define ca_statedata2     u.cau_backend.cb_statedata2
+#define ca_lockdb         u.cau_backend.cb_lockdb
 #define ca_trans_begin    u.cau_backend.cb_trans_begin
 #define ca_trans_validate u.cau_backend.cb_trans_validate
 #define ca_trans_complete u.cau_backend.cb_trans_complete

lib/src/clixon_datastore.c

@@ -314,14 +314,17 @@ xmldb_unlock_all(clicon_handle h,
     int       i;
     db_elmnt *de;
 
+    /* get all db:s */
     if (clicon_hash_keys(clicon_db_elmnt(h), &keys, &klen) < 0)
 	goto done;
-    for (i = 0; i < klen; i++) 
+    /* Identify the ones locked by client id */
+    for (i = 0; i < klen; i++) {
 	if ((de = clicon_db_elmnt_get(h, keys[i])) != NULL &&
 	    de->de_id == id){
 	    de->de_id = 0;
 	    clicon_db_elmnt_set(h, keys[i], de);
 	}
+    }
     retval = 0;
  done:
     if (keys)

lib/src/clixon_datastore_read.c

@@ -794,6 +794,8 @@ xmldb_get_cache(clicon_handle    h,
 	 * No, argument against: we may want to have a semantically wrong file and wish to edit?
 	 */
 	de0.de_xml = x0t;
+	if (de)
+	    de0.de_id = de->de_id;
 	clicon_db_elmnt_set(h, db, &de0); /* Content is copied */
     } /* x0t == NULL */
     else
@@ -955,6 +957,8 @@ xmldb_get_zerocopy(clicon_handle    h,
 	 * No, argument against: we may want to have a semantically wrong file and wish to edit?
 	 */
 	de0.de_xml = x0t;
+	if (de)
+	    de0.de_id = de->de_id;
 	clicon_db_elmnt_set(h, db, &de0);
     } /* x0t == NULL */
     else

test/test_paging_state.sh

@@ -7,8 +7,8 @@
 # Magic line must be first in script (see README.md)
 s="$_" ; . ./lib.sh || if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
-echo "...skipped: Must run interactvely"
+#echo "...skipped: Must run interactvely"
-if [ "$s" = $0 ]; then exit 0; else return 0; fi
+#if [ "$s" = $0 ]; then exit 0; else return 0; fi
 
 APPNAME=example
 
@@ -50,51 +50,11 @@ EOF
 # See draft-wwlh-netconf-list-pagination-00 A.2 (only stats and audit-log)
 # XXX members not currently used, only audit-logs as generated below
 cat<<EOF > $fstate
-<members xmlns="http://example.com/ns/example-social">
-   <member>
-      <member-id>alice</member-id>
-      <stats>
-          <joined>2020-07-08T12:38:32Z</joined>
-	  <membership-level>admin</membership-level>
-          <last-activity>2021-04-01T02:51:11Z</last-activity>
-      </stats>
-   </member>
-   <member>
-      <member-id>bob</member-id>
-      <stats>
-          <joined>2020-08-14T03:30:00Z</joined>
-	  <membership-level>standard</membership-level>
-          <last-activity>2020-08-14T03:34:30Z</last-activity>
-      </stats>
-   </member>
-   <member>
-      <member-id>eric</member-id>
-      <stats>
-          <joined>2020-09-17T19:38:32Z</joined>
-	  <membership-level>pro</membership-level>
-          <last-activity>2020-09-17T18:02:04Z</last-activity>
-      </stats>
-   </member>
-   <member>
-      <member-id>lin</member-id>
-      <stats>
-          <joined>2020-07-09T12:38:32Z</joined>
-	  <membership-level>standard</membership-level>
-          <last-activity>2021-04-01T02:51:11Z</last-activity>
-      </stats>
-   </member>
-   <member>
-      <member-id>joe</member-id>
-      <stats>
-          <joined>2020-10-08T12:38:32Z</joined>
-	  <membership-level>pro</membership-level>
-          <last-activity>2021-04-01T02:51:11Z</last-activity>
-      </stats>
-   </member>
-</members>
 EOF
 
 # Append generated state data to $fstate file
+# Generation of random timestamps (not used)
+# and succesive bob$i member-ids
 new "generate state with $perfnr list entries"
 echo "<audit-logs xmlns=\"http://example.com/ns/example-social\">" >> $fstate
 for (( i=0; i<$perfnr; i++ )); do  
@@ -103,17 +63,15 @@ for (( i=0; i<$perfnr; i++ )); do
     day=$(( ( RANDOM % 10 ) ))
     hour=$(( ( RANDOM % 10 ) ))
     echo "    <timestamp>2020-0$mon-0$dayT0$hour:48:11Z</timestamp>" >> $fstate
-    echo "    <member-id>bob</member-id>" >> $fstate
+    echo "    <member-id>bob$i</member-id>" >> $fstate
-    ip1=$(( ( RANDOM % 255 ) ))
+    echo "    <source-ip>192.168.1.32</source-ip>" >> $fstate
-    ip2=$(( ( RANDOM % 255 ) ))
-    echo "    <source-ip>192.168.$ip1.$ip2</source-ip>" >> $fstate
     echo "    <request>POST</request>" >> $fstate
     echo "    <outcome>true</outcome>" >> $fstate
     echo "  </audit-log>" >> $fstate
 done
 echo -n "</audit-logs>" >> $fstate # No CR
 
-new "test params: -f $cfg -s init -- -sS $fstate"
+new "test params: -f $cfg -s init -- -siS $fstate"
 
 if [ $BE -ne 0 ]; then
     new "kill old backend"
@@ -123,8 +81,8 @@ if [ $BE -ne 0 ]; then
     fi
     sudo pkill -f clixon_backend # to be sure
 
-    new "start backend -s init -f $cfg -- -sS $fstate"
+    new "start backend -s init -f $cfg -- -siS $fstate"
-    start_backend -s init -f $cfg -- -sS $fstate
+    start_backend -s init -f $cfg -- -siS $fstate
 fi
 
 new "wait backend"